Pileus Git - ~andy/linux/blob - drivers/net/ethernet/intel/i40e/i40e_main.c

   1 /*******************************************************************************
   2  *
   3  * Intel Ethernet Controller XL710 Family Linux Driver
   4  * Copyright(c) 2013 - 2014 Intel Corporation.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms and conditions of the GNU General Public License,
   8  * version 2, as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13  * more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program.  If not, see <http://www.gnu.org/licenses/>.
  17  *
  18  * The full GNU General Public License is included in this distribution in
  19  * the file called "COPYING".
  20  *
  21  * Contact Information:
  22  * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
  23  * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  24  *
  25  ******************************************************************************/
  26
  27 /* Local includes */
  28 #include "i40e.h"
  29 #ifdef CONFIG_I40E_VXLAN
  30 #include <net/vxlan.h>
  31 #endif
  32
  33 const char i40e_driver_name[] = "i40e";
  34 static const char i40e_driver_string[] =
  35                         "Intel(R) Ethernet Connection XL710 Network Driver";
  36
  37 #define DRV_KERN "-k"
  38
  39 #define DRV_VERSION_MAJOR 0
  40 #define DRV_VERSION_MINOR 3
  41 #define DRV_VERSION_BUILD 30
  42 #define DRV_VERSION __stringify(DRV_VERSION_MAJOR) "." \
  43              __stringify(DRV_VERSION_MINOR) "." \
  44              __stringify(DRV_VERSION_BUILD)    DRV_KERN
  45 const char i40e_driver_version_str[] = DRV_VERSION;
  46 static const char i40e_copyright[] = "Copyright (c) 2013 Intel Corporation.";
  47
  48 /* a bit of forward declarations */
  49 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi);
  50 static void i40e_handle_reset_warning(struct i40e_pf *pf);
  51 static int i40e_add_vsi(struct i40e_vsi *vsi);
  52 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi);
  53 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit);
  54 static int i40e_setup_misc_vector(struct i40e_pf *pf);
  55 static void i40e_determine_queue_usage(struct i40e_pf *pf);
  56 static int i40e_setup_pf_filter_control(struct i40e_pf *pf);
  57
  58 /* i40e_pci_tbl - PCI Device ID Table
  59  *
  60  * Last entry must be all 0s
  61  *
  62  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
  63  *   Class, Class Mask, private data (not used) }
  64  */
  65 static DEFINE_PCI_DEVICE_TABLE(i40e_pci_tbl) = {
  66         {PCI_VDEVICE(INTEL, I40E_SFP_XL710_DEVICE_ID), 0},
  67         {PCI_VDEVICE(INTEL, I40E_SFP_X710_DEVICE_ID), 0},
  68         {PCI_VDEVICE(INTEL, I40E_QEMU_DEVICE_ID), 0},
  69         {PCI_VDEVICE(INTEL, I40E_KX_A_DEVICE_ID), 0},
  70         {PCI_VDEVICE(INTEL, I40E_KX_B_DEVICE_ID), 0},
  71         {PCI_VDEVICE(INTEL, I40E_KX_C_DEVICE_ID), 0},
  72         {PCI_VDEVICE(INTEL, I40E_KX_D_DEVICE_ID), 0},
  73         {PCI_VDEVICE(INTEL, I40E_QSFP_A_DEVICE_ID), 0},
  74         {PCI_VDEVICE(INTEL, I40E_QSFP_B_DEVICE_ID), 0},
  75         {PCI_VDEVICE(INTEL, I40E_QSFP_C_DEVICE_ID), 0},
  76         /* required last entry */
  77         {0, }
  78 };
  79 MODULE_DEVICE_TABLE(pci, i40e_pci_tbl);
  80
  81 #define I40E_MAX_VF_COUNT 128
  82 static int debug = -1;
  83 module_param(debug, int, 0);
  84 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
  85
  86 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
  87 MODULE_DESCRIPTION("Intel(R) Ethernet Connection XL710 Network Driver");
  88 MODULE_LICENSE("GPL");
  89 MODULE_VERSION(DRV_VERSION);
  90
  91 /**
  92  * i40e_allocate_dma_mem_d - OS specific memory alloc for shared code
  93  * @hw:   pointer to the HW structure
  94  * @mem:  ptr to mem struct to fill out
  95  * @size: size of memory requested
  96  * @alignment: what to align the allocation to
  97  **/
  98 int i40e_allocate_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem,
  99                             u64 size, u32 alignment)
 100 {
 101         struct i40e_pf *pf = (struct i40e_pf *)hw->back;
 102
 103         mem->size = ALIGN(size, alignment);
 104         mem->va = dma_zalloc_coherent(&pf->pdev->dev, mem->size,
 105                                       &mem->pa, GFP_KERNEL);
 106         if (!mem->va)
 107                 return -ENOMEM;
 108
 109         return 0;
 110 }
 111
 112 /**
 113  * i40e_free_dma_mem_d - OS specific memory free for shared code
 114  * @hw:   pointer to the HW structure
 115  * @mem:  ptr to mem struct to free
 116  **/
 117 int i40e_free_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem)
 118 {
 119         struct i40e_pf *pf = (struct i40e_pf *)hw->back;
 120
 121         dma_free_coherent(&pf->pdev->dev, mem->size, mem->va, mem->pa);
 122         mem->va = NULL;
 123         mem->pa = 0;
 124         mem->size = 0;
 125
 126         return 0;
 127 }
 128
 129 /**
 130  * i40e_allocate_virt_mem_d - OS specific memory alloc for shared code
 131  * @hw:   pointer to the HW structure
 132  * @mem:  ptr to mem struct to fill out
 133  * @size: size of memory requested
 134  **/
 135 int i40e_allocate_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem,
 136                              u32 size)
 137 {
 138         mem->size = size;
 139         mem->va = kzalloc(size, GFP_KERNEL);
 140
 141         if (!mem->va)
 142                 return -ENOMEM;
 143
 144         return 0;
 145 }
 146
 147 /**
 148  * i40e_free_virt_mem_d - OS specific memory free for shared code
 149  * @hw:   pointer to the HW structure
 150  * @mem:  ptr to mem struct to free
 151  **/
 152 int i40e_free_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem)
 153 {
 154         /* it's ok to kfree a NULL pointer */
 155         kfree(mem->va);
 156         mem->va = NULL;
 157         mem->size = 0;
 158
 159         return 0;
 160 }
 161
 162 /**
 163  * i40e_get_lump - find a lump of free generic resource
 164  * @pf: board private structure
 165  * @pile: the pile of resource to search
 166  * @needed: the number of items needed
 167  * @id: an owner id to stick on the items assigned
 168  *
 169  * Returns the base item index of the lump, or negative for error
 170  *
 171  * The search_hint trick and lack of advanced fit-finding only work
 172  * because we're highly likely to have all the same size lump requests.
 173  * Linear search time and any fragmentation should be minimal.
 174  **/
 175 static int i40e_get_lump(struct i40e_pf *pf, struct i40e_lump_tracking *pile,
 176                          u16 needed, u16 id)
 177 {
 178         int ret = -ENOMEM;
 179         int i, j;
 180
 181         if (!pile || needed == 0 || id >= I40E_PILE_VALID_BIT) {
 182                 dev_info(&pf->pdev->dev,
 183                          "param err: pile=%p needed=%d id=0x%04x\n",
 184                          pile, needed, id);
 185                 return -EINVAL;
 186         }
 187
 188         /* start the linear search with an imperfect hint */
 189         i = pile->search_hint;
 190         while (i < pile->num_entries) {
 191                 /* skip already allocated entries */
 192                 if (pile->list[i] & I40E_PILE_VALID_BIT) {
 193                         i++;
 194                         continue;
 195                 }
 196
 197                 /* do we have enough in this lump? */
 198                 for (j = 0; (j < needed) && ((i+j) < pile->num_entries); j++) {
 199                         if (pile->list[i+j] & I40E_PILE_VALID_BIT)
 200                                 break;
 201                 }
 202
 203                 if (j == needed) {
 204                         /* there was enough, so assign it to the requestor */
 205                         for (j = 0; j < needed; j++)
 206                                 pile->list[i+j] = id | I40E_PILE_VALID_BIT;
 207                         ret = i;
 208                         pile->search_hint = i + j;
 209                         break;
 210                 } else {
 211                         /* not enough, so skip over it and continue looking */
 212                         i += j;
 213                 }
 214         }
 215
 216         return ret;
 217 }
 218
 219 /**
 220  * i40e_put_lump - return a lump of generic resource
 221  * @pile: the pile of resource to search
 222  * @index: the base item index
 223  * @id: the owner id of the items assigned
 224  *
 225  * Returns the count of items in the lump
 226  **/
 227 static int i40e_put_lump(struct i40e_lump_tracking *pile, u16 index, u16 id)
 228 {
 229         int valid_id = (id | I40E_PILE_VALID_BIT);
 230         int count = 0;
 231         int i;
 232
 233         if (!pile || index >= pile->num_entries)
 234                 return -EINVAL;
 235
 236         for (i = index;
 237              i < pile->num_entries && pile->list[i] == valid_id;
 238              i++) {
 239                 pile->list[i] = 0;
 240                 count++;
 241         }
 242
 243         if (count && index < pile->search_hint)
 244                 pile->search_hint = index;
 245
 246         return count;
 247 }
 248
 249 /**
 250  * i40e_service_event_schedule - Schedule the service task to wake up
 251  * @pf: board private structure
 252  *
 253  * If not already scheduled, this puts the task into the work queue
 254  **/
 255 static void i40e_service_event_schedule(struct i40e_pf *pf)
 256 {
 257         if (!test_bit(__I40E_DOWN, &pf->state) &&
 258             !test_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state) &&
 259             !test_and_set_bit(__I40E_SERVICE_SCHED, &pf->state))
 260                 schedule_work(&pf->service_task);
 261 }
 262
 263 /**
 264  * i40e_tx_timeout - Respond to a Tx Hang
 265  * @netdev: network interface device structure
 266  *
 267  * If any port has noticed a Tx timeout, it is likely that the whole
 268  * device is munged, not just the one netdev port, so go for the full
 269  * reset.
 270  **/
 271 static void i40e_tx_timeout(struct net_device *netdev)
 272 {
 273         struct i40e_netdev_priv *np = netdev_priv(netdev);
 274         struct i40e_vsi *vsi = np->vsi;
 275         struct i40e_pf *pf = vsi->back;
 276
 277         pf->tx_timeout_count++;
 278
 279         if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ*20)))
 280                 pf->tx_timeout_recovery_level = 0;
 281         pf->tx_timeout_last_recovery = jiffies;
 282         netdev_info(netdev, "tx_timeout recovery level %d\n",
 283                     pf->tx_timeout_recovery_level);
 284
 285         switch (pf->tx_timeout_recovery_level) {
 286         case 0:
 287                 /* disable and re-enable queues for the VSI */
 288                 if (in_interrupt()) {
 289                         set_bit(__I40E_REINIT_REQUESTED, &pf->state);
 290                         set_bit(__I40E_REINIT_REQUESTED, &vsi->state);
 291                 } else {
 292                         i40e_vsi_reinit_locked(vsi);
 293                 }
 294                 break;
 295         case 1:
 296                 set_bit(__I40E_PF_RESET_REQUESTED, &pf->state);
 297                 break;
 298         case 2:
 299                 set_bit(__I40E_CORE_RESET_REQUESTED, &pf->state);
 300                 break;
 301         case 3:
 302                 set_bit(__I40E_GLOBAL_RESET_REQUESTED, &pf->state);
 303                 break;
 304         default:
 305                 netdev_err(netdev, "tx_timeout recovery unsuccessful\n");
 306                 i40e_down(vsi);
 307                 break;
 308         }
 309         i40e_service_event_schedule(pf);
 310         pf->tx_timeout_recovery_level++;
 311 }
 312
 313 /**
 314  * i40e_release_rx_desc - Store the new tail and head values
 315  * @rx_ring: ring to bump
 316  * @val: new head index
 317  **/
 318 static inline void i40e_release_rx_desc(struct i40e_ring *rx_ring, u32 val)
 319 {
 320         rx_ring->next_to_use = val;
 321
 322         /* Force memory writes to complete before letting h/w
 323          * know there are new descriptors to fetch.  (Only
 324          * applicable for weak-ordered memory model archs,
 325          * such as IA-64).
 326          */
 327         wmb();
 328         writel(val, rx_ring->tail);
 329 }
 330
 331 /**
 332  * i40e_get_vsi_stats_struct - Get System Network Statistics
 333  * @vsi: the VSI we care about
 334  *
 335  * Returns the address of the device statistics structure.
 336  * The statistics are actually updated from the service task.
 337  **/
 338 struct rtnl_link_stats64 *i40e_get_vsi_stats_struct(struct i40e_vsi *vsi)
 339 {
 340         return &vsi->net_stats;
 341 }
 342
 343 /**
 344  * i40e_get_netdev_stats_struct - Get statistics for netdev interface
 345  * @netdev: network interface device structure
 346  *
 347  * Returns the address of the device statistics structure.
 348  * The statistics are actually updated from the service task.
 349  **/
 350 static struct rtnl_link_stats64 *i40e_get_netdev_stats_struct(
 351                                              struct net_device *netdev,
 352                                              struct rtnl_link_stats64 *stats)
 353 {
 354         struct i40e_netdev_priv *np = netdev_priv(netdev);
 355         struct i40e_vsi *vsi = np->vsi;
 356         struct rtnl_link_stats64 *vsi_stats = i40e_get_vsi_stats_struct(vsi);
 357         int i;
 358
 359         if (test_bit(__I40E_DOWN, &vsi->state))
 360                 return stats;
 361
 362         if (!vsi->tx_rings)
 363                 return stats;
 364
 365         rcu_read_lock();
 366         for (i = 0; i < vsi->num_queue_pairs; i++) {
 367                 struct i40e_ring *tx_ring, *rx_ring;
 368                 u64 bytes, packets;
 369                 unsigned int start;
 370
 371                 tx_ring = ACCESS_ONCE(vsi->tx_rings[i]);
 372                 if (!tx_ring)
 373                         continue;
 374
 375                 do {
 376                         start = u64_stats_fetch_begin_bh(&tx_ring->syncp);
 377                         packets = tx_ring->stats.packets;
 378                         bytes   = tx_ring->stats.bytes;
 379                 } while (u64_stats_fetch_retry_bh(&tx_ring->syncp, start));
 380
 381                 stats->tx_packets += packets;
 382                 stats->tx_bytes   += bytes;
 383                 rx_ring = &tx_ring[1];
 384
 385                 do {
 386                         start = u64_stats_fetch_begin_bh(&rx_ring->syncp);
 387                         packets = rx_ring->stats.packets;
 388                         bytes   = rx_ring->stats.bytes;
 389                 } while (u64_stats_fetch_retry_bh(&rx_ring->syncp, start));
 390
 391                 stats->rx_packets += packets;
 392                 stats->rx_bytes   += bytes;
 393         }
 394         rcu_read_unlock();
 395
 396         /* following stats updated by ixgbe_watchdog_task() */
 397         stats->multicast        = vsi_stats->multicast;
 398         stats->tx_errors        = vsi_stats->tx_errors;
 399         stats->tx_dropped       = vsi_stats->tx_dropped;
 400         stats->rx_errors        = vsi_stats->rx_errors;
 401         stats->rx_crc_errors    = vsi_stats->rx_crc_errors;
 402         stats->rx_length_errors = vsi_stats->rx_length_errors;
 403
 404         return stats;
 405 }
 406
 407 /**
 408  * i40e_vsi_reset_stats - Resets all stats of the given vsi
 409  * @vsi: the VSI to have its stats reset
 410  **/
 411 void i40e_vsi_reset_stats(struct i40e_vsi *vsi)
 412 {
 413         struct rtnl_link_stats64 *ns;
 414         int i;
 415
 416         if (!vsi)
 417                 return;
 418
 419         ns = i40e_get_vsi_stats_struct(vsi);
 420         memset(ns, 0, sizeof(*ns));
 421         memset(&vsi->net_stats_offsets, 0, sizeof(vsi->net_stats_offsets));
 422         memset(&vsi->eth_stats, 0, sizeof(vsi->eth_stats));
 423         memset(&vsi->eth_stats_offsets, 0, sizeof(vsi->eth_stats_offsets));
 424         if (vsi->rx_rings && vsi->rx_rings[0]) {
 425                 for (i = 0; i < vsi->num_queue_pairs; i++) {
 426                         memset(&vsi->rx_rings[i]->stats, 0 ,
 427                                sizeof(vsi->rx_rings[i]->stats));
 428                         memset(&vsi->rx_rings[i]->rx_stats, 0 ,
 429                                sizeof(vsi->rx_rings[i]->rx_stats));
 430                         memset(&vsi->tx_rings[i]->stats, 0 ,
 431                                sizeof(vsi->tx_rings[i]->stats));
 432                         memset(&vsi->tx_rings[i]->tx_stats, 0,
 433                                sizeof(vsi->tx_rings[i]->tx_stats));
 434                 }
 435         }
 436         vsi->stat_offsets_loaded = false;
 437 }
 438
 439 /**
 440  * i40e_pf_reset_stats - Reset all of the stats for the given pf
 441  * @pf: the PF to be reset
 442  **/
 443 void i40e_pf_reset_stats(struct i40e_pf *pf)
 444 {
 445         memset(&pf->stats, 0, sizeof(pf->stats));
 446         memset(&pf->stats_offsets, 0, sizeof(pf->stats_offsets));
 447         pf->stat_offsets_loaded = false;
 448 }
 449
 450 /**
 451  * i40e_stat_update48 - read and update a 48 bit stat from the chip
 452  * @hw: ptr to the hardware info
 453  * @hireg: the high 32 bit reg to read
 454  * @loreg: the low 32 bit reg to read
 455  * @offset_loaded: has the initial offset been loaded yet
 456  * @offset: ptr to current offset value
 457  * @stat: ptr to the stat
 458  *
 459  * Since the device stats are not reset at PFReset, they likely will not
 460  * be zeroed when the driver starts.  We'll save the first values read
 461  * and use them as offsets to be subtracted from the raw values in order
 462  * to report stats that count from zero.  In the process, we also manage
 463  * the potential roll-over.
 464  **/
 465 static void i40e_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg,
 466                                bool offset_loaded, u64 *offset, u64 *stat)
 467 {
 468         u64 new_data;
 469
 470         if (hw->device_id == I40E_QEMU_DEVICE_ID) {
 471                 new_data = rd32(hw, loreg);
 472                 new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32;
 473         } else {
 474                 new_data = rd64(hw, loreg);
 475         }
 476         if (!offset_loaded)
 477                 *offset = new_data;
 478         if (likely(new_data >= *offset))
 479                 *stat = new_data - *offset;
 480         else
 481                 *stat = (new_data + ((u64)1 << 48)) - *offset;
 482         *stat &= 0xFFFFFFFFFFFFULL;
 483 }
 484
 485 /**
 486  * i40e_stat_update32 - read and update a 32 bit stat from the chip
 487  * @hw: ptr to the hardware info
 488  * @reg: the hw reg to read
 489  * @offset_loaded: has the initial offset been loaded yet
 490  * @offset: ptr to current offset value
 491  * @stat: ptr to the stat
 492  **/
 493 static void i40e_stat_update32(struct i40e_hw *hw, u32 reg,
 494                                bool offset_loaded, u64 *offset, u64 *stat)
 495 {
 496         u32 new_data;
 497
 498         new_data = rd32(hw, reg);
 499         if (!offset_loaded)
 500                 *offset = new_data;
 501         if (likely(new_data >= *offset))
 502                 *stat = (u32)(new_data - *offset);
 503         else
 504                 *stat = (u32)((new_data + ((u64)1 << 32)) - *offset);
 505 }
 506
 507 /**
 508  * i40e_update_eth_stats - Update VSI-specific ethernet statistics counters.
 509  * @vsi: the VSI to be updated
 510  **/
 511 void i40e_update_eth_stats(struct i40e_vsi *vsi)
 512 {
 513         int stat_idx = le16_to_cpu(vsi->info.stat_counter_idx);
 514         struct i40e_pf *pf = vsi->back;
 515         struct i40e_hw *hw = &pf->hw;
 516         struct i40e_eth_stats *oes;
 517         struct i40e_eth_stats *es;     /* device's eth stats */
 518
 519         es = &vsi->eth_stats;
 520         oes = &vsi->eth_stats_offsets;
 521
 522         /* Gather up the stats that the hw collects */
 523         i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx),
 524                            vsi->stat_offsets_loaded,
 525                            &oes->tx_errors, &es->tx_errors);
 526         i40e_stat_update32(hw, I40E_GLV_RDPC(stat_idx),
 527                            vsi->stat_offsets_loaded,
 528                            &oes->rx_discards, &es->rx_discards);
 529
 530         i40e_stat_update48(hw, I40E_GLV_GORCH(stat_idx),
 531                            I40E_GLV_GORCL(stat_idx),
 532                            vsi->stat_offsets_loaded,
 533                            &oes->rx_bytes, &es->rx_bytes);
 534         i40e_stat_update48(hw, I40E_GLV_UPRCH(stat_idx),
 535                            I40E_GLV_UPRCL(stat_idx),
 536                            vsi->stat_offsets_loaded,
 537                            &oes->rx_unicast, &es->rx_unicast);
 538         i40e_stat_update48(hw, I40E_GLV_MPRCH(stat_idx),
 539                            I40E_GLV_MPRCL(stat_idx),
 540                            vsi->stat_offsets_loaded,
 541                            &oes->rx_multicast, &es->rx_multicast);
 542         i40e_stat_update48(hw, I40E_GLV_BPRCH(stat_idx),
 543                            I40E_GLV_BPRCL(stat_idx),
 544                            vsi->stat_offsets_loaded,
 545                            &oes->rx_broadcast, &es->rx_broadcast);
 546
 547         i40e_stat_update48(hw, I40E_GLV_GOTCH(stat_idx),
 548                            I40E_GLV_GOTCL(stat_idx),
 549                            vsi->stat_offsets_loaded,
 550                            &oes->tx_bytes, &es->tx_bytes);
 551         i40e_stat_update48(hw, I40E_GLV_UPTCH(stat_idx),
 552                            I40E_GLV_UPTCL(stat_idx),
 553                            vsi->stat_offsets_loaded,
 554                            &oes->tx_unicast, &es->tx_unicast);
 555         i40e_stat_update48(hw, I40E_GLV_MPTCH(stat_idx),
 556                            I40E_GLV_MPTCL(stat_idx),
 557                            vsi->stat_offsets_loaded,
 558                            &oes->tx_multicast, &es->tx_multicast);
 559         i40e_stat_update48(hw, I40E_GLV_BPTCH(stat_idx),
 560                            I40E_GLV_BPTCL(stat_idx),
 561                            vsi->stat_offsets_loaded,
 562                            &oes->tx_broadcast, &es->tx_broadcast);
 563         vsi->stat_offsets_loaded = true;
 564 }
 565
 566 /**
 567  * i40e_update_veb_stats - Update Switch component statistics
 568  * @veb: the VEB being updated
 569  **/
 570 static void i40e_update_veb_stats(struct i40e_veb *veb)
 571 {
 572         struct i40e_pf *pf = veb->pf;
 573         struct i40e_hw *hw = &pf->hw;
 574         struct i40e_eth_stats *oes;
 575         struct i40e_eth_stats *es;     /* device's eth stats */
 576         int idx = 0;
 577
 578         idx = veb->stats_idx;
 579         es = &veb->stats;
 580         oes = &veb->stats_offsets;
 581
 582         /* Gather up the stats that the hw collects */
 583         i40e_stat_update32(hw, I40E_GLSW_TDPC(idx),
 584                            veb->stat_offsets_loaded,
 585                            &oes->tx_discards, &es->tx_discards);
 586         if (hw->revision_id > 0)
 587                 i40e_stat_update32(hw, I40E_GLSW_RUPP(idx),
 588                                    veb->stat_offsets_loaded,
 589                                    &oes->rx_unknown_protocol,
 590                                    &es->rx_unknown_protocol);
 591         i40e_stat_update48(hw, I40E_GLSW_GORCH(idx), I40E_GLSW_GORCL(idx),
 592                            veb->stat_offsets_loaded,
 593                            &oes->rx_bytes, &es->rx_bytes);
 594         i40e_stat_update48(hw, I40E_GLSW_UPRCH(idx), I40E_GLSW_UPRCL(idx),
 595                            veb->stat_offsets_loaded,
 596                            &oes->rx_unicast, &es->rx_unicast);
 597         i40e_stat_update48(hw, I40E_GLSW_MPRCH(idx), I40E_GLSW_MPRCL(idx),
 598                            veb->stat_offsets_loaded,
 599                            &oes->rx_multicast, &es->rx_multicast);
 600         i40e_stat_update48(hw, I40E_GLSW_BPRCH(idx), I40E_GLSW_BPRCL(idx),
 601                            veb->stat_offsets_loaded,
 602                            &oes->rx_broadcast, &es->rx_broadcast);
 603
 604         i40e_stat_update48(hw, I40E_GLSW_GOTCH(idx), I40E_GLSW_GOTCL(idx),
 605                            veb->stat_offsets_loaded,
 606                            &oes->tx_bytes, &es->tx_bytes);
 607         i40e_stat_update48(hw, I40E_GLSW_UPTCH(idx), I40E_GLSW_UPTCL(idx),
 608                            veb->stat_offsets_loaded,
 609                            &oes->tx_unicast, &es->tx_unicast);
 610         i40e_stat_update48(hw, I40E_GLSW_MPTCH(idx), I40E_GLSW_MPTCL(idx),
 611                            veb->stat_offsets_loaded,
 612                            &oes->tx_multicast, &es->tx_multicast);
 613         i40e_stat_update48(hw, I40E_GLSW_BPTCH(idx), I40E_GLSW_BPTCL(idx),
 614                            veb->stat_offsets_loaded,
 615                            &oes->tx_broadcast, &es->tx_broadcast);
 616         veb->stat_offsets_loaded = true;
 617 }
 618
 619 /**
 620  * i40e_update_link_xoff_rx - Update XOFF received in link flow control mode
 621  * @pf: the corresponding PF
 622  *
 623  * Update the Rx XOFF counter (PAUSE frames) in link flow control mode
 624  **/
 625 static void i40e_update_link_xoff_rx(struct i40e_pf *pf)
 626 {
 627         struct i40e_hw_port_stats *osd = &pf->stats_offsets;
 628         struct i40e_hw_port_stats *nsd = &pf->stats;
 629         struct i40e_hw *hw = &pf->hw;
 630         u64 xoff = 0;
 631         u16 i, v;
 632
 633         if ((hw->fc.current_mode != I40E_FC_FULL) &&
 634             (hw->fc.current_mode != I40E_FC_RX_PAUSE))
 635                 return;
 636
 637         xoff = nsd->link_xoff_rx;
 638         i40e_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port),
 639                            pf->stat_offsets_loaded,
 640                            &osd->link_xoff_rx, &nsd->link_xoff_rx);
 641
 642         /* No new LFC xoff rx */
 643         if (!(nsd->link_xoff_rx - xoff))
 644                 return;
 645
 646         /* Clear the __I40E_HANG_CHECK_ARMED bit for all Tx rings */
 647         for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
 648                 struct i40e_vsi *vsi = pf->vsi[v];
 649
 650                 if (!vsi)
 651                         continue;
 652
 653                 for (i = 0; i < vsi->num_queue_pairs; i++) {
 654                         struct i40e_ring *ring = vsi->tx_rings[i];
 655                         clear_bit(__I40E_HANG_CHECK_ARMED, &ring->state);
 656                 }
 657         }
 658 }
 659
 660 /**
 661  * i40e_update_prio_xoff_rx - Update XOFF received in PFC mode
 662  * @pf: the corresponding PF
 663  *
 664  * Update the Rx XOFF counter (PAUSE frames) in PFC mode
 665  **/
 666 static void i40e_update_prio_xoff_rx(struct i40e_pf *pf)
 667 {
 668         struct i40e_hw_port_stats *osd = &pf->stats_offsets;
 669         struct i40e_hw_port_stats *nsd = &pf->stats;
 670         bool xoff[I40E_MAX_TRAFFIC_CLASS] = {false};
 671         struct i40e_dcbx_config *dcb_cfg;
 672         struct i40e_hw *hw = &pf->hw;
 673         u16 i, v;
 674         u8 tc;
 675
 676         dcb_cfg = &hw->local_dcbx_config;
 677
 678         /* See if DCB enabled with PFC TC */
 679         if (!(pf->flags & I40E_FLAG_DCB_ENABLED) ||
 680             !(dcb_cfg->pfc.pfcenable)) {
 681                 i40e_update_link_xoff_rx(pf);
 682                 return;
 683         }
 684
 685         for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) {
 686                 u64 prio_xoff = nsd->priority_xoff_rx[i];
 687                 i40e_stat_update32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i),
 688                                    pf->stat_offsets_loaded,
 689                                    &osd->priority_xoff_rx[i],
 690                                    &nsd->priority_xoff_rx[i]);
 691
 692                 /* No new PFC xoff rx */
 693                 if (!(nsd->priority_xoff_rx[i] - prio_xoff))
 694                         continue;
 695                 /* Get the TC for given priority */
 696                 tc = dcb_cfg->etscfg.prioritytable[i];
 697                 xoff[tc] = true;
 698         }
 699
 700         /* Clear the __I40E_HANG_CHECK_ARMED bit for Tx rings */
 701         for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
 702                 struct i40e_vsi *vsi = pf->vsi[v];
 703
 704                 if (!vsi)
 705                         continue;
 706
 707                 for (i = 0; i < vsi->num_queue_pairs; i++) {
 708                         struct i40e_ring *ring = vsi->tx_rings[i];
 709
 710                         tc = ring->dcb_tc;
 711                         if (xoff[tc])
 712                                 clear_bit(__I40E_HANG_CHECK_ARMED,
 713                                           &ring->state);
 714                 }
 715         }
 716 }
 717
 718 /**
 719  * i40e_update_stats - Update the board statistics counters.
 720  * @vsi: the VSI to be updated
 721  *
 722  * There are a few instances where we store the same stat in a
 723  * couple of different structs.  This is partly because we have
 724  * the netdev stats that need to be filled out, which is slightly
 725  * different from the "eth_stats" defined by the chip and used in
 726  * VF communications.  We sort it all out here in a central place.
 727  **/
 728 void i40e_update_stats(struct i40e_vsi *vsi)
 729 {
 730         struct i40e_pf *pf = vsi->back;
 731         struct i40e_hw *hw = &pf->hw;
 732         struct rtnl_link_stats64 *ons;
 733         struct rtnl_link_stats64 *ns;   /* netdev stats */
 734         struct i40e_eth_stats *oes;
 735         struct i40e_eth_stats *es;     /* device's eth stats */
 736         u32 tx_restart, tx_busy;
 737         u32 rx_page, rx_buf;
 738         u64 rx_p, rx_b;
 739         u64 tx_p, tx_b;
 740         int i;
 741         u16 q;
 742
 743         if (test_bit(__I40E_DOWN, &vsi->state) ||
 744             test_bit(__I40E_CONFIG_BUSY, &pf->state))
 745                 return;
 746
 747         ns = i40e_get_vsi_stats_struct(vsi);
 748         ons = &vsi->net_stats_offsets;
 749         es = &vsi->eth_stats;
 750         oes = &vsi->eth_stats_offsets;
 751
 752         /* Gather up the netdev and vsi stats that the driver collects
 753          * on the fly during packet processing
 754          */
 755         rx_b = rx_p = 0;
 756         tx_b = tx_p = 0;
 757         tx_restart = tx_busy = 0;
 758         rx_page = 0;
 759         rx_buf = 0;
 760         rcu_read_lock();
 761         for (q = 0; q < vsi->num_queue_pairs; q++) {
 762                 struct i40e_ring *p;
 763                 u64 bytes, packets;
 764                 unsigned int start;
 765
 766                 /* locate Tx ring */
 767                 p = ACCESS_ONCE(vsi->tx_rings[q]);
 768
 769                 do {
 770                         start = u64_stats_fetch_begin_bh(&p->syncp);
 771                         packets = p->stats.packets;
 772                         bytes = p->stats.bytes;
 773                 } while (u64_stats_fetch_retry_bh(&p->syncp, start));
 774                 tx_b += bytes;
 775                 tx_p += packets;
 776                 tx_restart += p->tx_stats.restart_queue;
 777                 tx_busy += p->tx_stats.tx_busy;
 778
 779                 /* Rx queue is part of the same block as Tx queue */
 780                 p = &p[1];
 781                 do {
 782                         start = u64_stats_fetch_begin_bh(&p->syncp);
 783                         packets = p->stats.packets;
 784                         bytes = p->stats.bytes;
 785                 } while (u64_stats_fetch_retry_bh(&p->syncp, start));
 786                 rx_b += bytes;
 787                 rx_p += packets;
 788                 rx_buf += p->rx_stats.alloc_buff_failed;
 789                 rx_page += p->rx_stats.alloc_page_failed;
 790         }
 791         rcu_read_unlock();
 792         vsi->tx_restart = tx_restart;
 793         vsi->tx_busy = tx_busy;
 794         vsi->rx_page_failed = rx_page;
 795         vsi->rx_buf_failed = rx_buf;
 796
 797         ns->rx_packets = rx_p;
 798         ns->rx_bytes = rx_b;
 799         ns->tx_packets = tx_p;
 800         ns->tx_bytes = tx_b;
 801
 802         i40e_update_eth_stats(vsi);
 803         /* update netdev stats from eth stats */
 804         ons->rx_errors = oes->rx_errors;
 805         ns->rx_errors = es->rx_errors;
 806         ons->tx_errors = oes->tx_errors;
 807         ns->tx_errors = es->tx_errors;
 808         ons->multicast = oes->rx_multicast;
 809         ns->multicast = es->rx_multicast;
 810         ons->tx_dropped = oes->tx_discards;
 811         ns->tx_dropped = es->tx_discards;
 812
 813         /* Get the port data only if this is the main PF VSI */
 814         if (vsi == pf->vsi[pf->lan_vsi]) {
 815                 struct i40e_hw_port_stats *nsd = &pf->stats;
 816                 struct i40e_hw_port_stats *osd = &pf->stats_offsets;
 817
 818                 i40e_stat_update48(hw, I40E_GLPRT_GORCH(hw->port),
 819                                    I40E_GLPRT_GORCL(hw->port),
 820                                    pf->stat_offsets_loaded,
 821                                    &osd->eth.rx_bytes, &nsd->eth.rx_bytes);
 822                 i40e_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port),
 823                                    I40E_GLPRT_GOTCL(hw->port),
 824                                    pf->stat_offsets_loaded,
 825                                    &osd->eth.tx_bytes, &nsd->eth.tx_bytes);
 826                 i40e_stat_update32(hw, I40E_GLPRT_RDPC(hw->port),
 827                                    pf->stat_offsets_loaded,
 828                                    &osd->eth.rx_discards,
 829                                    &nsd->eth.rx_discards);
 830                 i40e_stat_update32(hw, I40E_GLPRT_TDPC(hw->port),
 831                                    pf->stat_offsets_loaded,
 832                                    &osd->eth.tx_discards,
 833                                    &nsd->eth.tx_discards);
 834                 i40e_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port),
 835                                    I40E_GLPRT_MPRCL(hw->port),
 836                                    pf->stat_offsets_loaded,
 837                                    &osd->eth.rx_multicast,
 838                                    &nsd->eth.rx_multicast);
 839
 840                 i40e_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port),
 841                                    pf->stat_offsets_loaded,
 842                                    &osd->tx_dropped_link_down,
 843                                    &nsd->tx_dropped_link_down);
 844
 845                 i40e_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port),
 846                                    pf->stat_offsets_loaded,
 847                                    &osd->crc_errors, &nsd->crc_errors);
 848                 ns->rx_crc_errors = nsd->crc_errors;
 849
 850                 i40e_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port),
 851                                    pf->stat_offsets_loaded,
 852                                    &osd->illegal_bytes, &nsd->illegal_bytes);
 853                 ns->rx_errors = nsd->crc_errors
 854                                 + nsd->illegal_bytes;
 855
 856                 i40e_stat_update32(hw, I40E_GLPRT_MLFC(hw->port),
 857                                    pf->stat_offsets_loaded,
 858                                    &osd->mac_local_faults,
 859                                    &nsd->mac_local_faults);
 860                 i40e_stat_update32(hw, I40E_GLPRT_MRFC(hw->port),
 861                                    pf->stat_offsets_loaded,
 862                                    &osd->mac_remote_faults,
 863                                    &nsd->mac_remote_faults);
 864
 865                 i40e_stat_update32(hw, I40E_GLPRT_RLEC(hw->port),
 866                                    pf->stat_offsets_loaded,
 867                                    &osd->rx_length_errors,
 868                                    &nsd->rx_length_errors);
 869                 ns->rx_length_errors = nsd->rx_length_errors;
 870
 871                 i40e_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port),
 872                                    pf->stat_offsets_loaded,
 873                                    &osd->link_xon_rx, &nsd->link_xon_rx);
 874                 i40e_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port),
 875                                    pf->stat_offsets_loaded,
 876                                    &osd->link_xon_tx, &nsd->link_xon_tx);
 877                 i40e_update_prio_xoff_rx(pf);  /* handles I40E_GLPRT_LXOFFRXC */
 878                 i40e_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port),
 879                                    pf->stat_offsets_loaded,
 880                                    &osd->link_xoff_tx, &nsd->link_xoff_tx);
 881
 882                 for (i = 0; i < 8; i++) {
 883                         i40e_stat_update32(hw, I40E_GLPRT_PXONRXC(hw->port, i),
 884                                            pf->stat_offsets_loaded,
 885                                            &osd->priority_xon_rx[i],
 886                                            &nsd->priority_xon_rx[i]);
 887                         i40e_stat_update32(hw, I40E_GLPRT_PXONTXC(hw->port, i),
 888                                            pf->stat_offsets_loaded,
 889                                            &osd->priority_xon_tx[i],
 890                                            &nsd->priority_xon_tx[i]);
 891                         i40e_stat_update32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i),
 892                                            pf->stat_offsets_loaded,
 893                                            &osd->priority_xoff_tx[i],
 894                                            &nsd->priority_xoff_tx[i]);
 895                         i40e_stat_update32(hw,
 896                                            I40E_GLPRT_RXON2OFFCNT(hw->port, i),
 897                                            pf->stat_offsets_loaded,
 898                                            &osd->priority_xon_2_xoff[i],
 899                                            &nsd->priority_xon_2_xoff[i]);
 900                 }
 901
 902                 i40e_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port),
 903                                    I40E_GLPRT_PRC64L(hw->port),
 904                                    pf->stat_offsets_loaded,
 905                                    &osd->rx_size_64, &nsd->rx_size_64);
 906                 i40e_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port),
 907                                    I40E_GLPRT_PRC127L(hw->port),
 908                                    pf->stat_offsets_loaded,
 909                                    &osd->rx_size_127, &nsd->rx_size_127);
 910                 i40e_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port),
 911                                    I40E_GLPRT_PRC255L(hw->port),
 912                                    pf->stat_offsets_loaded,
 913                                    &osd->rx_size_255, &nsd->rx_size_255);
 914                 i40e_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port),
 915                                    I40E_GLPRT_PRC511L(hw->port),
 916                                    pf->stat_offsets_loaded,
 917                                    &osd->rx_size_511, &nsd->rx_size_511);
 918                 i40e_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port),
 919                                    I40E_GLPRT_PRC1023L(hw->port),
 920                                    pf->stat_offsets_loaded,
 921                                    &osd->rx_size_1023, &nsd->rx_size_1023);
 922                 i40e_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port),
 923                                    I40E_GLPRT_PRC1522L(hw->port),
 924                                    pf->stat_offsets_loaded,
 925                                    &osd->rx_size_1522, &nsd->rx_size_1522);
 926                 i40e_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port),
 927                                    I40E_GLPRT_PRC9522L(hw->port),
 928                                    pf->stat_offsets_loaded,
 929                                    &osd->rx_size_big, &nsd->rx_size_big);
 930
 931                 i40e_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port),
 932                                    I40E_GLPRT_PTC64L(hw->port),
 933                                    pf->stat_offsets_loaded,
 934                                    &osd->tx_size_64, &nsd->tx_size_64);
 935                 i40e_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port),
 936                                    I40E_GLPRT_PTC127L(hw->port),
 937                                    pf->stat_offsets_loaded,
 938                                    &osd->tx_size_127, &nsd->tx_size_127);
 939                 i40e_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port),
 940                                    I40E_GLPRT_PTC255L(hw->port),
 941                                    pf->stat_offsets_loaded,
 942                                    &osd->tx_size_255, &nsd->tx_size_255);
 943                 i40e_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port),
 944                                    I40E_GLPRT_PTC511L(hw->port),
 945                                    pf->stat_offsets_loaded,
 946                                    &osd->tx_size_511, &nsd->tx_size_511);
 947                 i40e_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port),
 948                                    I40E_GLPRT_PTC1023L(hw->port),
 949                                    pf->stat_offsets_loaded,
 950                                    &osd->tx_size_1023, &nsd->tx_size_1023);
 951                 i40e_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port),
 952                                    I40E_GLPRT_PTC1522L(hw->port),
 953                                    pf->stat_offsets_loaded,
 954                                    &osd->tx_size_1522, &nsd->tx_size_1522);
 955                 i40e_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port),
 956                                    I40E_GLPRT_PTC9522L(hw->port),
 957                                    pf->stat_offsets_loaded,
 958                                    &osd->tx_size_big, &nsd->tx_size_big);
 959
 960                 i40e_stat_update32(hw, I40E_GLPRT_RUC(hw->port),
 961                                    pf->stat_offsets_loaded,
 962                                    &osd->rx_undersize, &nsd->rx_undersize);
 963                 i40e_stat_update32(hw, I40E_GLPRT_RFC(hw->port),
 964                                    pf->stat_offsets_loaded,
 965                                    &osd->rx_fragments, &nsd->rx_fragments);
 966                 i40e_stat_update32(hw, I40E_GLPRT_ROC(hw->port),
 967                                    pf->stat_offsets_loaded,
 968                                    &osd->rx_oversize, &nsd->rx_oversize);
 969                 i40e_stat_update32(hw, I40E_GLPRT_RJC(hw->port),
 970                                    pf->stat_offsets_loaded,
 971                                    &osd->rx_jabber, &nsd->rx_jabber);
 972         }
 973
 974         pf->stat_offsets_loaded = true;
 975 }
 976
 977 /**
 978  * i40e_find_filter - Search VSI filter list for specific mac/vlan filter
 979  * @vsi: the VSI to be searched
 980  * @macaddr: the MAC address
 981  * @vlan: the vlan
 982  * @is_vf: make sure its a vf filter, else doesn't matter
 983  * @is_netdev: make sure its a netdev filter, else doesn't matter
 984  *
 985  * Returns ptr to the filter object or NULL
 986  **/
 987 static struct i40e_mac_filter *i40e_find_filter(struct i40e_vsi *vsi,
 988                                                 u8 *macaddr, s16 vlan,
 989                                                 bool is_vf, bool is_netdev)
 990 {
 991         struct i40e_mac_filter *f;
 992
 993         if (!vsi || !macaddr)
 994                 return NULL;
 995
 996         list_for_each_entry(f, &vsi->mac_filter_list, list) {
 997                 if ((ether_addr_equal(macaddr, f->macaddr)) &&
 998                     (vlan == f->vlan)    &&
 999                     (!is_vf || f->is_vf) &&
1000                     (!is_netdev || f->is_netdev))
1001                         return f;
1002         }
1003         return NULL;
1004 }
1005
1006 /**
1007  * i40e_find_mac - Find a mac addr in the macvlan filters list
1008  * @vsi: the VSI to be searched
1009  * @macaddr: the MAC address we are searching for
1010  * @is_vf: make sure its a vf filter, else doesn't matter
1011  * @is_netdev: make sure its a netdev filter, else doesn't matter
1012  *
1013  * Returns the first filter with the provided MAC address or NULL if
1014  * MAC address was not found
1015  **/
1016 struct i40e_mac_filter *i40e_find_mac(struct i40e_vsi *vsi, u8 *macaddr,
1017                                       bool is_vf, bool is_netdev)
1018 {
1019         struct i40e_mac_filter *f;
1020
1021         if (!vsi || !macaddr)
1022                 return NULL;
1023
1024         list_for_each_entry(f, &vsi->mac_filter_list, list) {
1025                 if ((ether_addr_equal(macaddr, f->macaddr)) &&
1026                     (!is_vf || f->is_vf) &&
1027                     (!is_netdev || f->is_netdev))
1028                         return f;
1029         }
1030         return NULL;
1031 }
1032
1033 /**
1034  * i40e_is_vsi_in_vlan - Check if VSI is in vlan mode
1035  * @vsi: the VSI to be searched
1036  *
1037  * Returns true if VSI is in vlan mode or false otherwise
1038  **/
1039 bool i40e_is_vsi_in_vlan(struct i40e_vsi *vsi)
1040 {
1041         struct i40e_mac_filter *f;
1042
1043         /* Only -1 for all the filters denotes not in vlan mode
1044          * so we have to go through all the list in order to make sure
1045          */
1046         list_for_each_entry(f, &vsi->mac_filter_list, list) {
1047                 if (f->vlan >= 0)
1048                         return true;
1049         }
1050
1051         return false;
1052 }
1053
1054 /**
1055  * i40e_put_mac_in_vlan - Make macvlan filters from macaddrs and vlans
1056  * @vsi: the VSI to be searched
1057  * @macaddr: the mac address to be filtered
1058  * @is_vf: true if it is a vf
1059  * @is_netdev: true if it is a netdev
1060  *
1061  * Goes through all the macvlan filters and adds a
1062  * macvlan filter for each unique vlan that already exists
1063  *
1064  * Returns first filter found on success, else NULL
1065  **/
1066 struct i40e_mac_filter *i40e_put_mac_in_vlan(struct i40e_vsi *vsi, u8 *macaddr,
1067                                              bool is_vf, bool is_netdev)
1068 {
1069         struct i40e_mac_filter *f;
1070
1071         list_for_each_entry(f, &vsi->mac_filter_list, list) {
1072                 if (!i40e_find_filter(vsi, macaddr, f->vlan,
1073                                       is_vf, is_netdev)) {
1074                         if (!i40e_add_filter(vsi, macaddr, f->vlan,
1075                                                 is_vf, is_netdev))
1076                                 return NULL;
1077                 }
1078         }
1079
1080         return list_first_entry_or_null(&vsi->mac_filter_list,
1081                                         struct i40e_mac_filter, list);
1082 }
1083
1084 /**
1085  * i40e_add_filter - Add a mac/vlan filter to the VSI
1086  * @vsi: the VSI to be searched
1087  * @macaddr: the MAC address
1088  * @vlan: the vlan
1089  * @is_vf: make sure its a vf filter, else doesn't matter
1090  * @is_netdev: make sure its a netdev filter, else doesn't matter
1091  *
1092  * Returns ptr to the filter object or NULL when no memory available.
1093  **/
1094 struct i40e_mac_filter *i40e_add_filter(struct i40e_vsi *vsi,
1095                                         u8 *macaddr, s16 vlan,
1096                                         bool is_vf, bool is_netdev)
1097 {
1098         struct i40e_mac_filter *f;
1099
1100         if (!vsi || !macaddr)
1101                 return NULL;
1102
1103         f = i40e_find_filter(vsi, macaddr, vlan, is_vf, is_netdev);
1104         if (!f) {
1105                 f = kzalloc(sizeof(*f), GFP_ATOMIC);
1106                 if (!f)
1107                         goto add_filter_out;
1108
1109                 memcpy(f->macaddr, macaddr, ETH_ALEN);
1110                 f->vlan = vlan;
1111                 f->changed = true;
1112
1113                 INIT_LIST_HEAD(&f->list);
1114                 list_add(&f->list, &vsi->mac_filter_list);
1115         }
1116
1117         /* increment counter and add a new flag if needed */
1118         if (is_vf) {
1119                 if (!f->is_vf) {
1120                         f->is_vf = true;
1121                         f->counter++;
1122                 }
1123         } else if (is_netdev) {
1124                 if (!f->is_netdev) {
1125                         f->is_netdev = true;
1126                         f->counter++;
1127                 }
1128         } else {
1129                 f->counter++;
1130         }
1131
1132         /* changed tells sync_filters_subtask to
1133          * push the filter down to the firmware
1134          */
1135         if (f->changed) {
1136                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1137                 vsi->back->flags |= I40E_FLAG_FILTER_SYNC;
1138         }
1139
1140 add_filter_out:
1141         return f;
1142 }
1143
1144 /**
1145  * i40e_del_filter - Remove a mac/vlan filter from the VSI
1146  * @vsi: the VSI to be searched
1147  * @macaddr: the MAC address
1148  * @vlan: the vlan
1149  * @is_vf: make sure it's a vf filter, else doesn't matter
1150  * @is_netdev: make sure it's a netdev filter, else doesn't matter
1151  **/
1152 void i40e_del_filter(struct i40e_vsi *vsi,
1153                      u8 *macaddr, s16 vlan,
1154                      bool is_vf, bool is_netdev)
1155 {
1156         struct i40e_mac_filter *f;
1157
1158         if (!vsi || !macaddr)
1159                 return;
1160
1161         f = i40e_find_filter(vsi, macaddr, vlan, is_vf, is_netdev);
1162         if (!f || f->counter == 0)
1163                 return;
1164
1165         if (is_vf) {
1166                 if (f->is_vf) {
1167                         f->is_vf = false;
1168                         f->counter--;
1169                 }
1170         } else if (is_netdev) {
1171                 if (f->is_netdev) {
1172                         f->is_netdev = false;
1173                         f->counter--;
1174                 }
1175         } else {
1176                 /* make sure we don't remove a filter in use by vf or netdev */
1177                 int min_f = 0;
1178                 min_f += (f->is_vf ? 1 : 0);
1179                 min_f += (f->is_netdev ? 1 : 0);
1180
1181                 if (f->counter > min_f)
1182                         f->counter--;
1183         }
1184
1185         /* counter == 0 tells sync_filters_subtask to
1186          * remove the filter from the firmware's list
1187          */
1188         if (f->counter == 0) {
1189                 f->changed = true;
1190                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1191                 vsi->back->flags |= I40E_FLAG_FILTER_SYNC;
1192         }
1193 }
1194
1195 /**
1196  * i40e_set_mac - NDO callback to set mac address
1197  * @netdev: network interface device structure
1198  * @p: pointer to an address structure
1199  *
1200  * Returns 0 on success, negative on failure
1201  **/
1202 static int i40e_set_mac(struct net_device *netdev, void *p)
1203 {
1204         struct i40e_netdev_priv *np = netdev_priv(netdev);
1205         struct i40e_vsi *vsi = np->vsi;
1206         struct sockaddr *addr = p;
1207         struct i40e_mac_filter *f;
1208
1209         if (!is_valid_ether_addr(addr->sa_data))
1210                 return -EADDRNOTAVAIL;
1211
1212         netdev_info(netdev, "set mac address=%pM\n", addr->sa_data);
1213
1214         if (ether_addr_equal(netdev->dev_addr, addr->sa_data))
1215                 return 0;
1216
1217         if (test_bit(__I40E_DOWN, &vsi->back->state) ||
1218             test_bit(__I40E_RESET_RECOVERY_PENDING, &vsi->back->state))
1219                 return -EADDRNOTAVAIL;
1220
1221         if (vsi->type == I40E_VSI_MAIN) {
1222                 i40e_status ret;
1223                 ret = i40e_aq_mac_address_write(&vsi->back->hw,
1224                                                 I40E_AQC_WRITE_TYPE_LAA_ONLY,
1225                                                 addr->sa_data, NULL);
1226                 if (ret) {
1227                         netdev_info(netdev,
1228                                     "Addr change for Main VSI failed: %d\n",
1229                                     ret);
1230                         return -EADDRNOTAVAIL;
1231                 }
1232
1233                 memcpy(vsi->back->hw.mac.addr, addr->sa_data, netdev->addr_len);
1234         }
1235
1236         /* In order to be sure to not drop any packets, add the new address
1237          * then delete the old one.
1238          */
1239         f = i40e_add_filter(vsi, addr->sa_data, I40E_VLAN_ANY, false, false);
1240         if (!f)
1241                 return -ENOMEM;
1242
1243         i40e_sync_vsi_filters(vsi);
1244         i40e_del_filter(vsi, netdev->dev_addr, I40E_VLAN_ANY, false, false);
1245         i40e_sync_vsi_filters(vsi);
1246
1247         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
1248
1249         return 0;
1250 }
1251
1252 /**
1253  * i40e_vsi_setup_queue_map - Setup a VSI queue map based on enabled_tc
1254  * @vsi: the VSI being setup
1255  * @ctxt: VSI context structure
1256  * @enabled_tc: Enabled TCs bitmap
1257  * @is_add: True if called before Add VSI
1258  *
1259  * Setup VSI queue mapping for enabled traffic classes.
1260  **/
1261 static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi,
1262                                      struct i40e_vsi_context *ctxt,
1263                                      u8 enabled_tc,
1264                                      bool is_add)
1265 {
1266         struct i40e_pf *pf = vsi->back;
1267         u16 sections = 0;
1268         u8 netdev_tc = 0;
1269         u16 numtc = 0;
1270         u16 qcount;
1271         u8 offset;
1272         u16 qmap;
1273         int i;
1274
1275         sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
1276         offset = 0;
1277
1278         if (enabled_tc && (vsi->back->flags & I40E_FLAG_DCB_ENABLED)) {
1279                 /* Find numtc from enabled TC bitmap */
1280                 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1281                         if (enabled_tc & (1 << i)) /* TC is enabled */
1282                                 numtc++;
1283                 }
1284                 if (!numtc) {
1285                         dev_warn(&pf->pdev->dev, "DCB is enabled but no TC enabled, forcing TC0\n");
1286                         numtc = 1;
1287                 }
1288         } else {
1289                 /* At least TC0 is enabled in case of non-DCB case */
1290                 numtc = 1;
1291         }
1292
1293         vsi->tc_config.numtc = numtc;
1294         vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1;
1295
1296         /* Setup queue offset/count for all TCs for given VSI */
1297         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1298                 /* See if the given TC is enabled for the given VSI */
1299                 if (vsi->tc_config.enabled_tc & (1 << i)) { /* TC is enabled */
1300                         int pow, num_qps;
1301
1302                         vsi->tc_config.tc_info[i].qoffset = offset;
1303                         switch (vsi->type) {
1304                         case I40E_VSI_MAIN:
1305                                 if (i == 0)
1306                                         qcount = pf->rss_size;
1307                                 else
1308                                         qcount = pf->num_tc_qps;
1309                                 vsi->tc_config.tc_info[i].qcount = qcount;
1310                                 break;
1311                         case I40E_VSI_FDIR:
1312                         case I40E_VSI_SRIOV:
1313                         case I40E_VSI_VMDQ2:
1314                         default:
1315                                 qcount = vsi->alloc_queue_pairs;
1316                                 vsi->tc_config.tc_info[i].qcount = qcount;
1317                                 WARN_ON(i != 0);
1318                                 break;
1319                         }
1320
1321                         /* find the power-of-2 of the number of queue pairs */
1322                         num_qps = vsi->tc_config.tc_info[i].qcount;
1323                         pow = 0;
1324                         while (num_qps &&
1325                               ((1 << pow) < vsi->tc_config.tc_info[i].qcount)) {
1326                                 pow++;
1327                                 num_qps >>= 1;
1328                         }
1329
1330                         vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++;
1331                         qmap =
1332                             (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
1333                             (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
1334
1335                         offset += vsi->tc_config.tc_info[i].qcount;
1336                 } else {
1337                         /* TC is not enabled so set the offset to
1338                          * default queue and allocate one queue
1339                          * for the given TC.
1340                          */
1341                         vsi->tc_config.tc_info[i].qoffset = 0;
1342                         vsi->tc_config.tc_info[i].qcount = 1;
1343                         vsi->tc_config.tc_info[i].netdev_tc = 0;
1344
1345                         qmap = 0;
1346                 }
1347                 ctxt->info.tc_mapping[i] = cpu_to_le16(qmap);
1348         }
1349
1350         /* Set actual Tx/Rx queue pairs */
1351         vsi->num_queue_pairs = offset;
1352
1353         /* Scheduler section valid can only be set for ADD VSI */
1354         if (is_add) {
1355                 sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
1356
1357                 ctxt->info.up_enable_bits = enabled_tc;
1358         }
1359         if (vsi->type == I40E_VSI_SRIOV) {
1360                 ctxt->info.mapping_flags |=
1361                                      cpu_to_le16(I40E_AQ_VSI_QUE_MAP_NONCONTIG);
1362                 for (i = 0; i < vsi->num_queue_pairs; i++)
1363                         ctxt->info.queue_mapping[i] =
1364                                                cpu_to_le16(vsi->base_queue + i);
1365         } else {
1366                 ctxt->info.mapping_flags |=
1367                                         cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
1368                 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
1369         }
1370         ctxt->info.valid_sections |= cpu_to_le16(sections);
1371 }
1372
1373 /**
1374  * i40e_set_rx_mode - NDO callback to set the netdev filters
1375  * @netdev: network interface device structure
1376  **/
1377 static void i40e_set_rx_mode(struct net_device *netdev)
1378 {
1379         struct i40e_netdev_priv *np = netdev_priv(netdev);
1380         struct i40e_mac_filter *f, *ftmp;
1381         struct i40e_vsi *vsi = np->vsi;
1382         struct netdev_hw_addr *uca;
1383         struct netdev_hw_addr *mca;
1384         struct netdev_hw_addr *ha;
1385
1386         /* add addr if not already in the filter list */
1387         netdev_for_each_uc_addr(uca, netdev) {
1388                 if (!i40e_find_mac(vsi, uca->addr, false, true)) {
1389                         if (i40e_is_vsi_in_vlan(vsi))
1390                                 i40e_put_mac_in_vlan(vsi, uca->addr,
1391                                                      false, true);
1392                         else
1393                                 i40e_add_filter(vsi, uca->addr, I40E_VLAN_ANY,
1394                                                 false, true);
1395                 }
1396         }
1397
1398         netdev_for_each_mc_addr(mca, netdev) {
1399                 if (!i40e_find_mac(vsi, mca->addr, false, true)) {
1400                         if (i40e_is_vsi_in_vlan(vsi))
1401                                 i40e_put_mac_in_vlan(vsi, mca->addr,
1402                                                      false, true);
1403                         else
1404                                 i40e_add_filter(vsi, mca->addr, I40E_VLAN_ANY,
1405                                                 false, true);
1406                 }
1407         }
1408
1409         /* remove filter if not in netdev list */
1410         list_for_each_entry_safe(f, ftmp, &vsi->mac_filter_list, list) {
1411                 bool found = false;
1412
1413                 if (!f->is_netdev)
1414                         continue;
1415
1416                 if (is_multicast_ether_addr(f->macaddr)) {
1417                         netdev_for_each_mc_addr(mca, netdev) {
1418                                 if (ether_addr_equal(mca->addr, f->macaddr)) {
1419                                         found = true;
1420                                         break;
1421                                 }
1422                         }
1423                 } else {
1424                         netdev_for_each_uc_addr(uca, netdev) {
1425                                 if (ether_addr_equal(uca->addr, f->macaddr)) {
1426                                         found = true;
1427                                         break;
1428                                 }
1429                         }
1430
1431                         for_each_dev_addr(netdev, ha) {
1432                                 if (ether_addr_equal(ha->addr, f->macaddr)) {
1433                                         found = true;
1434                                         break;
1435                                 }
1436                         }
1437                 }
1438                 if (!found)
1439                         i40e_del_filter(
1440                            vsi, f->macaddr, I40E_VLAN_ANY, false, true);
1441         }
1442
1443         /* check for other flag changes */
1444         if (vsi->current_netdev_flags != vsi->netdev->flags) {
1445                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1446                 vsi->back->flags |= I40E_FLAG_FILTER_SYNC;
1447         }
1448 }
1449
1450 /**
1451  * i40e_sync_vsi_filters - Update the VSI filter list to the HW
1452  * @vsi: ptr to the VSI
1453  *
1454  * Push any outstanding VSI filter changes through the AdminQ.
1455  *
1456  * Returns 0 or error value
1457  **/
1458 int i40e_sync_vsi_filters(struct i40e_vsi *vsi)
1459 {
1460         struct i40e_mac_filter *f, *ftmp;
1461         bool promisc_forced_on = false;
1462         bool add_happened = false;
1463         int filter_list_len = 0;
1464         u32 changed_flags = 0;
1465         i40e_status aq_ret = 0;
1466         struct i40e_pf *pf;
1467         int num_add = 0;
1468         int num_del = 0;
1469         u16 cmd_flags;
1470
1471         /* empty array typed pointers, kcalloc later */
1472         struct i40e_aqc_add_macvlan_element_data *add_list;
1473         struct i40e_aqc_remove_macvlan_element_data *del_list;
1474
1475         while (test_and_set_bit(__I40E_CONFIG_BUSY, &vsi->state))
1476                 usleep_range(1000, 2000);
1477         pf = vsi->back;
1478
1479         if (vsi->netdev) {
1480                 changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags;
1481                 vsi->current_netdev_flags = vsi->netdev->flags;
1482         }
1483
1484         if (vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) {
1485                 vsi->flags &= ~I40E_VSI_FLAG_FILTER_CHANGED;
1486
1487                 filter_list_len = pf->hw.aq.asq_buf_size /
1488                             sizeof(struct i40e_aqc_remove_macvlan_element_data);
1489                 del_list = kcalloc(filter_list_len,
1490                             sizeof(struct i40e_aqc_remove_macvlan_element_data),
1491                             GFP_KERNEL);
1492                 if (!del_list)
1493                         return -ENOMEM;
1494
1495                 list_for_each_entry_safe(f, ftmp, &vsi->mac_filter_list, list) {
1496                         if (!f->changed)
1497                                 continue;
1498
1499                         if (f->counter != 0)
1500                                 continue;
1501                         f->changed = false;
1502                         cmd_flags = 0;
1503
1504                         /* add to delete list */
1505                         memcpy(del_list[num_del].mac_addr,
1506                                f->macaddr, ETH_ALEN);
1507                         del_list[num_del].vlan_tag =
1508                                 cpu_to_le16((u16)(f->vlan ==
1509                                             I40E_VLAN_ANY ? 0 : f->vlan));
1510
1511                         cmd_flags |= I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
1512                         del_list[num_del].flags = cmd_flags;
1513                         num_del++;
1514
1515                         /* unlink from filter list */
1516                         list_del(&f->list);
1517                         kfree(f);
1518
1519                         /* flush a full buffer */
1520                         if (num_del == filter_list_len) {
1521                                 aq_ret = i40e_aq_remove_macvlan(&pf->hw,
1522                                             vsi->seid, del_list, num_del,
1523                                             NULL);
1524                                 num_del = 0;
1525                                 memset(del_list, 0, sizeof(*del_list));
1526
1527                                 if (aq_ret)
1528                                         dev_info(&pf->pdev->dev,
1529                                                  "ignoring delete macvlan error, err %d, aq_err %d while flushing a full buffer\n",
1530                                                  aq_ret,
1531                                                  pf->hw.aq.asq_last_status);
1532                         }
1533                 }
1534                 if (num_del) {
1535                         aq_ret = i40e_aq_remove_macvlan(&pf->hw, vsi->seid,
1536                                                      del_list, num_del, NULL);
1537                         num_del = 0;
1538
1539                         if (aq_ret)
1540                                 dev_info(&pf->pdev->dev,
1541                                          "ignoring delete macvlan error, err %d, aq_err %d\n",
1542                                          aq_ret, pf->hw.aq.asq_last_status);
1543                 }
1544
1545                 kfree(del_list);
1546                 del_list = NULL;
1547
1548                 /* do all the adds now */
1549                 filter_list_len = pf->hw.aq.asq_buf_size /
1550                                sizeof(struct i40e_aqc_add_macvlan_element_data),
1551                 add_list = kcalloc(filter_list_len,
1552                                sizeof(struct i40e_aqc_add_macvlan_element_data),
1553                                GFP_KERNEL);
1554                 if (!add_list)
1555                         return -ENOMEM;
1556
1557                 list_for_each_entry_safe(f, ftmp, &vsi->mac_filter_list, list) {
1558                         if (!f->changed)
1559                                 continue;
1560
1561                         if (f->counter == 0)
1562                                 continue;
1563                         f->changed = false;
1564                         add_happened = true;
1565                         cmd_flags = 0;
1566
1567                         /* add to add array */
1568                         memcpy(add_list[num_add].mac_addr,
1569                                f->macaddr, ETH_ALEN);
1570                         add_list[num_add].vlan_tag =
1571                                 cpu_to_le16(
1572                                  (u16)(f->vlan == I40E_VLAN_ANY ? 0 : f->vlan));
1573                         add_list[num_add].queue_number = 0;
1574
1575                         cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
1576                         add_list[num_add].flags = cpu_to_le16(cmd_flags);
1577                         num_add++;
1578
1579                         /* flush a full buffer */
1580                         if (num_add == filter_list_len) {
1581                                 aq_ret = i40e_aq_add_macvlan(&pf->hw, vsi->seid,
1582                                                              add_list, num_add,
1583                                                              NULL);
1584                                 num_add = 0;
1585
1586                                 if (aq_ret)
1587                                         break;
1588                                 memset(add_list, 0, sizeof(*add_list));
1589                         }
1590                 }
1591                 if (num_add) {
1592                         aq_ret = i40e_aq_add_macvlan(&pf->hw, vsi->seid,
1593                                                      add_list, num_add, NULL);
1594                         num_add = 0;
1595                 }
1596                 kfree(add_list);
1597                 add_list = NULL;
1598
1599                 if (add_happened && (!aq_ret)) {
1600                         /* do nothing */;
1601                 } else if (add_happened && (aq_ret)) {
1602                         dev_info(&pf->pdev->dev,
1603                                  "add filter failed, err %d, aq_err %d\n",
1604                                  aq_ret, pf->hw.aq.asq_last_status);
1605                         if ((pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOSPC) &&
1606                             !test_bit(__I40E_FILTER_OVERFLOW_PROMISC,
1607                                       &vsi->state)) {
1608                                 promisc_forced_on = true;
1609                                 set_bit(__I40E_FILTER_OVERFLOW_PROMISC,
1610                                         &vsi->state);
1611                                 dev_info(&pf->pdev->dev, "promiscuous mode forced on\n");
1612                         }
1613                 }
1614         }
1615
1616         /* check for changes in promiscuous modes */
1617         if (changed_flags & IFF_ALLMULTI) {
1618                 bool cur_multipromisc;
1619                 cur_multipromisc = !!(vsi->current_netdev_flags & IFF_ALLMULTI);
1620                 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(&vsi->back->hw,
1621                                                                vsi->seid,
1622                                                                cur_multipromisc,
1623                                                                NULL);
1624                 if (aq_ret)
1625                         dev_info(&pf->pdev->dev,
1626                                  "set multi promisc failed, err %d, aq_err %d\n",
1627                                  aq_ret, pf->hw.aq.asq_last_status);
1628         }
1629         if ((changed_flags & IFF_PROMISC) || promisc_forced_on) {
1630                 bool cur_promisc;
1631                 cur_promisc = (!!(vsi->current_netdev_flags & IFF_PROMISC) ||
1632                                test_bit(__I40E_FILTER_OVERFLOW_PROMISC,
1633                                         &vsi->state));
1634                 aq_ret = i40e_aq_set_vsi_unicast_promiscuous(&vsi->back->hw,
1635                                                              vsi->seid,
1636                                                              cur_promisc, NULL);
1637                 if (aq_ret)
1638                         dev_info(&pf->pdev->dev,
1639                                  "set uni promisc failed, err %d, aq_err %d\n",
1640                                  aq_ret, pf->hw.aq.asq_last_status);
1641                 aq_ret = i40e_aq_set_vsi_broadcast(&vsi->back->hw,
1642                                                    vsi->seid,
1643                                                    cur_promisc, NULL);
1644                 if (aq_ret)
1645                         dev_info(&pf->pdev->dev,
1646                                  "set brdcast promisc failed, err %d, aq_err %d\n",
1647                                  aq_ret, pf->hw.aq.asq_last_status);
1648         }
1649
1650         clear_bit(__I40E_CONFIG_BUSY, &vsi->state);
1651         return 0;
1652 }
1653
1654 /**
1655  * i40e_sync_filters_subtask - Sync the VSI filter list with HW
1656  * @pf: board private structure
1657  **/
1658 static void i40e_sync_filters_subtask(struct i40e_pf *pf)
1659 {
1660         int v;
1661
1662         if (!pf || !(pf->flags & I40E_FLAG_FILTER_SYNC))
1663                 return;
1664         pf->flags &= ~I40E_FLAG_FILTER_SYNC;
1665
1666         for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
1667                 if (pf->vsi[v] &&
1668                     (pf->vsi[v]->flags & I40E_VSI_FLAG_FILTER_CHANGED))
1669                         i40e_sync_vsi_filters(pf->vsi[v]);
1670         }
1671 }
1672
1673 /**
1674  * i40e_change_mtu - NDO callback to change the Maximum Transfer Unit
1675  * @netdev: network interface device structure
1676  * @new_mtu: new value for maximum frame size
1677  *
1678  * Returns 0 on success, negative on failure
1679  **/
1680 static int i40e_change_mtu(struct net_device *netdev, int new_mtu)
1681 {
1682         struct i40e_netdev_priv *np = netdev_priv(netdev);
1683         int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
1684         struct i40e_vsi *vsi = np->vsi;
1685
1686         /* MTU < 68 is an error and causes problems on some kernels */
1687         if ((new_mtu < 68) || (max_frame > I40E_MAX_RXBUFFER))
1688                 return -EINVAL;
1689
1690         netdev_info(netdev, "changing MTU from %d to %d\n",
1691                     netdev->mtu, new_mtu);
1692         netdev->mtu = new_mtu;
1693         if (netif_running(netdev))
1694                 i40e_vsi_reinit_locked(vsi);
1695
1696         return 0;
1697 }
1698
1699 /**
1700  * i40e_ioctl - Access the hwtstamp interface
1701  * @netdev: network interface device structure
1702  * @ifr: interface request data
1703  * @cmd: ioctl command
1704  **/
1705 int i40e_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
1706 {
1707         struct i40e_netdev_priv *np = netdev_priv(netdev);
1708         struct i40e_pf *pf = np->vsi->back;
1709
1710         switch (cmd) {
1711         case SIOCGHWTSTAMP:
1712                 return i40e_ptp_get_ts_config(pf, ifr);
1713         case SIOCSHWTSTAMP:
1714                 return i40e_ptp_set_ts_config(pf, ifr);
1715         default:
1716                 return -EOPNOTSUPP;
1717         }
1718 }
1719
1720 /**
1721  * i40e_vlan_stripping_enable - Turn on vlan stripping for the VSI
1722  * @vsi: the vsi being adjusted
1723  **/
1724 void i40e_vlan_stripping_enable(struct i40e_vsi *vsi)
1725 {
1726         struct i40e_vsi_context ctxt;
1727         i40e_status ret;
1728
1729         if ((vsi->info.valid_sections &
1730              cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
1731             ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_MODE_MASK) == 0))
1732                 return;  /* already enabled */
1733
1734         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
1735         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
1736                                     I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
1737
1738         ctxt.seid = vsi->seid;
1739         memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info));
1740         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
1741         if (ret) {
1742                 dev_info(&vsi->back->pdev->dev,
1743                          "%s: update vsi failed, aq_err=%d\n",
1744                          __func__, vsi->back->hw.aq.asq_last_status);
1745         }
1746 }
1747
1748 /**
1749  * i40e_vlan_stripping_disable - Turn off vlan stripping for the VSI
1750  * @vsi: the vsi being adjusted
1751  **/
1752 void i40e_vlan_stripping_disable(struct i40e_vsi *vsi)
1753 {
1754         struct i40e_vsi_context ctxt;
1755         i40e_status ret;
1756
1757         if ((vsi->info.valid_sections &
1758              cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
1759             ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_EMOD_MASK) ==
1760              I40E_AQ_VSI_PVLAN_EMOD_MASK))
1761                 return;  /* already disabled */
1762
1763         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
1764         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
1765                                     I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
1766
1767         ctxt.seid = vsi->seid;
1768         memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info));
1769         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
1770         if (ret) {
1771                 dev_info(&vsi->back->pdev->dev,
1772                          "%s: update vsi failed, aq_err=%d\n",
1773                          __func__, vsi->back->hw.aq.asq_last_status);
1774         }
1775 }
1776
1777 /**
1778  * i40e_vlan_rx_register - Setup or shutdown vlan offload
1779  * @netdev: network interface to be adjusted
1780  * @features: netdev features to test if VLAN offload is enabled or not
1781  **/
1782 static void i40e_vlan_rx_register(struct net_device *netdev, u32 features)
1783 {
1784         struct i40e_netdev_priv *np = netdev_priv(netdev);
1785         struct i40e_vsi *vsi = np->vsi;
1786
1787         if (features & NETIF_F_HW_VLAN_CTAG_RX)
1788                 i40e_vlan_stripping_enable(vsi);
1789         else
1790                 i40e_vlan_stripping_disable(vsi);
1791 }
1792
1793 /**
1794  * i40e_vsi_add_vlan - Add vsi membership for given vlan
1795  * @vsi: the vsi being configured
1796  * @vid: vlan id to be added (0 = untagged only , -1 = any)
1797  **/
1798 int i40e_vsi_add_vlan(struct i40e_vsi *vsi, s16 vid)
1799 {
1800         struct i40e_mac_filter *f, *add_f;
1801         bool is_netdev, is_vf;
1802
1803         is_vf = (vsi->type == I40E_VSI_SRIOV);
1804         is_netdev = !!(vsi->netdev);
1805
1806         if (is_netdev) {
1807                 add_f = i40e_add_filter(vsi, vsi->netdev->dev_addr, vid,
1808                                         is_vf, is_netdev);
1809                 if (!add_f) {
1810                         dev_info(&vsi->back->pdev->dev,
1811                                  "Could not add vlan filter %d for %pM\n",
1812                                  vid, vsi->netdev->dev_addr);
1813                         return -ENOMEM;
1814                 }
1815         }
1816
1817         list_for_each_entry(f, &vsi->mac_filter_list, list) {
1818                 add_f = i40e_add_filter(vsi, f->macaddr, vid, is_vf, is_netdev);
1819                 if (!add_f) {
1820                         dev_info(&vsi->back->pdev->dev,
1821                                  "Could not add vlan filter %d for %pM\n",
1822                                  vid, f->macaddr);
1823                         return -ENOMEM;
1824                 }
1825         }
1826
1827         /* Now if we add a vlan tag, make sure to check if it is the first
1828          * tag (i.e. a "tag" -1 does exist) and if so replace the -1 "tag"
1829          * with 0, so we now accept untagged and specified tagged traffic
1830          * (and not any taged and untagged)
1831          */
1832         if (vid > 0) {
1833                 if (is_netdev && i40e_find_filter(vsi, vsi->netdev->dev_addr,
1834                                                   I40E_VLAN_ANY,
1835                                                   is_vf, is_netdev)) {
1836                         i40e_del_filter(vsi, vsi->netdev->dev_addr,
1837                                         I40E_VLAN_ANY, is_vf, is_netdev);
1838                         add_f = i40e_add_filter(vsi, vsi->netdev->dev_addr, 0,
1839                                                 is_vf, is_netdev);
1840                         if (!add_f) {
1841                                 dev_info(&vsi->back->pdev->dev,
1842                                          "Could not add filter 0 for %pM\n",
1843                                          vsi->netdev->dev_addr);
1844                                 return -ENOMEM;
1845                         }
1846                 }
1847
1848                 list_for_each_entry(f, &vsi->mac_filter_list, list) {
1849                         if (i40e_find_filter(vsi, f->macaddr, I40E_VLAN_ANY,
1850                                              is_vf, is_netdev)) {
1851                                 i40e_del_filter(vsi, f->macaddr, I40E_VLAN_ANY,
1852                                                 is_vf, is_netdev);
1853                                 add_f = i40e_add_filter(vsi, f->macaddr,
1854                                                         0, is_vf, is_netdev);
1855                                 if (!add_f) {
1856                                         dev_info(&vsi->back->pdev->dev,
1857                                                  "Could not add filter 0 for %pM\n",
1858                                                  f->macaddr);
1859                                         return -ENOMEM;
1860                                 }
1861                         }
1862                 }
1863         }
1864
1865         if (test_bit(__I40E_DOWN, &vsi->back->state) ||
1866             test_bit(__I40E_RESET_RECOVERY_PENDING, &vsi->back->state))
1867                 return 0;
1868
1869         return i40e_sync_vsi_filters(vsi);
1870 }
1871
1872 /**
1873  * i40e_vsi_kill_vlan - Remove vsi membership for given vlan
1874  * @vsi: the vsi being configured
1875  * @vid: vlan id to be removed (0 = untagged only , -1 = any)
1876  *
1877  * Return: 0 on success or negative otherwise
1878  **/
1879 int i40e_vsi_kill_vlan(struct i40e_vsi *vsi, s16 vid)
1880 {
1881         struct net_device *netdev = vsi->netdev;
1882         struct i40e_mac_filter *f, *add_f;
1883         bool is_vf, is_netdev;
1884         int filter_count = 0;
1885
1886         is_vf = (vsi->type == I40E_VSI_SRIOV);
1887         is_netdev = !!(netdev);
1888
1889         if (is_netdev)
1890                 i40e_del_filter(vsi, netdev->dev_addr, vid, is_vf, is_netdev);
1891
1892         list_for_each_entry(f, &vsi->mac_filter_list, list)
1893                 i40e_del_filter(vsi, f->macaddr, vid, is_vf, is_netdev);
1894
1895         /* go through all the filters for this VSI and if there is only
1896          * vid == 0 it means there are no other filters, so vid 0 must
1897          * be replaced with -1. This signifies that we should from now
1898          * on accept any traffic (with any tag present, or untagged)
1899          */
1900         list_for_each_entry(f, &vsi->mac_filter_list, list) {
1901                 if (is_netdev) {
1902                         if (f->vlan &&
1903                             ether_addr_equal(netdev->dev_addr, f->macaddr))
1904                                 filter_count++;
1905                 }
1906
1907                 if (f->vlan)
1908                         filter_count++;
1909         }
1910
1911         if (!filter_count && is_netdev) {
1912                 i40e_del_filter(vsi, netdev->dev_addr, 0, is_vf, is_netdev);
1913                 f = i40e_add_filter(vsi, netdev->dev_addr, I40E_VLAN_ANY,
1914                                     is_vf, is_netdev);
1915                 if (!f) {
1916                         dev_info(&vsi->back->pdev->dev,
1917                                  "Could not add filter %d for %pM\n",
1918                                  I40E_VLAN_ANY, netdev->dev_addr);
1919                         return -ENOMEM;
1920                 }
1921         }
1922
1923         if (!filter_count) {
1924                 list_for_each_entry(f, &vsi->mac_filter_list, list) {
1925                         i40e_del_filter(vsi, f->macaddr, 0, is_vf, is_netdev);
1926                         add_f = i40e_add_filter(vsi, f->macaddr, I40E_VLAN_ANY,
1927                                             is_vf, is_netdev);
1928                         if (!add_f) {
1929                                 dev_info(&vsi->back->pdev->dev,
1930                                          "Could not add filter %d for %pM\n",
1931                                          I40E_VLAN_ANY, f->macaddr);
1932                                 return -ENOMEM;
1933                         }
1934                 }
1935         }
1936
1937         if (test_bit(__I40E_DOWN, &vsi->back->state) ||
1938             test_bit(__I40E_RESET_RECOVERY_PENDING, &vsi->back->state))
1939                 return 0;
1940
1941         return i40e_sync_vsi_filters(vsi);
1942 }
1943
1944 /**
1945  * i40e_vlan_rx_add_vid - Add a vlan id filter to HW offload
1946  * @netdev: network interface to be adjusted
1947  * @vid: vlan id to be added
1948  *
1949  * net_device_ops implementation for adding vlan ids
1950  **/
1951 static int i40e_vlan_rx_add_vid(struct net_device *netdev,
1952                                 __always_unused __be16 proto, u16 vid)
1953 {
1954         struct i40e_netdev_priv *np = netdev_priv(netdev);
1955         struct i40e_vsi *vsi = np->vsi;
1956         int ret = 0;
1957
1958         if (vid > 4095)
1959                 return -EINVAL;
1960
1961         netdev_info(netdev, "adding %pM vid=%d\n", netdev->dev_addr, vid);
1962
1963         /* If the network stack called us with vid = 0, we should
1964          * indicate to i40e_vsi_add_vlan() that we want to receive
1965          * any traffic (i.e. with any vlan tag, or untagged)
1966          */
1967         ret = i40e_vsi_add_vlan(vsi, vid ? vid : I40E_VLAN_ANY);
1968
1969         if (!ret && (vid < VLAN_N_VID))
1970                 set_bit(vid, vsi->active_vlans);
1971
1972         return ret;
1973 }
1974
1975 /**
1976  * i40e_vlan_rx_kill_vid - Remove a vlan id filter from HW offload
1977  * @netdev: network interface to be adjusted
1978  * @vid: vlan id to be removed
1979  *
1980  * net_device_ops implementation for adding vlan ids
1981  **/
1982 static int i40e_vlan_rx_kill_vid(struct net_device *netdev,
1983                                  __always_unused __be16 proto, u16 vid)
1984 {
1985         struct i40e_netdev_priv *np = netdev_priv(netdev);
1986         struct i40e_vsi *vsi = np->vsi;
1987
1988         netdev_info(netdev, "removing %pM vid=%d\n", netdev->dev_addr, vid);
1989
1990         /* return code is ignored as there is nothing a user
1991          * can do about failure to remove and a log message was
1992          * already printed from the other function
1993          */
1994         i40e_vsi_kill_vlan(vsi, vid);
1995
1996         clear_bit(vid, vsi->active_vlans);
1997
1998         return 0;
1999 }
2000
2001 /**
2002  * i40e_restore_vlan - Reinstate vlans when vsi/netdev comes back up
2003  * @vsi: the vsi being brought back up
2004  **/
2005 static void i40e_restore_vlan(struct i40e_vsi *vsi)
2006 {
2007         u16 vid;
2008
2009         if (!vsi->netdev)
2010                 return;
2011
2012         i40e_vlan_rx_register(vsi->netdev, vsi->netdev->features);
2013
2014         for_each_set_bit(vid, vsi->active_vlans, VLAN_N_VID)
2015                 i40e_vlan_rx_add_vid(vsi->netdev, htons(ETH_P_8021Q),
2016                                      vid);
2017 }
2018
2019 /**
2020  * i40e_vsi_add_pvid - Add pvid for the VSI
2021  * @vsi: the vsi being adjusted
2022  * @vid: the vlan id to set as a PVID
2023  **/
2024 int i40e_vsi_add_pvid(struct i40e_vsi *vsi, u16 vid)
2025 {
2026         struct i40e_vsi_context ctxt;
2027         i40e_status aq_ret;
2028
2029         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
2030         vsi->info.pvid = cpu_to_le16(vid);
2031         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_TAGGED |
2032                                     I40E_AQ_VSI_PVLAN_INSERT_PVID |
2033                                     I40E_AQ_VSI_PVLAN_EMOD_STR;
2034
2035         ctxt.seid = vsi->seid;
2036         memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info));
2037         aq_ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
2038         if (aq_ret) {
2039                 dev_info(&vsi->back->pdev->dev,
2040                          "%s: update vsi failed, aq_err=%d\n",
2041                          __func__, vsi->back->hw.aq.asq_last_status);
2042                 return -ENOENT;
2043         }
2044
2045         return 0;
2046 }
2047
2048 /**
2049  * i40e_vsi_remove_pvid - Remove the pvid from the VSI
2050  * @vsi: the vsi being adjusted
2051  *
2052  * Just use the vlan_rx_register() service to put it back to normal
2053  **/
2054 void i40e_vsi_remove_pvid(struct i40e_vsi *vsi)
2055 {
2056         i40e_vlan_stripping_disable(vsi);
2057
2058         vsi->info.pvid = 0;
2059 }
2060
2061 /**
2062  * i40e_vsi_setup_tx_resources - Allocate VSI Tx queue resources
2063  * @vsi: ptr to the VSI
2064  *
2065  * If this function returns with an error, then it's possible one or
2066  * more of the rings is populated (while the rest are not).  It is the
2067  * callers duty to clean those orphaned rings.
2068  *
2069  * Return 0 on success, negative on failure
2070  **/
2071 static int i40e_vsi_setup_tx_resources(struct i40e_vsi *vsi)
2072 {
2073         int i, err = 0;
2074
2075         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
2076                 err = i40e_setup_tx_descriptors(vsi->tx_rings[i]);
2077
2078         return err;
2079 }
2080
2081 /**
2082  * i40e_vsi_free_tx_resources - Free Tx resources for VSI queues
2083  * @vsi: ptr to the VSI
2084  *
2085  * Free VSI's transmit software resources
2086  **/
2087 static void i40e_vsi_free_tx_resources(struct i40e_vsi *vsi)
2088 {
2089         int i;
2090
2091         if (!vsi->tx_rings)
2092                 return;
2093
2094         for (i = 0; i < vsi->num_queue_pairs; i++)
2095                 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc)
2096                         i40e_free_tx_resources(vsi->tx_rings[i]);
2097 }
2098
2099 /**
2100  * i40e_vsi_setup_rx_resources - Allocate VSI queues Rx resources
2101  * @vsi: ptr to the VSI
2102  *
2103  * If this function returns with an error, then it's possible one or
2104  * more of the rings is populated (while the rest are not).  It is the
2105  * callers duty to clean those orphaned rings.
2106  *
2107  * Return 0 on success, negative on failure
2108  **/
2109 static int i40e_vsi_setup_rx_resources(struct i40e_vsi *vsi)
2110 {
2111         int i, err = 0;
2112
2113         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
2114                 err = i40e_setup_rx_descriptors(vsi->rx_rings[i]);
2115         return err;
2116 }
2117
2118 /**
2119  * i40e_vsi_free_rx_resources - Free Rx Resources for VSI queues
2120  * @vsi: ptr to the VSI
2121  *
2122  * Free all receive software resources
2123  **/
2124 static void i40e_vsi_free_rx_resources(struct i40e_vsi *vsi)
2125 {
2126         int i;
2127
2128         if (!vsi->rx_rings)
2129                 return;
2130
2131         for (i = 0; i < vsi->num_queue_pairs; i++)
2132                 if (vsi->rx_rings[i] && vsi->rx_rings[i]->desc)
2133                         i40e_free_rx_resources(vsi->rx_rings[i]);
2134 }
2135
2136 /**
2137  * i40e_configure_tx_ring - Configure a transmit ring context and rest
2138  * @ring: The Tx ring to configure
2139  *
2140  * Configure the Tx descriptor ring in the HMC context.
2141  **/
2142 static int i40e_configure_tx_ring(struct i40e_ring *ring)
2143 {
2144         struct i40e_vsi *vsi = ring->vsi;
2145         u16 pf_q = vsi->base_queue + ring->queue_index;
2146         struct i40e_hw *hw = &vsi->back->hw;
2147         struct i40e_hmc_obj_txq tx_ctx;
2148         i40e_status err = 0;
2149         u32 qtx_ctl = 0;
2150
2151         /* some ATR related tx ring init */
2152         if (vsi->back->flags & I40E_FLAG_FDIR_ATR_ENABLED) {
2153                 ring->atr_sample_rate = vsi->back->atr_sample_rate;
2154                 ring->atr_count = 0;
2155         } else {
2156                 ring->atr_sample_rate = 0;
2157         }
2158
2159         /* initialize XPS */
2160         if (ring->q_vector && ring->netdev &&
2161             !test_and_set_bit(__I40E_TX_XPS_INIT_DONE, &ring->state))
2162                 netif_set_xps_queue(ring->netdev,
2163                                     &ring->q_vector->affinity_mask,
2164                                     ring->queue_index);
2165
2166         /* clear the context structure first */
2167         memset(&tx_ctx, 0, sizeof(tx_ctx));
2168
2169         tx_ctx.new_context = 1;
2170         tx_ctx.base = (ring->dma / 128);
2171         tx_ctx.qlen = ring->count;
2172         tx_ctx.fd_ena = !!(vsi->back->flags & (I40E_FLAG_FDIR_ENABLED |
2173                                                I40E_FLAG_FDIR_ATR_ENABLED));
2174         tx_ctx.timesync_ena = !!(vsi->back->flags & I40E_FLAG_PTP);
2175
2176         /* As part of VSI creation/update, FW allocates certain
2177          * Tx arbitration queue sets for each TC enabled for
2178          * the VSI. The FW returns the handles to these queue
2179          * sets as part of the response buffer to Add VSI,
2180          * Update VSI, etc. AQ commands. It is expected that
2181          * these queue set handles be associated with the Tx
2182          * queues by the driver as part of the TX queue context
2183          * initialization. This has to be done regardless of
2184          * DCB as by default everything is mapped to TC0.
2185          */
2186         tx_ctx.rdylist = le16_to_cpu(vsi->info.qs_handle[ring->dcb_tc]);
2187         tx_ctx.rdylist_act = 0;
2188
2189         /* clear the context in the HMC */
2190         err = i40e_clear_lan_tx_queue_context(hw, pf_q);
2191         if (err) {
2192                 dev_info(&vsi->back->pdev->dev,
2193                          "Failed to clear LAN Tx queue context on Tx ring %d (pf_q %d), error: %d\n",
2194                          ring->queue_index, pf_q, err);
2195                 return -ENOMEM;
2196         }
2197
2198         /* set the context in the HMC */
2199         err = i40e_set_lan_tx_queue_context(hw, pf_q, &tx_ctx);
2200         if (err) {
2201                 dev_info(&vsi->back->pdev->dev,
2202                          "Failed to set LAN Tx queue context on Tx ring %d (pf_q %d, error: %d\n",
2203                          ring->queue_index, pf_q, err);
2204                 return -ENOMEM;
2205         }
2206
2207         /* Now associate this queue with this PCI function */
2208         if (vsi->type == I40E_VSI_VMDQ2)
2209                 qtx_ctl = I40E_QTX_CTL_VM_QUEUE;
2210         else
2211                 qtx_ctl = I40E_QTX_CTL_PF_QUEUE;
2212         qtx_ctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) &
2213                     I40E_QTX_CTL_PF_INDX_MASK);
2214         wr32(hw, I40E_QTX_CTL(pf_q), qtx_ctl);
2215         i40e_flush(hw);
2216
2217         clear_bit(__I40E_HANG_CHECK_ARMED, &ring->state);
2218
2219         /* cache tail off for easier writes later */
2220         ring->tail = hw->hw_addr + I40E_QTX_TAIL(pf_q);
2221
2222         return 0;
2223 }
2224
2225 /**
2226  * i40e_configure_rx_ring - Configure a receive ring context
2227  * @ring: The Rx ring to configure
2228  *
2229  * Configure the Rx descriptor ring in the HMC context.
2230  **/
2231 static int i40e_configure_rx_ring(struct i40e_ring *ring)
2232 {
2233         struct i40e_vsi *vsi = ring->vsi;
2234         u32 chain_len = vsi->back->hw.func_caps.rx_buf_chain_len;
2235         u16 pf_q = vsi->base_queue + ring->queue_index;
2236         struct i40e_hw *hw = &vsi->back->hw;
2237         struct i40e_hmc_obj_rxq rx_ctx;
2238         i40e_status err = 0;
2239
2240         ring->state = 0;
2241
2242         /* clear the context structure first */
2243         memset(&rx_ctx, 0, sizeof(rx_ctx));
2244
2245         ring->rx_buf_len = vsi->rx_buf_len;
2246         ring->rx_hdr_len = vsi->rx_hdr_len;
2247
2248         rx_ctx.dbuff = ring->rx_buf_len >> I40E_RXQ_CTX_DBUFF_SHIFT;
2249         rx_ctx.hbuff = ring->rx_hdr_len >> I40E_RXQ_CTX_HBUFF_SHIFT;
2250
2251         rx_ctx.base = (ring->dma / 128);
2252         rx_ctx.qlen = ring->count;
2253
2254         if (vsi->back->flags & I40E_FLAG_16BYTE_RX_DESC_ENABLED) {
2255                 set_ring_16byte_desc_enabled(ring);
2256                 rx_ctx.dsize = 0;
2257         } else {
2258                 rx_ctx.dsize = 1;
2259         }
2260
2261         rx_ctx.dtype = vsi->dtype;
2262         if (vsi->dtype) {
2263                 set_ring_ps_enabled(ring);
2264                 rx_ctx.hsplit_0 = I40E_RX_SPLIT_L2      |
2265                                   I40E_RX_SPLIT_IP      |
2266                                   I40E_RX_SPLIT_TCP_UDP |
2267                                   I40E_RX_SPLIT_SCTP;
2268         } else {
2269                 rx_ctx.hsplit_0 = 0;
2270         }
2271
2272         rx_ctx.rxmax = min_t(u16, vsi->max_frame,
2273                                   (chain_len * ring->rx_buf_len));
2274         rx_ctx.tphrdesc_ena = 1;
2275         rx_ctx.tphwdesc_ena = 1;
2276         rx_ctx.tphdata_ena = 1;
2277         rx_ctx.tphhead_ena = 1;
2278         if (hw->revision_id == 0)
2279                 rx_ctx.lrxqthresh = 0;
2280         else
2281                 rx_ctx.lrxqthresh = 2;
2282         rx_ctx.crcstrip = 1;
2283         rx_ctx.l2tsel = 1;
2284         rx_ctx.showiv = 1;
2285
2286         /* clear the context in the HMC */
2287         err = i40e_clear_lan_rx_queue_context(hw, pf_q);
2288         if (err) {
2289                 dev_info(&vsi->back->pdev->dev,
2290                          "Failed to clear LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
2291                          ring->queue_index, pf_q, err);
2292                 return -ENOMEM;
2293         }
2294
2295         /* set the context in the HMC */
2296         err = i40e_set_lan_rx_queue_context(hw, pf_q, &rx_ctx);
2297         if (err) {
2298                 dev_info(&vsi->back->pdev->dev,
2299                          "Failed to set LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
2300                          ring->queue_index, pf_q, err);
2301                 return -ENOMEM;
2302         }
2303
2304         /* cache tail for quicker writes, and clear the reg before use */
2305         ring->tail = hw->hw_addr + I40E_QRX_TAIL(pf_q);
2306         writel(0, ring->tail);
2307
2308         i40e_alloc_rx_buffers(ring, I40E_DESC_UNUSED(ring));
2309
2310         return 0;
2311 }
2312
2313 /**
2314  * i40e_vsi_configure_tx - Configure the VSI for Tx
2315  * @vsi: VSI structure describing this set of rings and resources
2316  *
2317  * Configure the Tx VSI for operation.
2318  **/
2319 static int i40e_vsi_configure_tx(struct i40e_vsi *vsi)
2320 {
2321         int err = 0;
2322         u16 i;
2323
2324         for (i = 0; (i < vsi->num_queue_pairs) && !err; i++)
2325                 err = i40e_configure_tx_ring(vsi->tx_rings[i]);
2326
2327         return err;
2328 }
2329
2330 /**
2331  * i40e_vsi_configure_rx - Configure the VSI for Rx
2332  * @vsi: the VSI being configured
2333  *
2334  * Configure the Rx VSI for operation.
2335  **/
2336 static int i40e_vsi_configure_rx(struct i40e_vsi *vsi)
2337 {
2338         int err = 0;
2339         u16 i;
2340
2341         if (vsi->netdev && (vsi->netdev->mtu > ETH_DATA_LEN))
2342                 vsi->max_frame = vsi->netdev->mtu + ETH_HLEN
2343                                + ETH_FCS_LEN + VLAN_HLEN;
2344         else
2345                 vsi->max_frame = I40E_RXBUFFER_2048;
2346
2347         /* figure out correct receive buffer length */
2348         switch (vsi->back->flags & (I40E_FLAG_RX_1BUF_ENABLED |
2349                                     I40E_FLAG_RX_PS_ENABLED)) {
2350         case I40E_FLAG_RX_1BUF_ENABLED:
2351                 vsi->rx_hdr_len = 0;
2352                 vsi->rx_buf_len = vsi->max_frame;
2353                 vsi->dtype = I40E_RX_DTYPE_NO_SPLIT;
2354                 break;
2355         case I40E_FLAG_RX_PS_ENABLED:
2356                 vsi->rx_hdr_len = I40E_RX_HDR_SIZE;
2357                 vsi->rx_buf_len = I40E_RXBUFFER_2048;
2358                 vsi->dtype = I40E_RX_DTYPE_HEADER_SPLIT;
2359                 break;
2360         default:
2361                 vsi->rx_hdr_len = I40E_RX_HDR_SIZE;
2362                 vsi->rx_buf_len = I40E_RXBUFFER_2048;
2363                 vsi->dtype = I40E_RX_DTYPE_SPLIT_ALWAYS;
2364                 break;
2365         }
2366
2367         /* round up for the chip's needs */
2368         vsi->rx_hdr_len = ALIGN(vsi->rx_hdr_len,
2369                                 (1 << I40E_RXQ_CTX_HBUFF_SHIFT));
2370         vsi->rx_buf_len = ALIGN(vsi->rx_buf_len,
2371                                 (1 << I40E_RXQ_CTX_DBUFF_SHIFT));
2372
2373         /* set up individual rings */
2374         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
2375                 err = i40e_configure_rx_ring(vsi->rx_rings[i]);
2376
2377         return err;
2378 }
2379
2380 /**
2381  * i40e_vsi_config_dcb_rings - Update rings to reflect DCB TC
2382  * @vsi: ptr to the VSI
2383  **/
2384 static void i40e_vsi_config_dcb_rings(struct i40e_vsi *vsi)
2385 {
2386         u16 qoffset, qcount;
2387         int i, n;
2388
2389         if (!(vsi->back->flags & I40E_FLAG_DCB_ENABLED))
2390                 return;
2391
2392         for (n = 0; n < I40E_MAX_TRAFFIC_CLASS; n++) {
2393                 if (!(vsi->tc_config.enabled_tc & (1 << n)))
2394                         continue;
2395
2396                 qoffset = vsi->tc_config.tc_info[n].qoffset;
2397                 qcount = vsi->tc_config.tc_info[n].qcount;
2398                 for (i = qoffset; i < (qoffset + qcount); i++) {
2399                         struct i40e_ring *rx_ring = vsi->rx_rings[i];
2400                         struct i40e_ring *tx_ring = vsi->tx_rings[i];
2401                         rx_ring->dcb_tc = n;
2402                         tx_ring->dcb_tc = n;
2403                 }
2404         }
2405 }
2406
2407 /**
2408  * i40e_set_vsi_rx_mode - Call set_rx_mode on a VSI
2409  * @vsi: ptr to the VSI
2410  **/
2411 static void i40e_set_vsi_rx_mode(struct i40e_vsi *vsi)
2412 {
2413         if (vsi->netdev)
2414                 i40e_set_rx_mode(vsi->netdev);
2415 }
2416
2417 /**
2418  * i40e_vsi_configure - Set up the VSI for action
2419  * @vsi: the VSI being configured
2420  **/
2421 static int i40e_vsi_configure(struct i40e_vsi *vsi)
2422 {
2423         int err;
2424
2425         i40e_set_vsi_rx_mode(vsi);
2426         i40e_restore_vlan(vsi);
2427         i40e_vsi_config_dcb_rings(vsi);
2428         err = i40e_vsi_configure_tx(vsi);
2429         if (!err)
2430                 err = i40e_vsi_configure_rx(vsi);
2431
2432         return err;
2433 }
2434
2435 /**
2436  * i40e_vsi_configure_msix - MSIX mode Interrupt Config in the HW
2437  * @vsi: the VSI being configured
2438  **/
2439 static void i40e_vsi_configure_msix(struct i40e_vsi *vsi)
2440 {
2441         struct i40e_pf *pf = vsi->back;
2442         struct i40e_q_vector *q_vector;
2443         struct i40e_hw *hw = &pf->hw;
2444         u16 vector;
2445         int i, q;
2446         u32 val;
2447         u32 qp;
2448
2449         /* The interrupt indexing is offset by 1 in the PFINT_ITRn
2450          * and PFINT_LNKLSTn registers, e.g.:
2451          *   PFINT_ITRn[0..n-1] gets msix-1..msix-n  (qpair interrupts)
2452          */
2453         qp = vsi->base_queue;
2454         vector = vsi->base_vector;
2455         for (i = 0; i < vsi->num_q_vectors; i++, vector++) {
2456                 q_vector = vsi->q_vectors[i];
2457                 q_vector->rx.itr = ITR_TO_REG(vsi->rx_itr_setting);
2458                 q_vector->rx.latency_range = I40E_LOW_LATENCY;
2459                 wr32(hw, I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1),
2460                      q_vector->rx.itr);
2461                 q_vector->tx.itr = ITR_TO_REG(vsi->tx_itr_setting);
2462                 q_vector->tx.latency_range = I40E_LOW_LATENCY;
2463                 wr32(hw, I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1),
2464                      q_vector->tx.itr);
2465
2466                 /* Linked list for the queuepairs assigned to this vector */
2467                 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), qp);
2468                 for (q = 0; q < q_vector->num_ringpairs; q++) {
2469                         val = I40E_QINT_RQCTL_CAUSE_ENA_MASK |
2470                               (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT)  |
2471                               (vector      << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
2472                               (qp          << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT)|
2473                               (I40E_QUEUE_TYPE_TX
2474                                       << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT);
2475
2476                         wr32(hw, I40E_QINT_RQCTL(qp), val);
2477
2478                         val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
2479                               (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT)  |
2480                               (vector      << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
2481                               ((qp+1)      << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT)|
2482                               (I40E_QUEUE_TYPE_RX
2483                                       << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
2484
2485                         /* Terminate the linked list */
2486                         if (q == (q_vector->num_ringpairs - 1))
2487                                 val |= (I40E_QUEUE_END_OF_LIST
2488                                            << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
2489
2490                         wr32(hw, I40E_QINT_TQCTL(qp), val);
2491                         qp++;
2492                 }
2493         }
2494
2495         i40e_flush(hw);
2496 }
2497
2498 /**
2499  * i40e_enable_misc_int_causes - enable the non-queue interrupts
2500  * @hw: ptr to the hardware info
2501  **/
2502 static void i40e_enable_misc_int_causes(struct i40e_hw *hw)
2503 {
2504         u32 val;
2505
2506         /* clear things first */
2507         wr32(hw, I40E_PFINT_ICR0_ENA, 0);  /* disable all */
2508         rd32(hw, I40E_PFINT_ICR0);         /* read to clear */
2509
2510         val = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK       |
2511               I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK    |
2512               I40E_PFINT_ICR0_ENA_GRST_MASK          |
2513               I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK |
2514               I40E_PFINT_ICR0_ENA_GPIO_MASK          |
2515               I40E_PFINT_ICR0_ENA_TIMESYNC_MASK      |
2516               I40E_PFINT_ICR0_ENA_STORM_DETECT_MASK  |
2517               I40E_PFINT_ICR0_ENA_HMC_ERR_MASK       |
2518               I40E_PFINT_ICR0_ENA_VFLR_MASK          |
2519               I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
2520
2521         wr32(hw, I40E_PFINT_ICR0_ENA, val);
2522
2523         /* SW_ITR_IDX = 0, but don't change INTENA */
2524         wr32(hw, I40E_PFINT_DYN_CTL0, I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK |
2525                                         I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK);
2526
2527         /* OTHER_ITR_IDX = 0 */
2528         wr32(hw, I40E_PFINT_STAT_CTL0, 0);
2529 }
2530
2531 /**
2532  * i40e_configure_msi_and_legacy - Legacy mode interrupt config in the HW
2533  * @vsi: the VSI being configured
2534  **/
2535 static void i40e_configure_msi_and_legacy(struct i40e_vsi *vsi)
2536 {
2537         struct i40e_q_vector *q_vector = vsi->q_vectors[0];
2538         struct i40e_pf *pf = vsi->back;
2539         struct i40e_hw *hw = &pf->hw;
2540         u32 val;
2541
2542         /* set the ITR configuration */
2543         q_vector->rx.itr = ITR_TO_REG(vsi->rx_itr_setting);
2544         q_vector->rx.latency_range = I40E_LOW_LATENCY;
2545         wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), q_vector->rx.itr);
2546         q_vector->tx.itr = ITR_TO_REG(vsi->tx_itr_setting);
2547         q_vector->tx.latency_range = I40E_LOW_LATENCY;
2548         wr32(hw, I40E_PFINT_ITR0(I40E_TX_ITR), q_vector->tx.itr);
2549
2550         i40e_enable_misc_int_causes(hw);
2551
2552         /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */
2553         wr32(hw, I40E_PFINT_LNKLST0, 0);
2554
2555         /* Associate the queue pair to the vector and enable the q int */
2556         val = I40E_QINT_RQCTL_CAUSE_ENA_MASK                  |
2557               (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) |
2558               (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
2559
2560         wr32(hw, I40E_QINT_RQCTL(0), val);
2561
2562         val = I40E_QINT_TQCTL_CAUSE_ENA_MASK                  |
2563               (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
2564               (I40E_QUEUE_END_OF_LIST << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
2565
2566         wr32(hw, I40E_QINT_TQCTL(0), val);
2567         i40e_flush(hw);
2568 }
2569
2570 /**
2571  * i40e_irq_dynamic_disable_icr0 - Disable default interrupt generation for icr0
2572  * @pf: board private structure
2573  **/
2574 void i40e_irq_dynamic_disable_icr0(struct i40e_pf *pf)
2575 {
2576         struct i40e_hw *hw = &pf->hw;
2577
2578         wr32(hw, I40E_PFINT_DYN_CTL0,
2579              I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
2580         i40e_flush(hw);
2581 }
2582
2583 /**
2584  * i40e_irq_dynamic_enable_icr0 - Enable default interrupt generation for icr0
2585  * @pf: board private structure
2586  **/
2587 void i40e_irq_dynamic_enable_icr0(struct i40e_pf *pf)
2588 {
2589         struct i40e_hw *hw = &pf->hw;
2590         u32 val;
2591
2592         val = I40E_PFINT_DYN_CTL0_INTENA_MASK   |
2593               I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
2594               (I40E_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT);
2595
2596         wr32(hw, I40E_PFINT_DYN_CTL0, val);
2597         i40e_flush(hw);
2598 }
2599
2600 /**
2601  * i40e_irq_dynamic_enable - Enable default interrupt generation settings
2602  * @vsi: pointer to a vsi
2603  * @vector: enable a particular Hw Interrupt vector
2604  **/
2605 void i40e_irq_dynamic_enable(struct i40e_vsi *vsi, int vector)
2606 {
2607         struct i40e_pf *pf = vsi->back;
2608         struct i40e_hw *hw = &pf->hw;
2609         u32 val;
2610
2611         val = I40E_PFINT_DYN_CTLN_INTENA_MASK |
2612               I40E_PFINT_DYN_CTLN_CLEARPBA_MASK |
2613               (I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
2614         wr32(hw, I40E_PFINT_DYN_CTLN(vector - 1), val);
2615         /* skip the flush */
2616 }
2617
2618 /**
2619  * i40e_msix_clean_rings - MSIX mode Interrupt Handler
2620  * @irq: interrupt number
2621  * @data: pointer to a q_vector
2622  **/
2623 static irqreturn_t i40e_msix_clean_rings(int irq, void *data)
2624 {
2625         struct i40e_q_vector *q_vector = data;
2626
2627         if (!q_vector->tx.ring && !q_vector->rx.ring)
2628                 return IRQ_HANDLED;
2629
2630         napi_schedule(&q_vector->napi);
2631
2632         return IRQ_HANDLED;
2633 }
2634
2635 /**
2636  * i40e_fdir_clean_rings - Interrupt Handler for FDIR rings
2637  * @irq: interrupt number
2638  * @data: pointer to a q_vector
2639  **/
2640 static irqreturn_t i40e_fdir_clean_rings(int irq, void *data)
2641 {
2642         struct i40e_q_vector *q_vector = data;
2643
2644         if (!q_vector->tx.ring && !q_vector->rx.ring)
2645                 return IRQ_HANDLED;
2646
2647         pr_info("fdir ring cleaning needed\n");
2648
2649         return IRQ_HANDLED;
2650 }
2651
2652 /**
2653  * i40e_vsi_request_irq_msix - Initialize MSI-X interrupts
2654  * @vsi: the VSI being configured
2655  * @basename: name for the vector
2656  *
2657  * Allocates MSI-X vectors and requests interrupts from the kernel.
2658  **/
2659 static int i40e_vsi_request_irq_msix(struct i40e_vsi *vsi, char *basename)
2660 {
2661         int q_vectors = vsi->num_q_vectors;
2662         struct i40e_pf *pf = vsi->back;
2663         int base = vsi->base_vector;
2664         int rx_int_idx = 0;
2665         int tx_int_idx = 0;
2666         int vector, err;
2667
2668         for (vector = 0; vector < q_vectors; vector++) {
2669                 struct i40e_q_vector *q_vector = vsi->q_vectors[vector];
2670
2671                 if (q_vector->tx.ring && q_vector->rx.ring) {
2672                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
2673                                  "%s-%s-%d", basename, "TxRx", rx_int_idx++);
2674                         tx_int_idx++;
2675                 } else if (q_vector->rx.ring) {
2676                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
2677                                  "%s-%s-%d", basename, "rx", rx_int_idx++);
2678                 } else if (q_vector->tx.ring) {
2679                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
2680                                  "%s-%s-%d", basename, "tx", tx_int_idx++);
2681                 } else {
2682                         /* skip this unused q_vector */
2683                         continue;
2684                 }
2685                 err = request_irq(pf->msix_entries[base + vector].vector,
2686                                   vsi->irq_handler,
2687                                   0,
2688                                   q_vector->name,
2689                                   q_vector);
2690                 if (err) {
2691                         dev_info(&pf->pdev->dev,
2692                                  "%s: request_irq failed, error: %d\n",
2693                                  __func__, err);
2694                         goto free_queue_irqs;
2695                 }
2696                 /* assign the mask for this irq */
2697                 irq_set_affinity_hint(pf->msix_entries[base + vector].vector,
2698                                       &q_vector->affinity_mask);
2699         }
2700
2701         return 0;
2702
2703 free_queue_irqs:
2704         while (vector) {
2705                 vector--;
2706                 irq_set_affinity_hint(pf->msix_entries[base + vector].vector,
2707                                       NULL);
2708                 free_irq(pf->msix_entries[base + vector].vector,
2709                          &(vsi->q_vectors[vector]));
2710         }
2711         return err;
2712 }
2713
2714 /**
2715  * i40e_vsi_disable_irq - Mask off queue interrupt generation on the VSI
2716  * @vsi: the VSI being un-configured
2717  **/
2718 static void i40e_vsi_disable_irq(struct i40e_vsi *vsi)
2719 {
2720         struct i40e_pf *pf = vsi->back;
2721         struct i40e_hw *hw = &pf->hw;
2722         int base = vsi->base_vector;
2723         int i;
2724
2725         for (i = 0; i < vsi->num_queue_pairs; i++) {
2726                 wr32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx), 0);
2727                 wr32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx), 0);
2728         }
2729
2730         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
2731                 for (i = vsi->base_vector;
2732                      i < (vsi->num_q_vectors + vsi->base_vector); i++)
2733                         wr32(hw, I40E_PFINT_DYN_CTLN(i - 1), 0);
2734
2735                 i40e_flush(hw);
2736                 for (i = 0; i < vsi->num_q_vectors; i++)
2737                         synchronize_irq(pf->msix_entries[i + base].vector);
2738         } else {
2739                 /* Legacy and MSI mode - this stops all interrupt handling */
2740                 wr32(hw, I40E_PFINT_ICR0_ENA, 0);
2741                 wr32(hw, I40E_PFINT_DYN_CTL0, 0);
2742                 i40e_flush(hw);
2743                 synchronize_irq(pf->pdev->irq);
2744         }
2745 }
2746
2747 /**
2748  * i40e_vsi_enable_irq - Enable IRQ for the given VSI
2749  * @vsi: the VSI being configured
2750  **/
2751 static int i40e_vsi_enable_irq(struct i40e_vsi *vsi)
2752 {
2753         struct i40e_pf *pf = vsi->back;
2754         int i;
2755
2756         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
2757                 for (i = vsi->base_vector;
2758                      i < (vsi->num_q_vectors + vsi->base_vector); i++)
2759                         i40e_irq_dynamic_enable(vsi, i);
2760         } else {
2761                 i40e_irq_dynamic_enable_icr0(pf);
2762         }
2763
2764         i40e_flush(&pf->hw);
2765         return 0;
2766 }
2767
2768 /**
2769  * i40e_stop_misc_vector - Stop the vector that handles non-queue events
2770  * @pf: board private structure
2771  **/
2772 static void i40e_stop_misc_vector(struct i40e_pf *pf)
2773 {
2774         /* Disable ICR 0 */
2775         wr32(&pf->hw, I40E_PFINT_ICR0_ENA, 0);
2776         i40e_flush(&pf->hw);
2777 }
2778
2779 /**
2780  * i40e_intr - MSI/Legacy and non-queue interrupt handler
2781  * @irq: interrupt number
2782  * @data: pointer to a q_vector
2783  *
2784  * This is the handler used for all MSI/Legacy interrupts, and deals
2785  * with both queue and non-queue interrupts.  This is also used in
2786  * MSIX mode to handle the non-queue interrupts.
2787  **/
2788 static irqreturn_t i40e_intr(int irq, void *data)
2789 {
2790         struct i40e_pf *pf = (struct i40e_pf *)data;
2791         struct i40e_hw *hw = &pf->hw;
2792         irqreturn_t ret = IRQ_NONE;
2793         u32 icr0, icr0_remaining;
2794         u32 val, ena_mask;
2795
2796         icr0 = rd32(hw, I40E_PFINT_ICR0);
2797         ena_mask = rd32(hw, I40E_PFINT_ICR0_ENA);
2798
2799         /* if sharing a legacy IRQ, we might get called w/o an intr pending */
2800         if ((icr0 & I40E_PFINT_ICR0_INTEVENT_MASK) == 0)
2801                 goto enable_intr;
2802
2803         /* if interrupt but no bits showing, must be SWINT */
2804         if (((icr0 & ~I40E_PFINT_ICR0_INTEVENT_MASK) == 0) ||
2805             (icr0 & I40E_PFINT_ICR0_SWINT_MASK))
2806                 pf->sw_int_count++;
2807
2808         /* only q0 is used in MSI/Legacy mode, and none are used in MSIX */
2809         if (icr0 & I40E_PFINT_ICR0_QUEUE_0_MASK) {
2810
2811                 /* temporarily disable queue cause for NAPI processing */
2812                 u32 qval = rd32(hw, I40E_QINT_RQCTL(0));
2813                 qval &= ~I40E_QINT_RQCTL_CAUSE_ENA_MASK;
2814                 wr32(hw, I40E_QINT_RQCTL(0), qval);
2815
2816                 qval = rd32(hw, I40E_QINT_TQCTL(0));
2817                 qval &= ~I40E_QINT_TQCTL_CAUSE_ENA_MASK;
2818                 wr32(hw, I40E_QINT_TQCTL(0), qval);
2819
2820                 if (!test_bit(__I40E_DOWN, &pf->state))
2821                         napi_schedule(&pf->vsi[pf->lan_vsi]->q_vectors[0]->napi);
2822         }
2823
2824         if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) {
2825                 ena_mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
2826                 set_bit(__I40E_ADMINQ_EVENT_PENDING, &pf->state);
2827         }
2828
2829         if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK) {
2830                 ena_mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
2831                 set_bit(__I40E_MDD_EVENT_PENDING, &pf->state);
2832         }
2833
2834         if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) {
2835                 ena_mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK;
2836                 set_bit(__I40E_VFLR_EVENT_PENDING, &pf->state);
2837         }
2838
2839         if (icr0 & I40E_PFINT_ICR0_GRST_MASK) {
2840                 if (!test_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state))
2841                         set_bit(__I40E_RESET_INTR_RECEIVED, &pf->state);
2842                 ena_mask &= ~I40E_PFINT_ICR0_ENA_GRST_MASK;
2843                 val = rd32(hw, I40E_GLGEN_RSTAT);
2844                 val = (val & I40E_GLGEN_RSTAT_RESET_TYPE_MASK)
2845                        >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT;
2846                 if (val == I40E_RESET_CORER)
2847                         pf->corer_count++;
2848                 else if (val == I40E_RESET_GLOBR)
2849                         pf->globr_count++;
2850                 else if (val == I40E_RESET_EMPR)
2851                         pf->empr_count++;
2852         }
2853
2854         if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK) {
2855                 icr0 &= ~I40E_PFINT_ICR0_HMC_ERR_MASK;
2856                 dev_info(&pf->pdev->dev, "HMC error interrupt\n");
2857         }
2858
2859         if (icr0 & I40E_PFINT_ICR0_TIMESYNC_MASK) {
2860                 u32 prttsyn_stat = rd32(hw, I40E_PRTTSYN_STAT_0);
2861
2862                 if (prttsyn_stat & I40E_PRTTSYN_STAT_0_TXTIME_MASK) {
2863                         ena_mask &= ~I40E_PFINT_ICR0_ENA_TIMESYNC_MASK;
2864                         i40e_ptp_tx_hwtstamp(pf);
2865                         prttsyn_stat &= ~I40E_PRTTSYN_STAT_0_TXTIME_MASK;
2866                 }
2867
2868                 wr32(hw, I40E_PRTTSYN_STAT_0, prttsyn_stat);
2869         }
2870
2871         /* If a critical error is pending we have no choice but to reset the
2872          * device.
2873          * Report and mask out any remaining unexpected interrupts.
2874          */
2875         icr0_remaining = icr0 & ena_mask;
2876         if (icr0_remaining) {
2877                 dev_info(&pf->pdev->dev, "unhandled interrupt icr0=0x%08x\n",
2878                          icr0_remaining);
2879                 if ((icr0_remaining & I40E_PFINT_ICR0_PE_CRITERR_MASK) ||
2880                     (icr0_remaining & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) ||
2881                     (icr0_remaining & I40E_PFINT_ICR0_ECC_ERR_MASK) ||
2882                     (icr0_remaining & I40E_PFINT_ICR0_MAL_DETECT_MASK)) {
2883                         dev_info(&pf->pdev->dev, "device will be reset\n");
2884                         set_bit(__I40E_PF_RESET_REQUESTED, &pf->state);
2885                         i40e_service_event_schedule(pf);
2886                 }
2887                 ena_mask &= ~icr0_remaining;
2888         }
2889         ret = IRQ_HANDLED;
2890
2891 enable_intr:
2892         /* re-enable interrupt causes */
2893         wr32(hw, I40E_PFINT_ICR0_ENA, ena_mask);
2894         if (!test_bit(__I40E_DOWN, &pf->state)) {
2895                 i40e_service_event_schedule(pf);
2896                 i40e_irq_dynamic_enable_icr0(pf);
2897         }
2898
2899         return ret;
2900 }
2901
2902 /**
2903  * i40e_map_vector_to_qp - Assigns the queue pair to the vector
2904  * @vsi: the VSI being configured
2905  * @v_idx: vector index
2906  * @qp_idx: queue pair index
2907  **/
2908 static void map_vector_to_qp(struct i40e_vsi *vsi, int v_idx, int qp_idx)
2909 {
2910         struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
2911         struct i40e_ring *tx_ring = vsi->tx_rings[qp_idx];
2912         struct i40e_ring *rx_ring = vsi->rx_rings[qp_idx];
2913
2914         tx_ring->q_vector = q_vector;
2915         tx_ring->next = q_vector->tx.ring;
2916         q_vector->tx.ring = tx_ring;
2917         q_vector->tx.count++;
2918
2919         rx_ring->q_vector = q_vector;
2920         rx_ring->next = q_vector->rx.ring;
2921         q_vector->rx.ring = rx_ring;
2922         q_vector->rx.count++;
2923 }
2924
2925 /**
2926  * i40e_vsi_map_rings_to_vectors - Maps descriptor rings to vectors
2927  * @vsi: the VSI being configured
2928  *
2929  * This function maps descriptor rings to the queue-specific vectors
2930  * we were allotted through the MSI-X enabling code.  Ideally, we'd have
2931  * one vector per queue pair, but on a constrained vector budget, we
2932  * group the queue pairs as "efficiently" as possible.
2933  **/
2934 static void i40e_vsi_map_rings_to_vectors(struct i40e_vsi *vsi)
2935 {
2936         int qp_remaining = vsi->num_queue_pairs;
2937         int q_vectors = vsi->num_q_vectors;
2938         int num_ringpairs;
2939         int v_start = 0;
2940         int qp_idx = 0;
2941
2942         /* If we don't have enough vectors for a 1-to-1 mapping, we'll have to
2943          * group them so there are multiple queues per vector.
2944          */
2945         for (; v_start < q_vectors && qp_remaining; v_start++) {
2946                 struct i40e_q_vector *q_vector = vsi->q_vectors[v_start];
2947
2948                 num_ringpairs = DIV_ROUND_UP(qp_remaining, q_vectors - v_start);
2949
2950                 q_vector->num_ringpairs = num_ringpairs;
2951
2952                 q_vector->rx.count = 0;
2953                 q_vector->tx.count = 0;
2954                 q_vector->rx.ring = NULL;
2955                 q_vector->tx.ring = NULL;
2956
2957                 while (num_ringpairs--) {
2958                         map_vector_to_qp(vsi, v_start, qp_idx);
2959                         qp_idx++;
2960                         qp_remaining--;
2961                 }
2962         }
2963 }
2964
2965 /**
2966  * i40e_vsi_request_irq - Request IRQ from the OS
2967  * @vsi: the VSI being configured
2968  * @basename: name for the vector
2969  **/
2970 static int i40e_vsi_request_irq(struct i40e_vsi *vsi, char *basename)
2971 {
2972         struct i40e_pf *pf = vsi->back;
2973         int err;
2974
2975         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
2976                 err = i40e_vsi_request_irq_msix(vsi, basename);
2977         else if (pf->flags & I40E_FLAG_MSI_ENABLED)
2978                 err = request_irq(pf->pdev->irq, i40e_intr, 0,
2979                                   pf->misc_int_name, pf);
2980         else
2981                 err = request_irq(pf->pdev->irq, i40e_intr, IRQF_SHARED,
2982                                   pf->misc_int_name, pf);
2983
2984         if (err)
2985                 dev_info(&pf->pdev->dev, "request_irq failed, Error %d\n", err);
2986
2987         return err;
2988 }
2989
2990 #ifdef CONFIG_NET_POLL_CONTROLLER
2991 /**
2992  * i40e_netpoll - A Polling 'interrupt'handler
2993  * @netdev: network interface device structure
2994  *
2995  * This is used by netconsole to send skbs without having to re-enable
2996  * interrupts.  It's not called while the normal interrupt routine is executing.
2997  **/
2998 static void i40e_netpoll(struct net_device *netdev)
2999 {
3000         struct i40e_netdev_priv *np = netdev_priv(netdev);
3001         struct i40e_vsi *vsi = np->vsi;
3002         struct i40e_pf *pf = vsi->back;
3003         int i;
3004
3005         /* if interface is down do nothing */
3006         if (test_bit(__I40E_DOWN, &vsi->state))
3007                 return;
3008
3009         pf->flags |= I40E_FLAG_IN_NETPOLL;
3010         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
3011                 for (i = 0; i < vsi->num_q_vectors; i++)
3012                         i40e_msix_clean_rings(0, vsi->q_vectors[i]);
3013         } else {
3014                 i40e_intr(pf->pdev->irq, netdev);
3015         }
3016         pf->flags &= ~I40E_FLAG_IN_NETPOLL;
3017 }
3018 #endif
3019
3020 /**
3021  * i40e_vsi_control_tx - Start or stop a VSI's rings
3022  * @vsi: the VSI being configured
3023  * @enable: start or stop the rings
3024  **/
3025 static int i40e_vsi_control_tx(struct i40e_vsi *vsi, bool enable)
3026 {
3027         struct i40e_pf *pf = vsi->back;
3028         struct i40e_hw *hw = &pf->hw;
3029         int i, j, pf_q;
3030         u32 tx_reg;
3031
3032         pf_q = vsi->base_queue;
3033         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
3034                 j = 1000;
3035                 do {
3036                         usleep_range(1000, 2000);
3037                         tx_reg = rd32(hw, I40E_QTX_ENA(pf_q));
3038                 } while (j-- && ((tx_reg >> I40E_QTX_ENA_QENA_REQ_SHIFT)
3039                                ^ (tx_reg >> I40E_QTX_ENA_QENA_STAT_SHIFT)) & 1);
3040
3041                 /* Skip if the queue is already in the requested state */
3042                 if (enable && (tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
3043                         continue;
3044                 if (!enable && !(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
3045                         continue;
3046
3047                 /* turn on/off the queue */
3048                 if (enable) {
3049                         wr32(hw, I40E_QTX_HEAD(pf_q), 0);
3050                         tx_reg |= I40E_QTX_ENA_QENA_REQ_MASK |
3051                                   I40E_QTX_ENA_QENA_STAT_MASK;
3052                 } else {
3053                         tx_reg &= ~I40E_QTX_ENA_QENA_REQ_MASK;
3054                 }
3055
3056                 wr32(hw, I40E_QTX_ENA(pf_q), tx_reg);
3057
3058                 /* wait for the change to finish */
3059                 for (j = 0; j < 10; j++) {
3060                         tx_reg = rd32(hw, I40E_QTX_ENA(pf_q));
3061                         if (enable) {
3062                                 if ((tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
3063                                         break;
3064                         } else {
3065                                 if (!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
3066                                         break;
3067                         }
3068
3069                         udelay(10);
3070                 }
3071                 if (j >= 10) {
3072                         dev_info(&pf->pdev->dev, "Tx ring %d %sable timeout\n",
3073                                  pf_q, (enable ? "en" : "dis"));
3074                         return -ETIMEDOUT;
3075                 }
3076         }
3077
3078         if (hw->revision_id == 0)
3079                 mdelay(50);
3080
3081         return 0;
3082 }
3083
3084 /**
3085  * i40e_vsi_control_rx - Start or stop a VSI's rings
3086  * @vsi: the VSI being configured
3087  * @enable: start or stop the rings
3088  **/
3089 static int i40e_vsi_control_rx(struct i40e_vsi *vsi, bool enable)
3090 {
3091         struct i40e_pf *pf = vsi->back;
3092         struct i40e_hw *hw = &pf->hw;
3093         int i, j, pf_q;
3094         u32 rx_reg;
3095
3096         pf_q = vsi->base_queue;
3097         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
3098                 j = 1000;
3099                 do {
3100                         usleep_range(1000, 2000);
3101                         rx_reg = rd32(hw, I40E_QRX_ENA(pf_q));
3102                 } while (j-- && ((rx_reg >> I40E_QRX_ENA_QENA_REQ_SHIFT)
3103                                ^ (rx_reg >> I40E_QRX_ENA_QENA_STAT_SHIFT)) & 1);
3104
3105                 if (enable) {
3106                         /* is STAT set ? */
3107                         if ((rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
3108                                 continue;
3109                 } else {
3110                         /* is !STAT set ? */
3111                         if (!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
3112                                 continue;
3113                 }
3114
3115                 /* turn on/off the queue */
3116                 if (enable)
3117                         rx_reg |= I40E_QRX_ENA_QENA_REQ_MASK |
3118                                   I40E_QRX_ENA_QENA_STAT_MASK;
3119                 else
3120                         rx_reg &= ~(I40E_QRX_ENA_QENA_REQ_MASK |
3121                                   I40E_QRX_ENA_QENA_STAT_MASK);
3122                 wr32(hw, I40E_QRX_ENA(pf_q), rx_reg);
3123
3124                 /* wait for the change to finish */
3125                 for (j = 0; j < 10; j++) {
3126                         rx_reg = rd32(hw, I40E_QRX_ENA(pf_q));
3127
3128                         if (enable) {
3129                                 if ((rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
3130                                         break;
3131                         } else {
3132                                 if (!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
3133                                         break;
3134                         }
3135
3136                         udelay(10);
3137                 }
3138                 if (j >= 10) {
3139                         dev_info(&pf->pdev->dev, "Rx ring %d %sable timeout\n",
3140                                  pf_q, (enable ? "en" : "dis"));
3141                         return -ETIMEDOUT;
3142                 }
3143         }
3144
3145         return 0;
3146 }
3147
3148 /**
3149  * i40e_vsi_control_rings - Start or stop a VSI's rings
3150  * @vsi: the VSI being configured
3151  * @enable: start or stop the rings
3152  **/
3153 int i40e_vsi_control_rings(struct i40e_vsi *vsi, bool request)
3154 {
3155         int ret = 0;
3156
3157         /* do rx first for enable and last for disable */
3158         if (request) {
3159                 ret = i40e_vsi_control_rx(vsi, request);
3160                 if (ret)
3161                         return ret;
3162                 ret = i40e_vsi_control_tx(vsi, request);
3163         } else {
3164                 /* Ignore return value, we need to shutdown whatever we can */
3165                 i40e_vsi_control_tx(vsi, request);
3166                 i40e_vsi_control_rx(vsi, request);
3167         }
3168
3169         return ret;
3170 }
3171
3172 /**
3173  * i40e_vsi_free_irq - Free the irq association with the OS
3174  * @vsi: the VSI being configured
3175  **/
3176 static void i40e_vsi_free_irq(struct i40e_vsi *vsi)
3177 {
3178         struct i40e_pf *pf = vsi->back;
3179         struct i40e_hw *hw = &pf->hw;
3180         int base = vsi->base_vector;
3181         u32 val, qp;
3182         int i;
3183
3184         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
3185                 if (!vsi->q_vectors)
3186                         return;
3187
3188                 for (i = 0; i < vsi->num_q_vectors; i++) {
3189                         u16 vector = i + base;
3190
3191                         /* free only the irqs that were actually requested */
3192                         if (!vsi->q_vectors[i] ||
3193                             !vsi->q_vectors[i]->num_ringpairs)
3194                                 continue;
3195
3196                         /* clear the affinity_mask in the IRQ descriptor */
3197                         irq_set_affinity_hint(pf->msix_entries[vector].vector,
3198                                               NULL);
3199                         free_irq(pf->msix_entries[vector].vector,
3200                                  vsi->q_vectors[i]);
3201
3202                         /* Tear down the interrupt queue link list
3203                          *
3204                          * We know that they come in pairs and always
3205                          * the Rx first, then the Tx.  To clear the
3206                          * link list, stick the EOL value into the
3207                          * next_q field of the registers.
3208                          */
3209                         val = rd32(hw, I40E_PFINT_LNKLSTN(vector - 1));
3210                         qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK)
3211                                 >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
3212                         val |= I40E_QUEUE_END_OF_LIST
3213                                 << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
3214                         wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), val);
3215
3216                         while (qp != I40E_QUEUE_END_OF_LIST) {
3217                                 u32 next;
3218
3219                                 val = rd32(hw, I40E_QINT_RQCTL(qp));
3220
3221                                 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK  |
3222                                          I40E_QINT_RQCTL_MSIX0_INDX_MASK |
3223                                          I40E_QINT_RQCTL_CAUSE_ENA_MASK  |
3224                                          I40E_QINT_RQCTL_INTEVENT_MASK);
3225
3226                                 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
3227                                          I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
3228
3229                                 wr32(hw, I40E_QINT_RQCTL(qp), val);
3230
3231                                 val = rd32(hw, I40E_QINT_TQCTL(qp));
3232
3233                                 next = (val & I40E_QINT_TQCTL_NEXTQ_INDX_MASK)
3234                                         >> I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT;
3235
3236                                 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK  |
3237                                          I40E_QINT_TQCTL_MSIX0_INDX_MASK |
3238                                          I40E_QINT_TQCTL_CAUSE_ENA_MASK  |
3239                                          I40E_QINT_TQCTL_INTEVENT_MASK);
3240
3241                                 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
3242                                          I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
3243
3244                                 wr32(hw, I40E_QINT_TQCTL(qp), val);
3245                                 qp = next;
3246                         }
3247                 }
3248         } else {
3249                 free_irq(pf->pdev->irq, pf);
3250
3251                 val = rd32(hw, I40E_PFINT_LNKLST0);
3252                 qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK)
3253                         >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
3254                 val |= I40E_QUEUE_END_OF_LIST
3255                         << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT;
3256                 wr32(hw, I40E_PFINT_LNKLST0, val);
3257
3258                 val = rd32(hw, I40E_QINT_RQCTL(qp));
3259                 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK  |
3260                          I40E_QINT_RQCTL_MSIX0_INDX_MASK |
3261                          I40E_QINT_RQCTL_CAUSE_ENA_MASK  |
3262                          I40E_QINT_RQCTL_INTEVENT_MASK);
3263
3264                 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
3265                         I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
3266
3267                 wr32(hw, I40E_QINT_RQCTL(qp), val);
3268
3269                 val = rd32(hw, I40E_QINT_TQCTL(qp));
3270
3271                 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK  |
3272                          I40E_QINT_TQCTL_MSIX0_INDX_MASK |
3273                          I40E_QINT_TQCTL_CAUSE_ENA_MASK  |
3274                          I40E_QINT_TQCTL_INTEVENT_MASK);
3275
3276                 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
3277                         I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
3278
3279                 wr32(hw, I40E_QINT_TQCTL(qp), val);
3280         }
3281 }
3282
3283 /**
3284  * i40e_free_q_vector - Free memory allocated for specific interrupt vector
3285  * @vsi: the VSI being configured
3286  * @v_idx: Index of vector to be freed
3287  *
3288  * This function frees the memory allocated to the q_vector.  In addition if
3289  * NAPI is enabled it will delete any references to the NAPI struct prior
3290  * to freeing the q_vector.
3291  **/
3292 static void i40e_free_q_vector(struct i40e_vsi *vsi, int v_idx)
3293 {
3294         struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
3295         struct i40e_ring *ring;
3296
3297         if (!q_vector)
3298                 return;
3299
3300         /* disassociate q_vector from rings */
3301         i40e_for_each_ring(ring, q_vector->tx)
3302                 ring->q_vector = NULL;
3303
3304         i40e_for_each_ring(ring, q_vector->rx)
3305                 ring->q_vector = NULL;
3306
3307         /* only VSI w/ an associated netdev is set up w/ NAPI */
3308         if (vsi->netdev)
3309                 netif_napi_del(&q_vector->napi);
3310
3311         vsi->q_vectors[v_idx] = NULL;
3312
3313         kfree_rcu(q_vector, rcu);
3314 }
3315
3316 /**
3317  * i40e_vsi_free_q_vectors - Free memory allocated for interrupt vectors
3318  * @vsi: the VSI being un-configured
3319  *
3320  * This frees the memory allocated to the q_vectors and
3321  * deletes references to the NAPI struct.
3322  **/
3323 static void i40e_vsi_free_q_vectors(struct i40e_vsi *vsi)
3324 {
3325         int v_idx;
3326
3327         for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++)
3328                 i40e_free_q_vector(vsi, v_idx);
3329 }
3330
3331 /**
3332  * i40e_reset_interrupt_capability - Disable interrupt setup in OS
3333  * @pf: board private structure
3334  **/
3335 static void i40e_reset_interrupt_capability(struct i40e_pf *pf)
3336 {
3337         /* If we're in Legacy mode, the interrupt was cleaned in vsi_close */
3338         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
3339                 pci_disable_msix(pf->pdev);
3340                 kfree(pf->msix_entries);
3341                 pf->msix_entries = NULL;
3342         } else if (pf->flags & I40E_FLAG_MSI_ENABLED) {
3343                 pci_disable_msi(pf->pdev);
3344         }
3345         pf->flags &= ~(I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED);
3346 }
3347
3348 /**
3349  * i40e_clear_interrupt_scheme - Clear the current interrupt scheme settings
3350  * @pf: board private structure
3351  *
3352  * We go through and clear interrupt specific resources and reset the structure
3353  * to pre-load conditions
3354  **/
3355 static void i40e_clear_interrupt_scheme(struct i40e_pf *pf)
3356 {
3357         int i;
3358
3359         i40e_put_lump(pf->irq_pile, 0, I40E_PILE_VALID_BIT-1);
3360         for (i = 0; i < pf->hw.func_caps.num_vsis; i++)
3361                 if (pf->vsi[i])
3362                         i40e_vsi_free_q_vectors(pf->vsi[i]);
3363         i40e_reset_interrupt_capability(pf);
3364 }
3365
3366 /**
3367  * i40e_napi_enable_all - Enable NAPI for all q_vectors in the VSI
3368  * @vsi: the VSI being configured
3369  **/
3370 static void i40e_napi_enable_all(struct i40e_vsi *vsi)
3371 {
3372         int q_idx;
3373
3374         if (!vsi->netdev)
3375                 return;
3376
3377         for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++)
3378                 napi_enable(&vsi->q_vectors[q_idx]->napi);
3379 }
3380
3381 /**
3382  * i40e_napi_disable_all - Disable NAPI for all q_vectors in the VSI
3383  * @vsi: the VSI being configured
3384  **/
3385 static void i40e_napi_disable_all(struct i40e_vsi *vsi)
3386 {
3387         int q_idx;
3388
3389         if (!vsi->netdev)
3390                 return;
3391
3392         for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++)
3393                 napi_disable(&vsi->q_vectors[q_idx]->napi);
3394 }
3395
3396 /**
3397  * i40e_quiesce_vsi - Pause a given VSI
3398  * @vsi: the VSI being paused
3399  **/
3400 static void i40e_quiesce_vsi(struct i40e_vsi *vsi)
3401 {
3402         if (test_bit(__I40E_DOWN, &vsi->state))
3403                 return;
3404
3405         set_bit(__I40E_NEEDS_RESTART, &vsi->state);
3406         if (vsi->netdev && netif_running(vsi->netdev)) {
3407                 vsi->netdev->netdev_ops->ndo_stop(vsi->netdev);
3408         } else {
3409                 set_bit(__I40E_DOWN, &vsi->state);
3410                 i40e_down(vsi);
3411         }
3412 }
3413
3414 /**
3415  * i40e_unquiesce_vsi - Resume a given VSI
3416  * @vsi: the VSI being resumed
3417  **/
3418 static void i40e_unquiesce_vsi(struct i40e_vsi *vsi)
3419 {
3420         if (!test_bit(__I40E_NEEDS_RESTART, &vsi->state))
3421                 return;
3422
3423         clear_bit(__I40E_NEEDS_RESTART, &vsi->state);
3424         if (vsi->netdev && netif_running(vsi->netdev))
3425                 vsi->netdev->netdev_ops->ndo_open(vsi->netdev);
3426         else
3427                 i40e_up(vsi);   /* this clears the DOWN bit */
3428 }
3429
3430 /**
3431  * i40e_pf_quiesce_all_vsi - Pause all VSIs on a PF
3432  * @pf: the PF
3433  **/
3434 static void i40e_pf_quiesce_all_vsi(struct i40e_pf *pf)
3435 {
3436         int v;
3437
3438         for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
3439                 if (pf->vsi[v])
3440                         i40e_quiesce_vsi(pf->vsi[v]);
3441         }
3442 }
3443
3444 /**
3445  * i40e_pf_unquiesce_all_vsi - Resume all VSIs on a PF
3446  * @pf: the PF
3447  **/
3448 static void i40e_pf_unquiesce_all_vsi(struct i40e_pf *pf)
3449 {
3450         int v;
3451
3452         for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
3453                 if (pf->vsi[v])
3454                         i40e_unquiesce_vsi(pf->vsi[v]);
3455         }
3456 }
3457
3458 /**
3459  * i40e_dcb_get_num_tc -  Get the number of TCs from DCBx config
3460  * @dcbcfg: the corresponding DCBx configuration structure
3461  *
3462  * Return the number of TCs from given DCBx configuration
3463  **/
3464 static u8 i40e_dcb_get_num_tc(struct i40e_dcbx_config *dcbcfg)
3465 {
3466         u8 num_tc = 0;
3467         int i;
3468
3469         /* Scan the ETS Config Priority Table to find
3470          * traffic class enabled for a given priority
3471          * and use the traffic class index to get the
3472          * number of traffic classes enabled
3473          */
3474         for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) {
3475                 if (dcbcfg->etscfg.prioritytable[i] > num_tc)
3476                         num_tc = dcbcfg->etscfg.prioritytable[i];
3477         }
3478
3479         /* Traffic class index starts from zero so
3480          * increment to return the actual count
3481          */
3482         return num_tc + 1;
3483 }
3484
3485 /**
3486  * i40e_dcb_get_enabled_tc - Get enabled traffic classes
3487  * @dcbcfg: the corresponding DCBx configuration structure
3488  *
3489  * Query the current DCB configuration and return the number of
3490  * traffic classes enabled from the given DCBX config
3491  **/
3492 static u8 i40e_dcb_get_enabled_tc(struct i40e_dcbx_config *dcbcfg)
3493 {
3494         u8 num_tc = i40e_dcb_get_num_tc(dcbcfg);
3495         u8 enabled_tc = 1;
3496         u8 i;
3497
3498         for (i = 0; i < num_tc; i++)
3499                 enabled_tc |= 1 << i;
3500
3501         return enabled_tc;
3502 }
3503
3504 /**
3505  * i40e_pf_get_num_tc - Get enabled traffic classes for PF
3506  * @pf: PF being queried
3507  *
3508  * Return number of traffic classes enabled for the given PF
3509  **/
3510 static u8 i40e_pf_get_num_tc(struct i40e_pf *pf)
3511 {
3512         struct i40e_hw *hw = &pf->hw;
3513         u8 i, enabled_tc;
3514         u8 num_tc = 0;
3515         struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
3516
3517         /* If DCB is not enabled then always in single TC */
3518         if (!(pf->flags & I40E_FLAG_DCB_ENABLED))
3519                 return 1;
3520
3521         /* MFP mode return count of enabled TCs for this PF */
3522         if (pf->flags & I40E_FLAG_MFP_ENABLED) {
3523                 enabled_tc = pf->hw.func_caps.enabled_tcmap;
3524                 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
3525                         if (enabled_tc & (1 << i))
3526                                 num_tc++;
3527                 }
3528                 return num_tc;
3529         }
3530
3531         /* SFP mode will be enabled for all TCs on port */
3532         return i40e_dcb_get_num_tc(dcbcfg);
3533 }
3534
3535 /**
3536  * i40e_pf_get_default_tc - Get bitmap for first enabled TC
3537  * @pf: PF being queried
3538  *
3539  * Return a bitmap for first enabled traffic class for this PF.
3540  **/
3541 static u8 i40e_pf_get_default_tc(struct i40e_pf *pf)
3542 {
3543         u8 enabled_tc = pf->hw.func_caps.enabled_tcmap;
3544         u8 i = 0;
3545
3546         if (!enabled_tc)
3547                 return 0x1; /* TC0 */
3548
3549         /* Find the first enabled TC */
3550         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
3551                 if (enabled_tc & (1 << i))
3552                         break;
3553         }
3554
3555         return 1 << i;
3556 }
3557
3558 /**
3559  * i40e_pf_get_pf_tc_map - Get bitmap for enabled traffic classes
3560  * @pf: PF being queried
3561  *
3562  * Return a bitmap for enabled traffic classes for this PF.
3563  **/
3564 static u8 i40e_pf_get_tc_map(struct i40e_pf *pf)
3565 {
3566         /* If DCB is not enabled for this PF then just return default TC */
3567         if (!(pf->flags & I40E_FLAG_DCB_ENABLED))
3568                 return i40e_pf_get_default_tc(pf);
3569
3570         /* MFP mode will have enabled TCs set by FW */
3571         if (pf->flags & I40E_FLAG_MFP_ENABLED)
3572                 return pf->hw.func_caps.enabled_tcmap;
3573
3574         /* SFP mode we want PF to be enabled for all TCs */
3575         return i40e_dcb_get_enabled_tc(&pf->hw.local_dcbx_config);
3576 }
3577
3578 /**
3579  * i40e_vsi_get_bw_info - Query VSI BW Information
3580  * @vsi: the VSI being queried
3581  *
3582  * Returns 0 on success, negative value on failure
3583  **/
3584 static int i40e_vsi_get_bw_info(struct i40e_vsi *vsi)
3585 {
3586         struct i40e_aqc_query_vsi_ets_sla_config_resp bw_ets_config = {0};
3587         struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0};
3588         struct i40e_pf *pf = vsi->back;
3589         struct i40e_hw *hw = &pf->hw;
3590         i40e_status aq_ret;
3591         u32 tc_bw_max;
3592         int i;
3593
3594         /* Get the VSI level BW configuration */
3595         aq_ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL);
3596         if (aq_ret) {
3597                 dev_info(&pf->pdev->dev,
3598                          "couldn't get pf vsi bw config, err %d, aq_err %d\n",
3599                          aq_ret, pf->hw.aq.asq_last_status);
3600                 return -EINVAL;
3601         }
3602
3603         /* Get the VSI level BW configuration per TC */
3604         aq_ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid, &bw_ets_config,
3605                                                   NULL);
3606         if (aq_ret) {
3607                 dev_info(&pf->pdev->dev,
3608                          "couldn't get pf vsi ets bw config, err %d, aq_err %d\n",
3609                          aq_ret, pf->hw.aq.asq_last_status);
3610                 return -EINVAL;
3611         }
3612
3613         if (bw_config.tc_valid_bits != bw_ets_config.tc_valid_bits) {
3614                 dev_info(&pf->pdev->dev,
3615                          "Enabled TCs mismatch from querying VSI BW info 0x%08x 0x%08x\n",
3616                          bw_config.tc_valid_bits,
3617                          bw_ets_config.tc_valid_bits);
3618                 /* Still continuing */
3619         }
3620
3621         vsi->bw_limit = le16_to_cpu(bw_config.port_bw_limit);
3622         vsi->bw_max_quanta = bw_config.max_bw;
3623         tc_bw_max = le16_to_cpu(bw_ets_config.tc_bw_max[0]) |
3624                     (le16_to_cpu(bw_ets_config.tc_bw_max[1]) << 16);
3625         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
3626                 vsi->bw_ets_share_credits[i] = bw_ets_config.share_credits[i];
3627                 vsi->bw_ets_limit_credits[i] =
3628                                         le16_to_cpu(bw_ets_config.credits[i]);
3629                 /* 3 bits out of 4 for each TC */
3630                 vsi->bw_ets_max_quanta[i] = (u8)((tc_bw_max >> (i*4)) & 0x7);
3631         }
3632
3633         return 0;
3634 }
3635
3636 /**
3637  * i40e_vsi_configure_bw_alloc - Configure VSI BW allocation per TC
3638  * @vsi: the VSI being configured
3639  * @enabled_tc: TC bitmap
3640  * @bw_credits: BW shared credits per TC
3641  *
3642  * Returns 0 on success, negative value on failure
3643  **/
3644 static int i40e_vsi_configure_bw_alloc(struct i40e_vsi *vsi, u8 enabled_tc,
3645                                        u8 *bw_share)
3646 {
3647         struct i40e_aqc_configure_vsi_tc_bw_data bw_data;
3648         i40e_status aq_ret;
3649         int i;
3650
3651         bw_data.tc_valid_bits = enabled_tc;
3652         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
3653                 bw_data.tc_bw_credits[i] = bw_share[i];
3654
3655         aq_ret = i40e_aq_config_vsi_tc_bw(&vsi->back->hw, vsi->seid, &bw_data,
3656                                           NULL);
3657         if (aq_ret) {
3658                 dev_info(&vsi->back->pdev->dev,
3659                          "%s: AQ command Config VSI BW allocation per TC failed = %d\n",
3660                          __func__, vsi->back->hw.aq.asq_last_status);
3661                 return -EINVAL;
3662         }
3663
3664         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
3665                 vsi->info.qs_handle[i] = bw_data.qs_handles[i];
3666
3667         return 0;
3668 }
3669
3670 /**
3671  * i40e_vsi_config_netdev_tc - Setup the netdev TC configuration
3672  * @vsi: the VSI being configured
3673  * @enabled_tc: TC map to be enabled
3674  *
3675  **/
3676 static void i40e_vsi_config_netdev_tc(struct i40e_vsi *vsi, u8 enabled_tc)
3677 {
3678         struct net_device *netdev = vsi->netdev;
3679         struct i40e_pf *pf = vsi->back;
3680         struct i40e_hw *hw = &pf->hw;
3681         u8 netdev_tc = 0;
3682         int i;
3683         struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
3684
3685         if (!netdev)
3686                 return;
3687
3688         if (!enabled_tc) {
3689                 netdev_reset_tc(netdev);
3690                 return;
3691         }
3692
3693         /* Set up actual enabled TCs on the VSI */
3694         if (netdev_set_num_tc(netdev, vsi->tc_config.numtc))
3695                 return;
3696
3697         /* set per TC queues for the VSI */
3698         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
3699                 /* Only set TC queues for enabled tcs
3700                  *
3701                  * e.g. For a VSI that has TC0 and TC3 enabled the
3702                  * enabled_tc bitmap would be 0x00001001; the driver
3703                  * will set the numtc for netdev as 2 that will be
3704                  * referenced by the netdev layer as TC 0 and 1.
3705                  */
3706                 if (vsi->tc_config.enabled_tc & (1 << i))
3707                         netdev_set_tc_queue(netdev,
3708                                         vsi->tc_config.tc_info[i].netdev_tc,
3709                                         vsi->tc_config.tc_info[i].qcount,
3710                                         vsi->tc_config.tc_info[i].qoffset);
3711         }
3712
3713         /* Assign UP2TC map for the VSI */
3714         for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) {
3715                 /* Get the actual TC# for the UP */
3716                 u8 ets_tc = dcbcfg->etscfg.prioritytable[i];
3717                 /* Get the mapped netdev TC# for the UP */
3718                 netdev_tc =  vsi->tc_config.tc_info[ets_tc].netdev_tc;
3719                 netdev_set_prio_tc_map(netdev, i, netdev_tc);
3720         }
3721 }
3722
3723 /**
3724  * i40e_vsi_update_queue_map - Update our copy of VSi info with new queue map
3725  * @vsi: the VSI being configured
3726  * @ctxt: the ctxt buffer returned from AQ VSI update param command
3727  **/
3728 static void i40e_vsi_update_queue_map(struct i40e_vsi *vsi,
3729                                       struct i40e_vsi_context *ctxt)
3730 {
3731         /* copy just the sections touched not the entire info
3732          * since not all sections are valid as returned by
3733          * update vsi params
3734          */
3735         vsi->info.mapping_flags = ctxt->info.mapping_flags;
3736         memcpy(&vsi->info.queue_mapping,
3737                &ctxt->info.queue_mapping, sizeof(vsi->info.queue_mapping));
3738         memcpy(&vsi->info.tc_mapping, ctxt->info.tc_mapping,
3739                sizeof(vsi->info.tc_mapping));
3740 }
3741
3742 /**
3743  * i40e_vsi_config_tc - Configure VSI Tx Scheduler for given TC map
3744  * @vsi: VSI to be configured
3745  * @enabled_tc: TC bitmap
3746  *
3747  * This configures a particular VSI for TCs that are mapped to the
3748  * given TC bitmap. It uses default bandwidth share for TCs across
3749  * VSIs to configure TC for a particular VSI.
3750  *
3751  * NOTE:
3752  * It is expected that the VSI queues have been quisced before calling
3753  * this function.
3754  **/
3755 static int i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 enabled_tc)
3756 {
3757         u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0};
3758         struct i40e_vsi_context ctxt;
3759         int ret = 0;
3760         int i;
3761
3762         /* Check if enabled_tc is same as existing or new TCs */
3763         if (vsi->tc_config.enabled_tc == enabled_tc)
3764                 return ret;
3765
3766         /* Enable ETS TCs with equal BW Share for now across all VSIs */
3767         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
3768                 if (enabled_tc & (1 << i))
3769                         bw_share[i] = 1;
3770         }
3771
3772         ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share);
3773         if (ret) {
3774                 dev_info(&vsi->back->pdev->dev,
3775                          "Failed configuring TC map %d for VSI %d\n",
3776                          enabled_tc, vsi->seid);
3777                 goto out;
3778         }
3779
3780         /* Update Queue Pairs Mapping for currently enabled UPs */
3781         ctxt.seid = vsi->seid;
3782         ctxt.pf_num = vsi->back->hw.pf_id;
3783         ctxt.vf_num = 0;
3784         ctxt.uplink_seid = vsi->uplink_seid;
3785         memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info));
3786         i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);
3787
3788         /* Update the VSI after updating the VSI queue-mapping information */
3789         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
3790         if (ret) {
3791                 dev_info(&vsi->back->pdev->dev,
3792                          "update vsi failed, aq_err=%d\n",
3793                          vsi->back->hw.aq.asq_last_status);
3794                 goto out;
3795         }
3796         /* update the local VSI info with updated queue map */
3797         i40e_vsi_update_queue_map(vsi, &ctxt);
3798         vsi->info.valid_sections = 0;
3799
3800         /* Update current VSI BW information */
3801         ret = i40e_vsi_get_bw_info(vsi);
3802         if (ret) {
3803                 dev_info(&vsi->back->pdev->dev,
3804                          "Failed updating vsi bw info, aq_err=%d\n",
3805                          vsi->back->hw.aq.asq_last_status);
3806                 goto out;
3807         }
3808
3809         /* Update the netdev TC setup */
3810         i40e_vsi_config_netdev_tc(vsi, enabled_tc);
3811 out:
3812         return ret;
3813 }
3814
3815 /**
3816  * i40e_up_complete - Finish the last steps of bringing up a connection
3817  * @vsi: the VSI being configured
3818  **/
3819 static int i40e_up_complete(struct i40e_vsi *vsi)
3820 {
3821         struct i40e_pf *pf = vsi->back;
3822         int err;
3823
3824         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
3825                 i40e_vsi_configure_msix(vsi);
3826         else
3827                 i40e_configure_msi_and_legacy(vsi);
3828
3829         /* start rings */
3830         err = i40e_vsi_control_rings(vsi, true);
3831         if (err)
3832                 return err;
3833
3834         clear_bit(__I40E_DOWN, &vsi->state);
3835         i40e_napi_enable_all(vsi);
3836         i40e_vsi_enable_irq(vsi);
3837
3838         if ((pf->hw.phy.link_info.link_info & I40E_AQ_LINK_UP) &&
3839             (vsi->netdev)) {
3840                 netdev_info(vsi->netdev, "NIC Link is Up\n");
3841                 netif_tx_start_all_queues(vsi->netdev);
3842                 netif_carrier_on(vsi->netdev);
3843         } else if (vsi->netdev) {
3844                 netdev_info(vsi->netdev, "NIC Link is Down\n");
3845         }
3846         i40e_service_event_schedule(pf);
3847
3848         return 0;
3849 }
3850
3851 /**
3852  * i40e_vsi_reinit_locked - Reset the VSI
3853  * @vsi: the VSI being configured
3854  *
3855  * Rebuild the ring structs after some configuration
3856  * has changed, e.g. MTU size.
3857  **/
3858 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi)
3859 {
3860         struct i40e_pf *pf = vsi->back;
3861
3862         WARN_ON(in_interrupt());
3863         while (test_and_set_bit(__I40E_CONFIG_BUSY, &pf->state))
3864                 usleep_range(1000, 2000);
3865         i40e_down(vsi);
3866
3867         /* Give a VF some time to respond to the reset.  The
3868          * two second wait is based upon the watchdog cycle in
3869          * the VF driver.
3870          */
3871         if (vsi->type == I40E_VSI_SRIOV)
3872                 msleep(2000);
3873         i40e_up(vsi);
3874         clear_bit(__I40E_CONFIG_BUSY, &pf->state);
3875 }
3876
3877 /**
3878  * i40e_up - Bring the connection back up after being down
3879  * @vsi: the VSI being configured
3880  **/
3881 int i40e_up(struct i40e_vsi *vsi)
3882 {
3883         int err;
3884
3885         err = i40e_vsi_configure(vsi);
3886         if (!err)
3887                 err = i40e_up_complete(vsi);
3888
3889         return err;
3890 }
3891
3892 /**
3893  * i40e_down - Shutdown the connection processing
3894  * @vsi: the VSI being stopped
3895  **/
3896 void i40e_down(struct i40e_vsi *vsi)
3897 {
3898         int i;
3899
3900         /* It is assumed that the caller of this function
3901          * sets the vsi->state __I40E_DOWN bit.
3902          */
3903         if (vsi->netdev) {
3904                 netif_carrier_off(vsi->netdev);
3905                 netif_tx_disable(vsi->netdev);
3906         }
3907         i40e_vsi_disable_irq(vsi);
3908         i40e_vsi_control_rings(vsi, false);
3909         i40e_napi_disable_all(vsi);
3910
3911         for (i = 0; i < vsi->num_queue_pairs; i++) {
3912                 i40e_clean_tx_ring(vsi->tx_rings[i]);
3913                 i40e_clean_rx_ring(vsi->rx_rings[i]);
3914         }
3915 }
3916
3917 /**
3918  * i40e_setup_tc - configure multiple traffic classes
3919  * @netdev: net device to configure
3920  * @tc: number of traffic classes to enable
3921  **/
3922 static int i40e_setup_tc(struct net_device *netdev, u8 tc)
3923 {
3924         struct i40e_netdev_priv *np = netdev_priv(netdev);
3925         struct i40e_vsi *vsi = np->vsi;
3926         struct i40e_pf *pf = vsi->back;
3927         u8 enabled_tc = 0;
3928         int ret = -EINVAL;
3929         int i;
3930
3931         /* Check if DCB enabled to continue */
3932         if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) {
3933                 netdev_info(netdev, "DCB is not enabled for adapter\n");
3934                 goto exit;
3935         }
3936
3937         /* Check if MFP enabled */
3938         if (pf->flags & I40E_FLAG_MFP_ENABLED) {
3939                 netdev_info(netdev, "Configuring TC not supported in MFP mode\n");
3940                 goto exit;
3941         }
3942
3943         /* Check whether tc count is within enabled limit */
3944         if (tc > i40e_pf_get_num_tc(pf)) {
3945                 netdev_info(netdev, "TC count greater than enabled on link for adapter\n");
3946                 goto exit;
3947         }
3948
3949         /* Generate TC map for number of tc requested */
3950         for (i = 0; i < tc; i++)
3951                 enabled_tc |= (1 << i);
3952
3953         /* Requesting same TC configuration as already enabled */
3954         if (enabled_tc == vsi->tc_config.enabled_tc)
3955                 return 0;
3956
3957         /* Quiesce VSI queues */
3958         i40e_quiesce_vsi(vsi);
3959
3960         /* Configure VSI for enabled TCs */
3961         ret = i40e_vsi_config_tc(vsi, enabled_tc);
3962         if (ret) {
3963                 netdev_info(netdev, "Failed configuring TC for VSI seid=%d\n",
3964                             vsi->seid);
3965                 goto exit;
3966         }
3967
3968         /* Unquiesce VSI */
3969         i40e_unquiesce_vsi(vsi);
3970
3971 exit:
3972         return ret;
3973 }
3974
3975 /**
3976  * i40e_open - Called when a network interface is made active
3977  * @netdev: network interface device structure
3978  *
3979  * The open entry point is called when a network interface is made
3980  * active by the system (IFF_UP).  At this point all resources needed
3981  * for transmit and receive operations are allocated, the interrupt
3982  * handler is registered with the OS, the netdev watchdog subtask is
3983  * enabled, and the stack is notified that the interface is ready.
3984  *
3985  * Returns 0 on success, negative value on failure
3986  **/
3987 static int i40e_open(struct net_device *netdev)
3988 {
3989         struct i40e_netdev_priv *np = netdev_priv(netdev);
3990         struct i40e_vsi *vsi = np->vsi;
3991         struct i40e_pf *pf = vsi->back;
3992         char int_name[IFNAMSIZ];
3993         int err;
3994
3995         /* disallow open during test */
3996         if (test_bit(__I40E_TESTING, &pf->state))
3997                 return -EBUSY;
3998
3999         netif_carrier_off(netdev);
4000
4001         /* allocate descriptors */
4002         err = i40e_vsi_setup_tx_resources(vsi);
4003         if (err)
4004                 goto err_setup_tx;
4005         err = i40e_vsi_setup_rx_resources(vsi);
4006         if (err)
4007                 goto err_setup_rx;
4008
4009         err = i40e_vsi_configure(vsi);
4010         if (err)
4011                 goto err_setup_rx;
4012
4013         snprintf(int_name, sizeof(int_name) - 1, "%s-%s",
4014                  dev_driver_string(&pf->pdev->dev), netdev->name);
4015         err = i40e_vsi_request_irq(vsi, int_name);
4016         if (err)
4017                 goto err_setup_rx;
4018
4019         /* Notify the stack of the actual queue counts. */
4020         err = netif_set_real_num_tx_queues(netdev, vsi->num_queue_pairs);
4021         if (err)
4022                 goto err_set_queues;
4023
4024         err = netif_set_real_num_rx_queues(netdev, vsi->num_queue_pairs);
4025         if (err)
4026                 goto err_set_queues;
4027
4028         err = i40e_up_complete(vsi);
4029         if (err)
4030                 goto err_up_complete;
4031
4032 #ifdef CONFIG_I40E_VXLAN
4033         vxlan_get_rx_port(netdev);
4034 #endif
4035
4036         return 0;
4037
4038 err_up_complete:
4039         i40e_down(vsi);
4040 err_set_queues:
4041         i40e_vsi_free_irq(vsi);
4042 err_setup_rx:
4043         i40e_vsi_free_rx_resources(vsi);
4044 err_setup_tx:
4045         i40e_vsi_free_tx_resources(vsi);
4046         if (vsi == pf->vsi[pf->lan_vsi])
4047                 i40e_do_reset(pf, (1 << __I40E_PF_RESET_REQUESTED));
4048
4049         return err;
4050 }
4051
4052 /**
4053  * i40e_close - Disables a network interface
4054  * @netdev: network interface device structure
4055  *
4056  * The close entry point is called when an interface is de-activated
4057  * by the OS.  The hardware is still under the driver's control, but
4058  * this netdev interface is disabled.
4059  *
4060  * Returns 0, this is not allowed to fail
4061  **/
4062 static int i40e_close(struct net_device *netdev)
4063 {
4064         struct i40e_netdev_priv *np = netdev_priv(netdev);
4065         struct i40e_vsi *vsi = np->vsi;
4066
4067         if (test_and_set_bit(__I40E_DOWN, &vsi->state))
4068                 return 0;
4069
4070         i40e_down(vsi);
4071         i40e_vsi_free_irq(vsi);
4072
4073         i40e_vsi_free_tx_resources(vsi);
4074         i40e_vsi_free_rx_resources(vsi);
4075
4076         return 0;
4077 }
4078
4079 /**
4080  * i40e_do_reset - Start a PF or Core Reset sequence
4081  * @pf: board private structure
4082  * @reset_flags: which reset is requested
4083  *
4084  * The essential difference in resets is that the PF Reset
4085  * doesn't clear the packet buffers, doesn't reset the PE
4086  * firmware, and doesn't bother the other PFs on the chip.
4087  **/
4088 void i40e_do_reset(struct i40e_pf *pf, u32 reset_flags)
4089 {
4090         u32 val;
4091
4092         WARN_ON(in_interrupt());
4093
4094         /* do the biggest reset indicated */
4095         if (reset_flags & (1 << __I40E_GLOBAL_RESET_REQUESTED)) {
4096
4097                 /* Request a Global Reset
4098                  *
4099                  * This will start the chip's countdown to the actual full
4100                  * chip reset event, and a warning interrupt to be sent
4101                  * to all PFs, including the requestor.  Our handler
4102                  * for the warning interrupt will deal with the shutdown
4103                  * and recovery of the switch setup.
4104                  */
4105                 dev_info(&pf->pdev->dev, "GlobalR requested\n");
4106                 val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
4107                 val |= I40E_GLGEN_RTRIG_GLOBR_MASK;
4108                 wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
4109
4110         } else if (reset_flags & (1 << __I40E_CORE_RESET_REQUESTED)) {
4111
4112                 /* Request a Core Reset
4113                  *
4114                  * Same as Global Reset, except does *not* include the MAC/PHY
4115                  */
4116                 dev_info(&pf->pdev->dev, "CoreR requested\n");
4117                 val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
4118                 val |= I40E_GLGEN_RTRIG_CORER_MASK;
4119                 wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
4120                 i40e_flush(&pf->hw);
4121
4122         } else if (reset_flags & (1 << __I40E_EMP_RESET_REQUESTED)) {
4123
4124                 /* Request a Firmware Reset
4125                  *
4126                  * Same as Global reset, plus restarting the
4127                  * embedded firmware engine.
4128                  */
4129                 /* enable EMP Reset */
4130                 val = rd32(&pf->hw, I40E_GLGEN_RSTENA_EMP);
4131                 val |= I40E_GLGEN_RSTENA_EMP_EMP_RST_ENA_MASK;
4132                 wr32(&pf->hw, I40E_GLGEN_RSTENA_EMP, val);
4133
4134                 /* force the reset */
4135                 val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
4136                 val |= I40E_GLGEN_RTRIG_EMPFWR_MASK;
4137                 wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
4138                 i40e_flush(&pf->hw);
4139
4140         } else if (reset_flags & (1 << __I40E_PF_RESET_REQUESTED)) {
4141
4142                 /* Request a PF Reset
4143                  *
4144                  * Resets only the PF-specific registers
4145                  *
4146                  * This goes directly to the tear-down and rebuild of
4147                  * the switch, since we need to do all the recovery as
4148                  * for the Core Reset.
4149                  */
4150                 dev_info(&pf->pdev->dev, "PFR requested\n");
4151                 i40e_handle_reset_warning(pf);
4152
4153         } else if (reset_flags & (1 << __I40E_REINIT_REQUESTED)) {
4154                 int v;
4155
4156                 /* Find the VSI(s) that requested a re-init */
4157                 dev_info(&pf->pdev->dev,
4158                          "VSI reinit requested\n");
4159                 for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
4160                         struct i40e_vsi *vsi = pf->vsi[v];
4161                         if (vsi != NULL &&
4162                             test_bit(__I40E_REINIT_REQUESTED, &vsi->state)) {
4163                                 i40e_vsi_reinit_locked(pf->vsi[v]);
4164                                 clear_bit(__I40E_REINIT_REQUESTED, &vsi->state);
4165                         }
4166                 }
4167
4168                 /* no further action needed, so return now */
4169                 return;
4170         } else {
4171                 dev_info(&pf->pdev->dev,
4172                          "bad reset request 0x%08x\n", reset_flags);
4173                 return;
4174         }
4175 }
4176
4177 /**
4178  * i40e_do_reset_safe - Protected reset path for userland calls.
4179  * @pf: board private structure
4180  * @reset_flags: which reset is requested
4181  *
4182  **/
4183 void i40e_do_reset_safe(struct i40e_pf *pf, u32 reset_flags)
4184 {
4185         rtnl_lock();
4186         i40e_do_reset(pf, reset_flags);
4187         rtnl_unlock();
4188 }
4189
4190 /**
4191  * i40e_handle_lan_overflow_event - Handler for LAN queue overflow event
4192  * @pf: board private structure
4193  * @e: event info posted on ARQ
4194  *
4195  * Handler for LAN Queue Overflow Event generated by the firmware for PF
4196  * and VF queues
4197  **/
4198 static void i40e_handle_lan_overflow_event(struct i40e_pf *pf,
4199                                            struct i40e_arq_event_info *e)
4200 {
4201         struct i40e_aqc_lan_overflow *data =
4202                 (struct i40e_aqc_lan_overflow *)&e->desc.params.raw;
4203         u32 queue = le32_to_cpu(data->prtdcb_rupto);
4204         u32 qtx_ctl = le32_to_cpu(data->otx_ctl);
4205         struct i40e_hw *hw = &pf->hw;
4206         struct i40e_vf *vf;
4207         u16 vf_id;
4208
4209         dev_info(&pf->pdev->dev, "%s: Rx Queue Number = %d QTX_CTL=0x%08x\n",
4210                  __func__, queue, qtx_ctl);
4211
4212         /* Queue belongs to VF, find the VF and issue VF reset */
4213         if (((qtx_ctl & I40E_QTX_CTL_PFVF_Q_MASK)
4214             >> I40E_QTX_CTL_PFVF_Q_SHIFT) == I40E_QTX_CTL_VF_QUEUE) {
4215                 vf_id = (u16)((qtx_ctl & I40E_QTX_CTL_VFVM_INDX_MASK)
4216                          >> I40E_QTX_CTL_VFVM_INDX_SHIFT);
4217                 vf_id -= hw->func_caps.vf_base_id;
4218                 vf = &pf->vf[vf_id];
4219                 i40e_vc_notify_vf_reset(vf);
4220                 /* Allow VF to process pending reset notification */
4221                 msleep(20);
4222                 i40e_reset_vf(vf, false);
4223         }
4224 }
4225
4226 /**
4227  * i40e_service_event_complete - Finish up the service event
4228  * @pf: board private structure
4229  **/
4230 static void i40e_service_event_complete(struct i40e_pf *pf)
4231 {
4232         BUG_ON(!test_bit(__I40E_SERVICE_SCHED, &pf->state));
4233
4234         /* flush memory to make sure state is correct before next watchog */
4235         smp_mb__before_clear_bit();
4236         clear_bit(__I40E_SERVICE_SCHED, &pf->state);
4237 }
4238
4239 /**
4240  * i40e_fdir_reinit_subtask - Worker thread to reinit FDIR filter table
4241  * @pf: board private structure
4242  **/
4243 static void i40e_fdir_reinit_subtask(struct i40e_pf *pf)
4244 {
4245         if (!(pf->flags & I40E_FLAG_FDIR_REQUIRES_REINIT))
4246                 return;
4247
4248         pf->flags &= ~I40E_FLAG_FDIR_REQUIRES_REINIT;
4249
4250         /* if interface is down do nothing */
4251         if (test_bit(__I40E_DOWN, &pf->state))
4252                 return;
4253 }
4254
4255 /**
4256  * i40e_vsi_link_event - notify VSI of a link event
4257  * @vsi: vsi to be notified
4258  * @link_up: link up or down
4259  **/
4260 static void i40e_vsi_link_event(struct i40e_vsi *vsi, bool link_up)
4261 {
4262         if (!vsi)
4263                 return;
4264
4265         switch (vsi->type) {
4266         case I40E_VSI_MAIN:
4267                 if (!vsi->netdev || !vsi->netdev_registered)
4268                         break;
4269
4270                 if (link_up) {
4271                         netif_carrier_on(vsi->netdev);
4272                         netif_tx_wake_all_queues(vsi->netdev);
4273                 } else {
4274                         netif_carrier_off(vsi->netdev);
4275                         netif_tx_stop_all_queues(vsi->netdev);
4276                 }
4277                 break;
4278
4279         case I40E_VSI_SRIOV:
4280                 break;
4281
4282         case I40E_VSI_VMDQ2:
4283         case I40E_VSI_CTRL:
4284         case I40E_VSI_MIRROR:
4285         default:
4286                 /* there is no notification for other VSIs */
4287                 break;
4288         }
4289 }
4290
4291 /**
4292  * i40e_veb_link_event - notify elements on the veb of a link event
4293  * @veb: veb to be notified
4294  * @link_up: link up or down
4295  **/
4296 static void i40e_veb_link_event(struct i40e_veb *veb, bool link_up)
4297 {
4298         struct i40e_pf *pf;
4299         int i;
4300
4301         if (!veb || !veb->pf)
4302                 return;
4303         pf = veb->pf;
4304
4305         /* depth first... */
4306         for (i = 0; i < I40E_MAX_VEB; i++)
4307                 if (pf->veb[i] && (pf->veb[i]->uplink_seid == veb->seid))
4308                         i40e_veb_link_event(pf->veb[i], link_up);
4309
4310         /* ... now the local VSIs */
4311         for (i = 0; i < pf->hw.func_caps.num_vsis; i++)
4312                 if (pf->vsi[i] && (pf->vsi[i]->uplink_seid == veb->seid))
4313                         i40e_vsi_link_event(pf->vsi[i], link_up);
4314 }
4315
4316 /**
4317  * i40e_link_event - Update netif_carrier status
4318  * @pf: board private structure
4319  **/
4320 static void i40e_link_event(struct i40e_pf *pf)
4321 {
4322         bool new_link, old_link;
4323
4324         new_link = (pf->hw.phy.link_info.link_info & I40E_AQ_LINK_UP);
4325         old_link = (pf->hw.phy.link_info_old.link_info & I40E_AQ_LINK_UP);
4326
4327         if (new_link == old_link)
4328                 return;
4329
4330         if (!test_bit(__I40E_DOWN, &pf->vsi[pf->lan_vsi]->state))
4331                 netdev_info(pf->vsi[pf->lan_vsi]->netdev,
4332                             "NIC Link is %s\n", (new_link ? "Up" : "Down"));
4333
4334         /* Notify the base of the switch tree connected to
4335          * the link.  Floating VEBs are not notified.
4336          */
4337         if (pf->lan_veb != I40E_NO_VEB && pf->veb[pf->lan_veb])
4338                 i40e_veb_link_event(pf->veb[pf->lan_veb], new_link);
4339         else
4340                 i40e_vsi_link_event(pf->vsi[pf->lan_vsi], new_link);
4341
4342         if (pf->vf)
4343                 i40e_vc_notify_link_state(pf);
4344
4345         if (pf->flags & I40E_FLAG_PTP)
4346                 i40e_ptp_set_increment(pf);
4347 }
4348
4349 /**
4350  * i40e_check_hang_subtask - Check for hung queues and dropped interrupts
4351  * @pf: board private structure
4352  *
4353  * Set the per-queue flags to request a check for stuck queues in the irq
4354  * clean functions, then force interrupts to be sure the irq clean is called.
4355  **/
4356 static void i40e_check_hang_subtask(struct i40e_pf *pf)
4357 {
4358         int i, v;
4359
4360         /* If we're down or resetting, just bail */
4361         if (test_bit(__I40E_CONFIG_BUSY, &pf->state))
4362                 return;
4363
4364         /* for each VSI/netdev
4365          *     for each Tx queue
4366          *         set the check flag
4367          *     for each q_vector
4368          *         force an interrupt
4369          */
4370         for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
4371                 struct i40e_vsi *vsi = pf->vsi[v];
4372                 int armed = 0;
4373
4374                 if (!pf->vsi[v] ||
4375                     test_bit(__I40E_DOWN, &vsi->state) ||
4376                     (vsi->netdev && !netif_carrier_ok(vsi->netdev)))
4377                         continue;
4378
4379                 for (i = 0; i < vsi->num_queue_pairs; i++) {
4380                         set_check_for_tx_hang(vsi->tx_rings[i]);
4381                         if (test_bit(__I40E_HANG_CHECK_ARMED,
4382                                      &vsi->tx_rings[i]->state))
4383                                 armed++;
4384                 }
4385
4386                 if (armed) {
4387                         if (!(pf->flags & I40E_FLAG_MSIX_ENABLED)) {
4388                                 wr32(&vsi->back->hw, I40E_PFINT_DYN_CTL0,
4389                                      (I40E_PFINT_DYN_CTL0_INTENA_MASK |
4390                                       I40E_PFINT_DYN_CTL0_SWINT_TRIG_MASK));
4391                         } else {
4392                                 u16 vec = vsi->base_vector - 1;
4393                                 u32 val = (I40E_PFINT_DYN_CTLN_INTENA_MASK |
4394                                            I40E_PFINT_DYN_CTLN_SWINT_TRIG_MASK);
4395                                 for (i = 0; i < vsi->num_q_vectors; i++, vec++)
4396                                         wr32(&vsi->back->hw,
4397                                              I40E_PFINT_DYN_CTLN(vec), val);
4398                         }
4399                         i40e_flush(&vsi->back->hw);
4400                 }
4401         }
4402 }
4403
4404 /**
4405  * i40e_watchdog_subtask - Check and bring link up
4406  * @pf: board private structure
4407  **/
4408 static void i40e_watchdog_subtask(struct i40e_pf *pf)
4409 {
4410         int i;
4411
4412         /* if interface is down do nothing */
4413         if (test_bit(__I40E_DOWN, &pf->state) ||
4414             test_bit(__I40E_CONFIG_BUSY, &pf->state))
4415                 return;
4416
4417         /* Update the stats for active netdevs so the network stack
4418          * can look at updated numbers whenever it cares to
4419          */
4420         for (i = 0; i < pf->hw.func_caps.num_vsis; i++)
4421                 if (pf->vsi[i] && pf->vsi[i]->netdev)
4422                         i40e_update_stats(pf->vsi[i]);
4423
4424         /* Update the stats for the active switching components */
4425         for (i = 0; i < I40E_MAX_VEB; i++)
4426                 if (pf->veb[i])
4427                         i40e_update_veb_stats(pf->veb[i]);
4428
4429         i40e_ptp_rx_hang(pf->vsi[pf->lan_vsi]);
4430 }
4431
4432 /**
4433  * i40e_reset_subtask - Set up for resetting the device and driver
4434  * @pf: board private structure
4435  **/
4436 static void i40e_reset_subtask(struct i40e_pf *pf)
4437 {
4438         u32 reset_flags = 0;
4439
4440         rtnl_lock();
4441         if (test_bit(__I40E_REINIT_REQUESTED, &pf->state)) {
4442                 reset_flags |= (1 << __I40E_REINIT_REQUESTED);
4443                 clear_bit(__I40E_REINIT_REQUESTED, &pf->state);
4444         }
4445         if (test_bit(__I40E_PF_RESET_REQUESTED, &pf->state)) {
4446                 reset_flags |= (1 << __I40E_PF_RESET_REQUESTED);
4447                 clear_bit(__I40E_PF_RESET_REQUESTED, &pf->state);
4448         }
4449         if (test_bit(__I40E_CORE_RESET_REQUESTED, &pf->state)) {
4450                 reset_flags |= (1 << __I40E_CORE_RESET_REQUESTED);
4451                 clear_bit(__I40E_CORE_RESET_REQUESTED, &pf->state);
4452         }
4453         if (test_bit(__I40E_GLOBAL_RESET_REQUESTED, &pf->state)) {
4454                 reset_flags |= (1 << __I40E_GLOBAL_RESET_REQUESTED);
4455                 clear_bit(__I40E_GLOBAL_RESET_REQUESTED, &pf->state);
4456         }
4457
4458         /* If there's a recovery already waiting, it takes
4459          * precedence before starting a new reset sequence.
4460          */
4461         if (test_bit(__I40E_RESET_INTR_RECEIVED, &pf->state)) {
4462                 i40e_handle_reset_warning(pf);
4463                 goto unlock;
4464         }
4465
4466         /* If we're already down or resetting, just bail */
4467         if (reset_flags &&
4468             !test_bit(__I40E_DOWN, &pf->state) &&
4469             !test_bit(__I40E_CONFIG_BUSY, &pf->state))
4470                 i40e_do_reset(pf, reset_flags);
4471
4472 unlock:
4473         rtnl_unlock();
4474 }
4475
4476 /**
4477  * i40e_handle_link_event - Handle link event
4478  * @pf: board private structure
4479  * @e: event info posted on ARQ
4480  **/
4481 static void i40e_handle_link_event(struct i40e_pf *pf,
4482                                    struct i40e_arq_event_info *e)
4483 {
4484         struct i40e_hw *hw = &pf->hw;
4485         struct i40e_aqc_get_link_status *status =
4486                 (struct i40e_aqc_get_link_status *)&e->desc.params.raw;
4487         struct i40e_link_status *hw_link_info = &hw->phy.link_info;
4488
4489         /* save off old link status information */
4490         memcpy(&pf->hw.phy.link_info_old, hw_link_info,
4491                sizeof(pf->hw.phy.link_info_old));
4492
4493         /* update link status */
4494         hw_link_info->phy_type = (enum i40e_aq_phy_type)status->phy_type;
4495         hw_link_info->link_speed = (enum i40e_aq_link_speed)status->link_speed;
4496         hw_link_info->link_info = status->link_info;
4497         hw_link_info->an_info = status->an_info;
4498         hw_link_info->ext_info = status->ext_info;
4499         hw_link_info->lse_enable =
4500                 le16_to_cpu(status->command_flags) &
4501                             I40E_AQ_LSE_ENABLE;
4502
4503         /* process the event */
4504         i40e_link_event(pf);
4505
4506         /* Do a new status request to re-enable LSE reporting
4507          * and load new status information into the hw struct,
4508          * then see if the status changed while processing the
4509          * initial event.
4510          */
4511         i40e_aq_get_link_info(&pf->hw, true, NULL, NULL);
4512         i40e_link_event(pf);
4513 }
4514
4515 /**
4516  * i40e_clean_adminq_subtask - Clean the AdminQ rings
4517  * @pf: board private structure
4518  **/
4519 static void i40e_clean_adminq_subtask(struct i40e_pf *pf)
4520 {
4521         struct i40e_arq_event_info event;
4522         struct i40e_hw *hw = &pf->hw;
4523         u16 pending, i = 0;
4524         i40e_status ret;
4525         u16 opcode;
4526         u32 val;
4527
4528         if (!test_bit(__I40E_ADMINQ_EVENT_PENDING, &pf->state))
4529                 return;
4530
4531         event.msg_size = I40E_MAX_AQ_BUF_SIZE;
4532         event.msg_buf = kzalloc(event.msg_size, GFP_KERNEL);
4533         if (!event.msg_buf)
4534                 return;
4535
4536         do {
4537                 event.msg_size = I40E_MAX_AQ_BUF_SIZE; /* reinit each time */
4538                 ret = i40e_clean_arq_element(hw, &event, &pending);
4539                 if (ret == I40E_ERR_ADMIN_QUEUE_NO_WORK) {
4540                         dev_info(&pf->pdev->dev, "No ARQ event found\n");
4541                         break;
4542                 } else if (ret) {
4543                         dev_info(&pf->pdev->dev, "ARQ event error %d\n", ret);
4544                         break;
4545                 }
4546
4547                 opcode = le16_to_cpu(event.desc.opcode);
4548                 switch (opcode) {
4549
4550                 case i40e_aqc_opc_get_link_status:
4551                         i40e_handle_link_event(pf, &event);
4552                         break;
4553                 case i40e_aqc_opc_send_msg_to_pf:
4554                         ret = i40e_vc_process_vf_msg(pf,
4555                                         le16_to_cpu(event.desc.retval),
4556                                         le32_to_cpu(event.desc.cookie_high),
4557                                         le32_to_cpu(event.desc.cookie_low),
4558                                         event.msg_buf,
4559                                         event.msg_size);
4560                         break;
4561                 case i40e_aqc_opc_lldp_update_mib:
4562                         dev_info(&pf->pdev->dev, "ARQ: Update LLDP MIB event received\n");
4563                         break;
4564                 case i40e_aqc_opc_event_lan_overflow:
4565                         dev_info(&pf->pdev->dev, "ARQ LAN queue overflow event received\n");
4566                         i40e_handle_lan_overflow_event(pf, &event);
4567                         break;
4568                 case i40e_aqc_opc_send_msg_to_peer:
4569                         dev_info(&pf->pdev->dev, "ARQ: Msg from other pf\n");
4570                         break;
4571                 default:
4572                         dev_info(&pf->pdev->dev,
4573                                  "ARQ Error: Unknown event 0x%04x received\n",
4574                                  opcode);
4575                         break;
4576                 }
4577         } while (pending && (i++ < pf->adminq_work_limit));
4578
4579         clear_bit(__I40E_ADMINQ_EVENT_PENDING, &pf->state);
4580         /* re-enable Admin queue interrupt cause */
4581         val = rd32(hw, I40E_PFINT_ICR0_ENA);
4582         val |=  I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
4583         wr32(hw, I40E_PFINT_ICR0_ENA, val);
4584         i40e_flush(hw);
4585
4586         kfree(event.msg_buf);
4587 }
4588
4589 /**
4590  * i40e_reconstitute_veb - rebuild the VEB and anything connected to it
4591  * @veb: pointer to the VEB instance
4592  *
4593  * This is a recursive function that first builds the attached VSIs then
4594  * recurses in to build the next layer of VEB.  We track the connections
4595  * through our own index numbers because the seid's from the HW could
4596  * change across the reset.
4597  **/
4598 static int i40e_reconstitute_veb(struct i40e_veb *veb)
4599 {
4600         struct i40e_vsi *ctl_vsi = NULL;
4601         struct i40e_pf *pf = veb->pf;
4602         int v, veb_idx;
4603         int ret;
4604
4605         /* build VSI that owns this VEB, temporarily attached to base VEB */
4606         for (v = 0; v < pf->hw.func_caps.num_vsis && !ctl_vsi; v++) {
4607                 if (pf->vsi[v] &&
4608                     pf->vsi[v]->veb_idx == veb->idx &&
4609                     pf->vsi[v]->flags & I40E_VSI_FLAG_VEB_OWNER) {
4610                         ctl_vsi = pf->vsi[v];
4611                         break;
4612                 }
4613         }
4614         if (!ctl_vsi) {
4615                 dev_info(&pf->pdev->dev,
4616                          "missing owner VSI for veb_idx %d\n", veb->idx);
4617                 ret = -ENOENT;
4618                 goto end_reconstitute;
4619         }
4620         if (ctl_vsi != pf->vsi[pf->lan_vsi])
4621                 ctl_vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid;
4622         ret = i40e_add_vsi(ctl_vsi);
4623         if (ret) {
4624                 dev_info(&pf->pdev->dev,
4625                          "rebuild of owner VSI failed: %d\n", ret);
4626                 goto end_reconstitute;
4627         }
4628         i40e_vsi_reset_stats(ctl_vsi);
4629
4630         /* create the VEB in the switch and move the VSI onto the VEB */
4631         ret = i40e_add_veb(veb, ctl_vsi);
4632         if (ret)
4633                 goto end_reconstitute;
4634
4635         /* create the remaining VSIs attached to this VEB */
4636         for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
4637                 if (!pf->vsi[v] || pf->vsi[v] == ctl_vsi)
4638                         continue;
4639
4640                 if (pf->vsi[v]->veb_idx == veb->idx) {
4641                         struct i40e_vsi *vsi = pf->vsi[v];
4642                         vsi->uplink_seid = veb->seid;
4643                         ret = i40e_add_vsi(vsi);
4644                         if (ret) {
4645                                 dev_info(&pf->pdev->dev,
4646                                          "rebuild of vsi_idx %d failed: %d\n",
4647                                          v, ret);
4648                                 goto end_reconstitute;
4649                         }
4650                         i40e_vsi_reset_stats(vsi);
4651                 }
4652         }
4653
4654         /* create any VEBs attached to this VEB - RECURSION */
4655         for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) {
4656                 if (pf->veb[veb_idx] && pf->veb[veb_idx]->veb_idx == veb->idx) {
4657                         pf->veb[veb_idx]->uplink_seid = veb->seid;
4658                         ret = i40e_reconstitute_veb(pf->veb[veb_idx]);
4659                         if (ret)
4660                                 break;
4661                 }
4662         }
4663
4664 end_reconstitute:
4665         return ret;
4666 }
4667
4668 /**
4669  * i40e_get_capabilities - get info about the HW
4670  * @pf: the PF struct
4671  **/
4672 static int i40e_get_capabilities(struct i40e_pf *pf)
4673 {
4674         struct i40e_aqc_list_capabilities_element_resp *cap_buf;
4675         u16 data_size;
4676         int buf_len;
4677         int err;
4678
4679         buf_len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp);
4680         do {
4681                 cap_buf = kzalloc(buf_len, GFP_KERNEL);
4682                 if (!cap_buf)
4683                         return -ENOMEM;
4684
4685                 /* this loads the data into the hw struct for us */
4686                 err = i40e_aq_discover_capabilities(&pf->hw, cap_buf, buf_len,
4687                                             &data_size,
4688                                             i40e_aqc_opc_list_func_capabilities,
4689                                             NULL);
4690                 /* data loaded, buffer no longer needed */
4691                 kfree(cap_buf);
4692
4693                 if (pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) {
4694                         /* retry with a larger buffer */
4695                         buf_len = data_size;
4696                 } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK) {
4697                         dev_info(&pf->pdev->dev,
4698                                  "capability discovery failed: aq=%d\n",
4699                                  pf->hw.aq.asq_last_status);
4700                         return -ENODEV;
4701                 }
4702         } while (err);
4703
4704         if (pf->hw.revision_id == 0 && (pf->flags & I40E_FLAG_MFP_ENABLED)) {
4705                 pf->hw.func_caps.num_msix_vectors += 1;
4706                 pf->hw.func_caps.num_tx_qp =
4707                         min_t(int, pf->hw.func_caps.num_tx_qp,
4708                               I40E_MAX_NPAR_QPS);
4709         }
4710
4711         if (pf->hw.debug_mask & I40E_DEBUG_USER)
4712                 dev_info(&pf->pdev->dev,
4713                          "pf=%d, num_vfs=%d, msix_pf=%d, msix_vf=%d, fd_g=%d, fd_b=%d, pf_max_q=%d num_vsi=%d\n",
4714                          pf->hw.pf_id, pf->hw.func_caps.num_vfs,
4715                          pf->hw.func_caps.num_msix_vectors,
4716                          pf->hw.func_caps.num_msix_vectors_vf,
4717                          pf->hw.func_caps.fd_filters_guaranteed,
4718                          pf->hw.func_caps.fd_filters_best_effort,
4719                          pf->hw.func_caps.num_tx_qp,
4720                          pf->hw.func_caps.num_vsis);
4721
4722 #define DEF_NUM_VSI (1 + (pf->hw.func_caps.fcoe ? 1 : 0) \
4723                        + pf->hw.func_caps.num_vfs)
4724         if (pf->hw.revision_id == 0 && (DEF_NUM_VSI > pf->hw.func_caps.num_vsis)) {
4725                 dev_info(&pf->pdev->dev,
4726                          "got num_vsis %d, setting num_vsis to %d\n",
4727                          pf->hw.func_caps.num_vsis, DEF_NUM_VSI);
4728                 pf->hw.func_caps.num_vsis = DEF_NUM_VSI;
4729         }
4730
4731         return 0;
4732 }
4733
4734 /**
4735  * i40e_fdir_setup - initialize the Flow Director resources
4736  * @pf: board private structure
4737  **/
4738 static void i40e_fdir_setup(struct i40e_pf *pf)
4739 {
4740         struct i40e_vsi *vsi;
4741         bool new_vsi = false;
4742         int err, i;
4743
4744         if (!(pf->flags & (I40E_FLAG_FDIR_ENABLED |
4745                            I40E_FLAG_FDIR_ATR_ENABLED)))
4746                 return;
4747
4748         pf->atr_sample_rate = I40E_DEFAULT_ATR_SAMPLE_RATE;
4749
4750         /* find existing or make new FDIR VSI */
4751         vsi = NULL;
4752         for (i = 0; i < pf->hw.func_caps.num_vsis; i++)
4753                 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR)
4754                         vsi = pf->vsi[i];
4755         if (!vsi) {
4756                 vsi = i40e_vsi_setup(pf, I40E_VSI_FDIR, pf->mac_seid, 0);
4757                 if (!vsi) {
4758                         dev_info(&pf->pdev->dev, "Couldn't create FDir VSI\n");
4759                         pf->flags &= ~I40E_FLAG_FDIR_ENABLED;
4760                         return;
4761                 }
4762                 new_vsi = true;
4763         }
4764         WARN_ON(vsi->base_queue != I40E_FDIR_RING);
4765         i40e_vsi_setup_irqhandler(vsi, i40e_fdir_clean_rings);
4766
4767         err = i40e_vsi_setup_tx_resources(vsi);
4768         if (!err)
4769                 err = i40e_vsi_setup_rx_resources(vsi);
4770         if (!err)
4771                 err = i40e_vsi_configure(vsi);
4772         if (!err && new_vsi) {
4773                 char int_name[IFNAMSIZ + 9];
4774                 snprintf(int_name, sizeof(int_name) - 1, "%s-fdir",
4775                          dev_driver_string(&pf->pdev->dev));
4776                 err = i40e_vsi_request_irq(vsi, int_name);
4777         }
4778         if (!err)
4779                 err = i40e_up_complete(vsi);
4780
4781         clear_bit(__I40E_NEEDS_RESTART, &vsi->state);
4782 }
4783
4784 /**
4785  * i40e_fdir_teardown - release the Flow Director resources
4786  * @pf: board private structure
4787  **/
4788 static void i40e_fdir_teardown(struct i40e_pf *pf)
4789 {
4790         int i;
4791
4792         for (i = 0; i < pf->hw.func_caps.num_vsis; i++) {
4793                 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR) {
4794                         i40e_vsi_release(pf->vsi[i]);
4795                         break;
4796                 }
4797         }
4798 }
4799
4800 /**
4801  * i40e_prep_for_reset - prep for the core to reset
4802  * @pf: board private structure
4803  *
4804  * Close up the VFs and other things in prep for pf Reset.
4805   **/
4806 static int i40e_prep_for_reset(struct i40e_pf *pf)
4807 {
4808         struct i40e_hw *hw = &pf->hw;
4809         i40e_status ret;
4810         u32 v;
4811
4812         clear_bit(__I40E_RESET_INTR_RECEIVED, &pf->state);
4813         if (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state))
4814                 return 0;
4815
4816         dev_info(&pf->pdev->dev, "Tearing down internal switch for reset\n");
4817
4818         if (i40e_check_asq_alive(hw))
4819                 i40e_vc_notify_reset(pf);
4820
4821         /* quiesce the VSIs and their queues that are not already DOWN */
4822         i40e_pf_quiesce_all_vsi(pf);
4823
4824         for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
4825                 if (pf->vsi[v])
4826                         pf->vsi[v]->seid = 0;
4827         }
4828
4829         i40e_shutdown_adminq(&pf->hw);
4830
4831         /* call shutdown HMC */
4832         ret = i40e_shutdown_lan_hmc(hw);
4833         if (ret) {
4834                 dev_info(&pf->pdev->dev, "shutdown_lan_hmc failed: %d\n", ret);
4835                 clear_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state);
4836         }
4837         return ret;
4838 }
4839
4840 /**
4841  * i40e_reset_and_rebuild - reset and rebuild using a saved config
4842  * @pf: board private structure
4843  * @reinit: if the Main VSI needs to re-initialized.
4844  **/
4845 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit)
4846 {
4847         struct i40e_driver_version dv;
4848         struct i40e_hw *hw = &pf->hw;
4849         i40e_status ret;
4850         u32 v;
4851
4852         /* Now we wait for GRST to settle out.
4853          * We don't have to delete the VEBs or VSIs from the hw switch
4854          * because the reset will make them disappear.
4855          */
4856         ret = i40e_pf_reset(hw);
4857         if (ret)
4858                 dev_info(&pf->pdev->dev, "PF reset failed, %d\n", ret);
4859         pf->pfr_count++;
4860
4861         if (test_bit(__I40E_DOWN, &pf->state))
4862                 goto end_core_reset;
4863         dev_info(&pf->pdev->dev, "Rebuilding internal switch\n");
4864
4865         /* rebuild the basics for the AdminQ, HMC, and initial HW switch */
4866         ret = i40e_init_adminq(&pf->hw);
4867         if (ret) {
4868                 dev_info(&pf->pdev->dev, "Rebuild AdminQ failed, %d\n", ret);
4869                 goto end_core_reset;
4870         }
4871
4872         ret = i40e_get_capabilities(pf);
4873         if (ret) {
4874                 dev_info(&pf->pdev->dev, "i40e_get_capabilities failed, %d\n",
4875                          ret);
4876                 goto end_core_reset;
4877         }
4878
4879         ret = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
4880                                 hw->func_caps.num_rx_qp,
4881                                 pf->fcoe_hmc_cntx_num, pf->fcoe_hmc_filt_num);
4882         if (ret) {
4883                 dev_info(&pf->pdev->dev, "init_lan_hmc failed: %d\n", ret);
4884                 goto end_core_reset;
4885         }
4886         ret = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
4887         if (ret) {
4888                 dev_info(&pf->pdev->dev, "configure_lan_hmc failed: %d\n", ret);
4889                 goto end_core_reset;
4890         }
4891
4892         /* do basic switch setup */
4893         ret = i40e_setup_pf_switch(pf, reinit);
4894         if (ret)
4895                 goto end_core_reset;
4896
4897         /* Rebuild the VSIs and VEBs that existed before reset.
4898          * They are still in our local switch element arrays, so only
4899          * need to rebuild the switch model in the HW.
4900          *
4901          * If there were VEBs but the reconstitution failed, we'll try
4902          * try to recover minimal use by getting the basic PF VSI working.
4903          */
4904         if (pf->vsi[pf->lan_vsi]->uplink_seid != pf->mac_seid) {
4905                 dev_info(&pf->pdev->dev, "attempting to rebuild switch\n");
4906                 /* find the one VEB connected to the MAC, and find orphans */
4907                 for (v = 0; v < I40E_MAX_VEB; v++) {
4908                         if (!pf->veb[v])
4909                                 continue;
4910
4911                         if (pf->veb[v]->uplink_seid == pf->mac_seid ||
4912                             pf->veb[v]->uplink_seid == 0) {
4913                                 ret = i40e_reconstitute_veb(pf->veb[v]);
4914
4915                                 if (!ret)
4916                                         continue;
4917
4918                                 /* If Main VEB failed, we're in deep doodoo,
4919                                  * so give up rebuilding the switch and set up
4920                                  * for minimal rebuild of PF VSI.
4921                                  * If orphan failed, we'll report the error
4922                                  * but try to keep going.
4923                                  */
4924                                 if (pf->veb[v]->uplink_seid == pf->mac_seid) {
4925                                         dev_info(&pf->pdev->dev,
4926                                                  "rebuild of switch failed: %d, will try to set up simple PF connection\n",
4927                                                  ret);
4928                                         pf->vsi[pf->lan_vsi]->uplink_seid
4929                                                                 = pf->mac_seid;
4930                                         break;
4931                                 } else if (pf->veb[v]->uplink_seid == 0) {
4932                                         dev_info(&pf->pdev->dev,
4933                                                  "rebuild of orphan VEB failed: %d\n",
4934                                                  ret);
4935                                 }
4936                         }
4937                 }
4938         }
4939
4940         if (pf->vsi[pf->lan_vsi]->uplink_seid == pf->mac_seid) {
4941                 dev_info(&pf->pdev->dev, "attempting to rebuild PF VSI\n");
4942                 /* no VEB, so rebuild only the Main VSI */
4943                 ret = i40e_add_vsi(pf->vsi[pf->lan_vsi]);
4944                 if (ret) {
4945                         dev_info(&pf->pdev->dev,
4946                                  "rebuild of Main VSI failed: %d\n", ret);
4947                         goto end_core_reset;
4948                 }
4949         }
4950
4951         /* reinit the misc interrupt */
4952         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
4953                 ret = i40e_setup_misc_vector(pf);
4954
4955         /* restart the VSIs that were rebuilt and running before the reset */
4956         i40e_pf_unquiesce_all_vsi(pf);
4957
4958         /* tell the firmware that we're starting */
4959         dv.major_version = DRV_VERSION_MAJOR;
4960         dv.minor_version = DRV_VERSION_MINOR;
4961         dv.build_version = DRV_VERSION_BUILD;
4962         dv.subbuild_version = 0;
4963         i40e_aq_send_driver_version(&pf->hw, &dv, NULL);
4964
4965         dev_info(&pf->pdev->dev, "PF reset done\n");
4966
4967 end_core_reset:
4968         clear_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state);
4969 }
4970
4971 /**
4972  * i40e_handle_reset_warning - prep for the pf to reset, reset and rebuild
4973  * @pf: board private structure
4974  *
4975  * Close up the VFs and other things in prep for a Core Reset,
4976  * then get ready to rebuild the world.
4977  **/
4978 static void i40e_handle_reset_warning(struct i40e_pf *pf)
4979 {
4980         i40e_status ret;
4981
4982         ret = i40e_prep_for_reset(pf);
4983         if (!ret)
4984                 i40e_reset_and_rebuild(pf, false);
4985 }
4986
4987 /**
4988  * i40e_handle_mdd_event
4989  * @pf: pointer to the pf structure
4990  *
4991  * Called from the MDD irq handler to identify possibly malicious vfs
4992  **/
4993 static void i40e_handle_mdd_event(struct i40e_pf *pf)
4994 {
4995         struct i40e_hw *hw = &pf->hw;
4996         bool mdd_detected = false;
4997         struct i40e_vf *vf;
4998         u32 reg;
4999         int i;
5000
5001         if (!test_bit(__I40E_MDD_EVENT_PENDING, &pf->state))
5002                 return;
5003
5004         /* find what triggered the MDD event */
5005         reg = rd32(hw, I40E_GL_MDET_TX);
5006         if (reg & I40E_GL_MDET_TX_VALID_MASK) {
5007                 u8 func = (reg & I40E_GL_MDET_TX_FUNCTION_MASK)
5008                                 >> I40E_GL_MDET_TX_FUNCTION_SHIFT;
5009                 u8 event = (reg & I40E_GL_MDET_TX_EVENT_SHIFT)
5010                                 >> I40E_GL_MDET_TX_EVENT_SHIFT;
5011                 u8 queue = (reg & I40E_GL_MDET_TX_QUEUE_MASK)
5012                                 >> I40E_GL_MDET_TX_QUEUE_SHIFT;
5013                 dev_info(&pf->pdev->dev,
5014                          "Malicious Driver Detection TX event 0x%02x on q %d of function 0x%02x\n",
5015                          event, queue, func);
5016                 wr32(hw, I40E_GL_MDET_TX, 0xffffffff);
5017                 mdd_detected = true;
5018         }
5019         reg = rd32(hw, I40E_GL_MDET_RX);
5020         if (reg & I40E_GL_MDET_RX_VALID_MASK) {
5021                 u8 func = (reg & I40E_GL_MDET_RX_FUNCTION_MASK)
5022                                 >> I40E_GL_MDET_RX_FUNCTION_SHIFT;
5023                 u8 event = (reg & I40E_GL_MDET_RX_EVENT_SHIFT)
5024                                 >> I40E_GL_MDET_RX_EVENT_SHIFT;
5025                 u8 queue = (reg & I40E_GL_MDET_RX_QUEUE_MASK)
5026                                 >> I40E_GL_MDET_RX_QUEUE_SHIFT;
5027                 dev_info(&pf->pdev->dev,
5028                          "Malicious Driver Detection RX event 0x%02x on q %d of function 0x%02x\n",
5029                          event, queue, func);
5030                 wr32(hw, I40E_GL_MDET_RX, 0xffffffff);
5031                 mdd_detected = true;
5032         }
5033
5034         /* see if one of the VFs needs its hand slapped */
5035         for (i = 0; i < pf->num_alloc_vfs && mdd_detected; i++) {
5036                 vf = &(pf->vf[i]);
5037                 reg = rd32(hw, I40E_VP_MDET_TX(i));
5038                 if (reg & I40E_VP_MDET_TX_VALID_MASK) {
5039                         wr32(hw, I40E_VP_MDET_TX(i), 0xFFFF);
5040                         vf->num_mdd_events++;
5041                         dev_info(&pf->pdev->dev, "MDD TX event on VF %d\n", i);
5042                 }
5043
5044                 reg = rd32(hw, I40E_VP_MDET_RX(i));
5045                 if (reg & I40E_VP_MDET_RX_VALID_MASK) {
5046                         wr32(hw, I40E_VP_MDET_RX(i), 0xFFFF);
5047                         vf->num_mdd_events++;
5048                         dev_info(&pf->pdev->dev, "MDD RX event on VF %d\n", i);
5049                 }
5050
5051                 if (vf->num_mdd_events > I40E_DEFAULT_NUM_MDD_EVENTS_ALLOWED) {
5052                         dev_info(&pf->pdev->dev,
5053                                  "Too many MDD events on VF %d, disabled\n", i);
5054                         dev_info(&pf->pdev->dev,
5055                                  "Use PF Control I/F to re-enable the VF\n");
5056                         set_bit(I40E_VF_STAT_DISABLED, &vf->vf_states);
5057                 }
5058         }
5059
5060         /* re-enable mdd interrupt cause */
5061         clear_bit(__I40E_MDD_EVENT_PENDING, &pf->state);
5062         reg = rd32(hw, I40E_PFINT_ICR0_ENA);
5063         reg |=  I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
5064         wr32(hw, I40E_PFINT_ICR0_ENA, reg);
5065         i40e_flush(hw);
5066 }
5067
5068 #ifdef CONFIG_I40E_VXLAN
5069 /**
5070  * i40e_sync_vxlan_filters_subtask - Sync the VSI filter list with HW
5071  * @pf: board private structure
5072  **/
5073 static void i40e_sync_vxlan_filters_subtask(struct i40e_pf *pf)
5074 {
5075         const int vxlan_hdr_qwords = 4;
5076         struct i40e_hw *hw = &pf->hw;
5077         i40e_status ret;
5078         u8 filter_index;
5079         __be16 port;
5080         int i;
5081
5082         if (!(pf->flags & I40E_FLAG_VXLAN_FILTER_SYNC))
5083                 return;
5084
5085         pf->flags &= ~I40E_FLAG_VXLAN_FILTER_SYNC;
5086
5087         for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) {
5088                 if (pf->pending_vxlan_bitmap & (1 << i)) {
5089                         pf->pending_vxlan_bitmap &= ~(1 << i);
5090                         port = pf->vxlan_ports[i];
5091                         ret = port ?
5092                               i40e_aq_add_udp_tunnel(hw, ntohs(port),
5093                                                      vxlan_hdr_qwords,
5094                                                      I40E_AQC_TUNNEL_TYPE_VXLAN,
5095                                                      &filter_index, NULL)
5096                               : i40e_aq_del_udp_tunnel(hw, i, NULL);
5097
5098                         if (ret) {
5099                                 dev_info(&pf->pdev->dev, "Failed to execute AQ command for %s port %d with index %d\n",
5100                                          port ? "adding" : "deleting",
5101                                          ntohs(port), port ? i : i);
5102
5103                                 pf->vxlan_ports[i] = 0;
5104                         } else {
5105                                 dev_info(&pf->pdev->dev, "%s port %d with AQ command with index %d\n",
5106                                          port ? "Added" : "Deleted",
5107                                          ntohs(port), port ? i : filter_index);
5108                         }
5109                 }
5110         }
5111 }
5112
5113 #endif
5114 /**
5115  * i40e_service_task - Run the driver's async subtasks
5116  * @work: pointer to work_struct containing our data
5117  **/
5118 static void i40e_service_task(struct work_struct *work)
5119 {
5120         struct i40e_pf *pf = container_of(work,
5121                                           struct i40e_pf,
5122                                           service_task);
5123         unsigned long start_time = jiffies;
5124
5125         i40e_reset_subtask(pf);
5126         i40e_handle_mdd_event(pf);
5127         i40e_vc_process_vflr_event(pf);
5128         i40e_watchdog_subtask(pf);
5129         i40e_fdir_reinit_subtask(pf);
5130         i40e_check_hang_subtask(pf);
5131         i40e_sync_filters_subtask(pf);
5132 #ifdef CONFIG_I40E_VXLAN
5133         i40e_sync_vxlan_filters_subtask(pf);
5134 #endif
5135         i40e_clean_adminq_subtask(pf);
5136
5137         i40e_service_event_complete(pf);
5138
5139         /* If the tasks have taken longer than one timer cycle or there
5140          * is more work to be done, reschedule the service task now
5141          * rather than wait for the timer to tick again.
5142          */
5143         if (time_after(jiffies, (start_time + pf->service_timer_period)) ||
5144             test_bit(__I40E_ADMINQ_EVENT_PENDING, &pf->state)            ||
5145             test_bit(__I40E_MDD_EVENT_PENDING, &pf->state)               ||
5146             test_bit(__I40E_VFLR_EVENT_PENDING, &pf->state))
5147                 i40e_service_event_schedule(pf);
5148 }
5149
5150 /**
5151  * i40e_service_timer - timer callback
5152  * @data: pointer to PF struct
5153  **/
5154 static void i40e_service_timer(unsigned long data)
5155 {
5156         struct i40e_pf *pf = (struct i40e_pf *)data;
5157
5158         mod_timer(&pf->service_timer,
5159                   round_jiffies(jiffies + pf->service_timer_period));
5160         i40e_service_event_schedule(pf);
5161 }
5162
5163 /**
5164  * i40e_set_num_rings_in_vsi - Determine number of rings in the VSI
5165  * @vsi: the VSI being configured
5166  **/
5167 static int i40e_set_num_rings_in_vsi(struct i40e_vsi *vsi)
5168 {
5169         struct i40e_pf *pf = vsi->back;
5170
5171         switch (vsi->type) {
5172         case I40E_VSI_MAIN:
5173                 vsi->alloc_queue_pairs = pf->num_lan_qps;
5174                 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
5175                                       I40E_REQ_DESCRIPTOR_MULTIPLE);
5176                 if (pf->flags & I40E_FLAG_MSIX_ENABLED)
5177                         vsi->num_q_vectors = pf->num_lan_msix;
5178                 else
5179                         vsi->num_q_vectors = 1;
5180
5181                 break;
5182
5183         case I40E_VSI_FDIR:
5184                 vsi->alloc_queue_pairs = 1;
5185                 vsi->num_desc = ALIGN(I40E_FDIR_RING_COUNT,
5186                                       I40E_REQ_DESCRIPTOR_MULTIPLE);
5187                 vsi->num_q_vectors = 1;
5188                 break;
5189
5190         case I40E_VSI_VMDQ2:
5191                 vsi->alloc_queue_pairs = pf->num_vmdq_qps;
5192                 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
5193                                       I40E_REQ_DESCRIPTOR_MULTIPLE);
5194                 vsi->num_q_vectors = pf->num_vmdq_msix;
5195                 break;
5196
5197         case I40E_VSI_SRIOV:
5198                 vsi->alloc_queue_pairs = pf->num_vf_qps;
5199                 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
5200                                       I40E_REQ_DESCRIPTOR_MULTIPLE);
5201                 break;
5202
5203         default:
5204                 WARN_ON(1);
5205                 return -ENODATA;
5206         }
5207
5208         return 0;
5209 }
5210
5211 /**
5212  * i40e_vsi_alloc_arrays - Allocate queue and vector pointer arrays for the vsi
5213  * @type: VSI pointer
5214  * @alloc_qvectors: a bool to specify if q_vectors need to be allocated.
5215  *
5216  * On error: returns error code (negative)
5217  * On success: returns 0
5218  **/
5219 static int i40e_vsi_alloc_arrays(struct i40e_vsi *vsi, bool alloc_qvectors)
5220 {
5221         int size;
5222         int ret = 0;
5223
5224         /* allocate memory for both Tx and Rx ring pointers */
5225         size = sizeof(struct i40e_ring *) * vsi->alloc_queue_pairs * 2;
5226         vsi->tx_rings = kzalloc(size, GFP_KERNEL);
5227         if (!vsi->tx_rings)
5228                 return -ENOMEM;
5229         vsi->rx_rings = &vsi->tx_rings[vsi->alloc_queue_pairs];
5230
5231         if (alloc_qvectors) {
5232                 /* allocate memory for q_vector pointers */
5233                 size = sizeof(struct i40e_q_vectors *) * vsi->num_q_vectors;
5234                 vsi->q_vectors = kzalloc(size, GFP_KERNEL);
5235                 if (!vsi->q_vectors) {
5236                         ret = -ENOMEM;
5237                         goto err_vectors;
5238                 }
5239         }
5240         return ret;
5241
5242 err_vectors:
5243         kfree(vsi->tx_rings);
5244         return ret;
5245 }
5246
5247 /**
5248  * i40e_vsi_mem_alloc - Allocates the next available struct vsi in the PF
5249  * @pf: board private structure
5250  * @type: type of VSI
5251  *
5252  * On error: returns error code (negative)
5253  * On success: returns vsi index in PF (positive)
5254  **/
5255 static int i40e_vsi_mem_alloc(struct i40e_pf *pf, enum i40e_vsi_type type)
5256 {
5257         int ret = -ENODEV;
5258         struct i40e_vsi *vsi;
5259         int vsi_idx;
5260         int i;
5261
5262         /* Need to protect the allocation of the VSIs at the PF level */
5263         mutex_lock(&pf->switch_mutex);
5264
5265         /* VSI list may be fragmented if VSI creation/destruction has
5266          * been happening.  We can afford to do a quick scan to look
5267          * for any free VSIs in the list.
5268          *
5269          * find next empty vsi slot, looping back around if necessary
5270          */
5271         i = pf->next_vsi;
5272         while (i < pf->hw.func_caps.num_vsis && pf->vsi[i])
5273                 i++;
5274         if (i >= pf->hw.func_caps.num_vsis) {
5275                 i = 0;
5276                 while (i < pf->next_vsi && pf->vsi[i])
5277                         i++;
5278         }
5279
5280         if (i < pf->hw.func_caps.num_vsis && !pf->vsi[i]) {
5281                 vsi_idx = i;             /* Found one! */
5282         } else {
5283                 ret = -ENODEV;
5284                 goto unlock_pf;  /* out of VSI slots! */
5285         }
5286         pf->next_vsi = ++i;
5287
5288         vsi = kzalloc(sizeof(*vsi), GFP_KERNEL);
5289         if (!vsi) {
5290                 ret = -ENOMEM;
5291                 goto unlock_pf;
5292         }
5293         vsi->type = type;
5294         vsi->back = pf;
5295         set_bit(__I40E_DOWN, &vsi->state);
5296         vsi->flags = 0;
5297         vsi->idx = vsi_idx;
5298         vsi->rx_itr_setting = pf->rx_itr_default;
5299         vsi->tx_itr_setting = pf->tx_itr_default;
5300         vsi->netdev_registered = false;
5301         vsi->work_limit = I40E_DEFAULT_IRQ_WORK;
5302         INIT_LIST_HEAD(&vsi->mac_filter_list);
5303
5304         ret = i40e_set_num_rings_in_vsi(vsi);
5305         if (ret)
5306                 goto err_rings;
5307
5308         ret = i40e_vsi_alloc_arrays(vsi, true);
5309         if (ret)
5310                 goto err_rings;
5311
5312         /* Setup default MSIX irq handler for VSI */
5313         i40e_vsi_setup_irqhandler(vsi, i40e_msix_clean_rings);
5314
5315         pf->vsi[vsi_idx] = vsi;
5316         ret = vsi_idx;
5317         goto unlock_pf;
5318
5319 err_rings:
5320         pf->next_vsi = i - 1;
5321         kfree(vsi);
5322 unlock_pf:
5323         mutex_unlock(&pf->switch_mutex);
5324         return ret;
5325 }
5326
5327 /**
5328  * i40e_vsi_free_arrays - Free queue and vector pointer arrays for the VSI
5329  * @type: VSI pointer
5330  * @free_qvectors: a bool to specify if q_vectors need to be freed.
5331  *
5332  * On error: returns error code (negative)
5333  * On success: returns 0
5334  **/
5335 static void i40e_vsi_free_arrays(struct i40e_vsi *vsi, bool free_qvectors)
5336 {
5337         /* free the ring and vector containers */
5338         if (free_qvectors) {
5339                 kfree(vsi->q_vectors);
5340                 vsi->q_vectors = NULL;
5341         }
5342         kfree(vsi->tx_rings);
5343         vsi->tx_rings = NULL;
5344         vsi->rx_rings = NULL;
5345 }
5346
5347 /**
5348  * i40e_vsi_clear - Deallocate the VSI provided
5349  * @vsi: the VSI being un-configured
5350  **/
5351 static int i40e_vsi_clear(struct i40e_vsi *vsi)
5352 {
5353         struct i40e_pf *pf;
5354
5355         if (!vsi)
5356                 return 0;
5357
5358         if (!vsi->back)
5359                 goto free_vsi;
5360         pf = vsi->back;
5361
5362         mutex_lock(&pf->switch_mutex);
5363         if (!pf->vsi[vsi->idx]) {
5364                 dev_err(&pf->pdev->dev, "pf->vsi[%d] is NULL, just free vsi[%d](%p,type %d)\n",
5365                         vsi->idx, vsi->idx, vsi, vsi->type);
5366                 goto unlock_vsi;
5367         }
5368
5369         if (pf->vsi[vsi->idx] != vsi) {
5370                 dev_err(&pf->pdev->dev,
5371                         "pf->vsi[%d](%p, type %d) != vsi[%d](%p,type %d): no free!\n",
5372                         pf->vsi[vsi->idx]->idx,
5373                         pf->vsi[vsi->idx],
5374                         pf->vsi[vsi->idx]->type,
5375                         vsi->idx, vsi, vsi->type);
5376                 goto unlock_vsi;
5377         }
5378
5379         /* updates the pf for this cleared vsi */
5380         i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);
5381         i40e_put_lump(pf->irq_pile, vsi->base_vector, vsi->idx);
5382
5383         i40e_vsi_free_arrays(vsi, true);
5384
5385         pf->vsi[vsi->idx] = NULL;
5386         if (vsi->idx < pf->next_vsi)
5387                 pf->next_vsi = vsi->idx;
5388
5389 unlock_vsi:
5390         mutex_unlock(&pf->switch_mutex);
5391 free_vsi:
5392         kfree(vsi);
5393
5394         return 0;
5395 }
5396
5397 /**
5398  * i40e_vsi_clear_rings - Deallocates the Rx and Tx rings for the provided VSI
5399  * @vsi: the VSI being cleaned
5400  **/
5401 static void i40e_vsi_clear_rings(struct i40e_vsi *vsi)
5402 {
5403         int i;
5404
5405         if (vsi->tx_rings && vsi->tx_rings[0]) {
5406                 for (i = 0; i < vsi->alloc_queue_pairs; i++) {
5407                         kfree_rcu(vsi->tx_rings[i], rcu);
5408                         vsi->tx_rings[i] = NULL;
5409                         vsi->rx_rings[i] = NULL;
5410                 }
5411         }
5412 }
5413
5414 /**
5415  * i40e_alloc_rings - Allocates the Rx and Tx rings for the provided VSI
5416  * @vsi: the VSI being configured
5417  **/
5418 static int i40e_alloc_rings(struct i40e_vsi *vsi)
5419 {
5420         struct i40e_pf *pf = vsi->back;
5421         int i;
5422
5423         /* Set basic values in the rings to be used later during open() */
5424         for (i = 0; i < vsi->alloc_queue_pairs; i++) {
5425                 struct i40e_ring *tx_ring;
5426                 struct i40e_ring *rx_ring;
5427
5428                 /* allocate space for both Tx and Rx in one shot */
5429                 tx_ring = kzalloc(sizeof(struct i40e_ring) * 2, GFP_KERNEL);
5430                 if (!tx_ring)
5431                         goto err_out;
5432
5433                 tx_ring->queue_index = i;
5434                 tx_ring->reg_idx = vsi->base_queue + i;
5435                 tx_ring->ring_active = false;
5436                 tx_ring->vsi = vsi;
5437                 tx_ring->netdev = vsi->netdev;
5438                 tx_ring->dev = &pf->pdev->dev;
5439                 tx_ring->count = vsi->num_desc;
5440                 tx_ring->size = 0;
5441                 tx_ring->dcb_tc = 0;
5442                 vsi->tx_rings[i] = tx_ring;
5443
5444                 rx_ring = &tx_ring[1];
5445                 rx_ring->queue_index = i;
5446                 rx_ring->reg_idx = vsi->base_queue + i;
5447                 rx_ring->ring_active = false;
5448                 rx_ring->vsi = vsi;
5449                 rx_ring->netdev = vsi->netdev;
5450                 rx_ring->dev = &pf->pdev->dev;
5451                 rx_ring->count = vsi->num_desc;
5452                 rx_ring->size = 0;
5453                 rx_ring->dcb_tc = 0;
5454                 if (pf->flags & I40E_FLAG_16BYTE_RX_DESC_ENABLED)
5455                         set_ring_16byte_desc_enabled(rx_ring);
5456                 else
5457                         clear_ring_16byte_desc_enabled(rx_ring);
5458                 vsi->rx_rings[i] = rx_ring;
5459         }
5460
5461         return 0;
5462
5463 err_out:
5464         i40e_vsi_clear_rings(vsi);
5465         return -ENOMEM;
5466 }
5467
5468 /**
5469  * i40e_reserve_msix_vectors - Reserve MSI-X vectors in the kernel
5470  * @pf: board private structure
5471  * @vectors: the number of MSI-X vectors to request
5472  *
5473  * Returns the number of vectors reserved, or error
5474  **/
5475 static int i40e_reserve_msix_vectors(struct i40e_pf *pf, int vectors)
5476 {
5477         int err = 0;
5478
5479         pf->num_msix_entries = 0;
5480         while (vectors >= I40E_MIN_MSIX) {
5481                 err = pci_enable_msix(pf->pdev, pf->msix_entries, vectors);
5482                 if (err == 0) {
5483                         /* good to go */
5484                         pf->num_msix_entries = vectors;
5485                         break;
5486                 } else if (err < 0) {
5487                         /* total failure */
5488                         dev_info(&pf->pdev->dev,
5489                                  "MSI-X vector reservation failed: %d\n", err);
5490                         vectors = 0;
5491                         break;
5492                 } else {
5493                         /* err > 0 is the hint for retry */
5494                         dev_info(&pf->pdev->dev,
5495                                  "MSI-X vectors wanted %d, retrying with %d\n",
5496                                  vectors, err);
5497                         vectors = err;
5498                 }
5499         }
5500
5501         if (vectors > 0 && vectors < I40E_MIN_MSIX) {
5502                 dev_info(&pf->pdev->dev,
5503                          "Couldn't get enough vectors, only %d available\n",
5504                          vectors);
5505                 vectors = 0;
5506         }
5507
5508         return vectors;
5509 }
5510
5511 /**
5512  * i40e_init_msix - Setup the MSIX capability
5513  * @pf: board private structure
5514  *
5515  * Work with the OS to set up the MSIX vectors needed.
5516  *
5517  * Returns 0 on success, negative on failure
5518  **/
5519 static int i40e_init_msix(struct i40e_pf *pf)
5520 {
5521         i40e_status err = 0;
5522         struct i40e_hw *hw = &pf->hw;
5523         int v_budget, i;
5524         int vec;
5525
5526         if (!(pf->flags & I40E_FLAG_MSIX_ENABLED))
5527                 return -ENODEV;
5528
5529         /* The number of vectors we'll request will be comprised of:
5530          *   - Add 1 for "other" cause for Admin Queue events, etc.
5531          *   - The number of LAN queue pairs
5532          *      - Queues being used for RSS.
5533          *              We don't need as many as max_rss_size vectors.
5534          *              use rss_size instead in the calculation since that
5535          *              is governed by number of cpus in the system.
5536          *      - assumes symmetric Tx/Rx pairing
5537          *   - The number of VMDq pairs
5538          * Once we count this up, try the request.
5539          *
5540          * If we can't get what we want, we'll simplify to nearly nothing
5541          * and try again.  If that still fails, we punt.
5542          */
5543         pf->num_lan_msix = pf->num_lan_qps - (pf->rss_size_max - pf->rss_size);
5544         pf->num_vmdq_msix = pf->num_vmdq_qps;
5545         v_budget = 1 + pf->num_lan_msix;
5546         v_budget += (pf->num_vmdq_vsis * pf->num_vmdq_msix);
5547         if (pf->flags & I40E_FLAG_FDIR_ENABLED)
5548                 v_budget++;
5549
5550         /* Scale down if necessary, and the rings will share vectors */
5551         v_budget = min_t(int, v_budget, hw->func_caps.num_msix_vectors);
5552
5553         pf->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry),
5554                                    GFP_KERNEL);
5555         if (!pf->msix_entries)
5556                 return -ENOMEM;
5557
5558         for (i = 0; i < v_budget; i++)
5559                 pf->msix_entries[i].entry = i;
5560         vec = i40e_reserve_msix_vectors(pf, v_budget);
5561         if (vec < I40E_MIN_MSIX) {
5562                 pf->flags &= ~I40E_FLAG_MSIX_ENABLED;
5563                 kfree(pf->msix_entries);
5564                 pf->msix_entries = NULL;
5565                 return -ENODEV;
5566
5567         } else if (vec == I40E_MIN_MSIX) {
5568                 /* Adjust for minimal MSIX use */
5569                 dev_info(&pf->pdev->dev, "Features disabled, not enough MSIX vectors\n");
5570                 pf->flags &= ~I40E_FLAG_VMDQ_ENABLED;
5571                 pf->num_vmdq_vsis = 0;
5572                 pf->num_vmdq_qps = 0;
5573                 pf->num_vmdq_msix = 0;
5574                 pf->num_lan_qps = 1;
5575                 pf->num_lan_msix = 1;
5576
5577         } else if (vec != v_budget) {
5578                 /* Scale vector usage down */
5579                 pf->num_vmdq_msix = 1;    /* force VMDqs to only one vector */
5580                 vec--;                    /* reserve the misc vector */
5581
5582                 /* partition out the remaining vectors */
5583                 switch (vec) {
5584                 case 2:
5585                         pf->num_vmdq_vsis = 1;
5586                         pf->num_lan_msix = 1;
5587                         break;
5588                 case 3:
5589                         pf->num_vmdq_vsis = 1;
5590                         pf->num_lan_msix = 2;
5591                         break;
5592                 default:
5593                         pf->num_lan_msix = min_t(int, (vec / 2),
5594                                                  pf->num_lan_qps);
5595                         pf->num_vmdq_vsis = min_t(int, (vec - pf->num_lan_msix),
5596                                                   I40E_DEFAULT_NUM_VMDQ_VSI);
5597                         break;
5598                 }
5599         }
5600
5601         return err;
5602 }
5603
5604 /**
5605  * i40e_alloc_q_vector - Allocate memory for a single interrupt vector
5606  * @vsi: the VSI being configured
5607  * @v_idx: index of the vector in the vsi struct
5608  *
5609  * We allocate one q_vector.  If allocation fails we return -ENOMEM.
5610  **/
5611 static int i40e_alloc_q_vector(struct i40e_vsi *vsi, int v_idx)
5612 {
5613         struct i40e_q_vector *q_vector;
5614
5615         /* allocate q_vector */
5616         q_vector = kzalloc(sizeof(struct i40e_q_vector), GFP_KERNEL);
5617         if (!q_vector)
5618                 return -ENOMEM;
5619
5620         q_vector->vsi = vsi;
5621         q_vector->v_idx = v_idx;
5622         cpumask_set_cpu(v_idx, &q_vector->affinity_mask);
5623         if (vsi->netdev)
5624                 netif_napi_add(vsi->netdev, &q_vector->napi,
5625                                i40e_napi_poll, vsi->work_limit);
5626
5627         q_vector->rx.latency_range = I40E_LOW_LATENCY;
5628         q_vector->tx.latency_range = I40E_LOW_LATENCY;
5629
5630         /* tie q_vector and vsi together */
5631         vsi->q_vectors[v_idx] = q_vector;
5632
5633         return 0;
5634 }
5635
5636 /**
5637  * i40e_alloc_q_vectors - Allocate memory for interrupt vectors
5638  * @vsi: the VSI being configured
5639  *
5640  * We allocate one q_vector per queue interrupt.  If allocation fails we
5641  * return -ENOMEM.
5642  **/
5643 static int i40e_alloc_q_vectors(struct i40e_vsi *vsi)
5644 {
5645         struct i40e_pf *pf = vsi->back;
5646         int v_idx, num_q_vectors;
5647         int err;
5648
5649         /* if not MSIX, give the one vector only to the LAN VSI */
5650         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
5651                 num_q_vectors = vsi->num_q_vectors;
5652         else if (vsi == pf->vsi[pf->lan_vsi])
5653                 num_q_vectors = 1;
5654         else
5655                 return -EINVAL;
5656
5657         for (v_idx = 0; v_idx < num_q_vectors; v_idx++) {
5658                 err = i40e_alloc_q_vector(vsi, v_idx);
5659                 if (err)
5660                         goto err_out;
5661         }
5662
5663         return 0;
5664
5665 err_out:
5666         while (v_idx--)
5667                 i40e_free_q_vector(vsi, v_idx);
5668
5669         return err;
5670 }
5671
5672 /**
5673  * i40e_init_interrupt_scheme - Determine proper interrupt scheme
5674  * @pf: board private structure to initialize
5675  **/
5676 static void i40e_init_interrupt_scheme(struct i40e_pf *pf)
5677 {
5678         int err = 0;
5679
5680         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
5681                 err = i40e_init_msix(pf);
5682                 if (err) {
5683                         pf->flags &= ~(I40E_FLAG_MSIX_ENABLED      |
5684                                         I40E_FLAG_RSS_ENABLED      |
5685                                         I40E_FLAG_DCB_ENABLED      |
5686                                         I40E_FLAG_SRIOV_ENABLED    |
5687                                         I40E_FLAG_FDIR_ENABLED     |
5688                                         I40E_FLAG_FDIR_ATR_ENABLED |
5689                                         I40E_FLAG_VMDQ_ENABLED);
5690
5691                         /* rework the queue expectations without MSIX */
5692                         i40e_determine_queue_usage(pf);
5693                 }
5694         }
5695
5696         if (!(pf->flags & I40E_FLAG_MSIX_ENABLED) &&
5697             (pf->flags & I40E_FLAG_MSI_ENABLED)) {
5698                 dev_info(&pf->pdev->dev, "MSIX not available, trying MSI\n");
5699                 err = pci_enable_msi(pf->pdev);
5700                 if (err) {
5701                         dev_info(&pf->pdev->dev, "MSI init failed - %d\n", err);
5702                         pf->flags &= ~I40E_FLAG_MSI_ENABLED;
5703                 }
5704         }
5705
5706         if (!(pf->flags & (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED)))
5707                 dev_info(&pf->pdev->dev, "MSIX and MSI not available, falling back to Legacy IRQ\n");
5708
5709         /* track first vector for misc interrupts */
5710         err = i40e_get_lump(pf, pf->irq_pile, 1, I40E_PILE_VALID_BIT-1);
5711 }
5712
5713 /**
5714  * i40e_setup_misc_vector - Setup the misc vector to handle non queue events
5715  * @pf: board private structure
5716  *
5717  * This sets up the handler for MSIX 0, which is used to manage the
5718  * non-queue interrupts, e.g. AdminQ and errors.  This is not used
5719  * when in MSI or Legacy interrupt mode.
5720  **/
5721 static int i40e_setup_misc_vector(struct i40e_pf *pf)
5722 {
5723         struct i40e_hw *hw = &pf->hw;
5724         int err = 0;
5725
5726         /* Only request the irq if this is the first time through, and
5727          * not when we're rebuilding after a Reset
5728          */
5729         if (!test_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state)) {
5730                 err = request_irq(pf->msix_entries[0].vector,
5731                                   i40e_intr, 0, pf->misc_int_name, pf);
5732                 if (err) {
5733                         dev_info(&pf->pdev->dev,
5734                                  "request_irq for msix_misc failed: %d\n", err);
5735                         return -EFAULT;
5736                 }
5737         }
5738
5739         i40e_enable_misc_int_causes(hw);
5740
5741         /* associate no queues to the misc vector */
5742         wr32(hw, I40E_PFINT_LNKLST0, I40E_QUEUE_END_OF_LIST);
5743         wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), I40E_ITR_8K);
5744
5745         i40e_flush(hw);
5746
5747         i40e_irq_dynamic_enable_icr0(pf);
5748
5749         return err;
5750 }
5751
5752 /**
5753  * i40e_config_rss - Prepare for RSS if used
5754  * @pf: board private structure
5755  **/
5756 static int i40e_config_rss(struct i40e_pf *pf)
5757 {
5758         /* Set of random keys generated using kernel random number generator */
5759         static const u32 seed[I40E_PFQF_HKEY_MAX_INDEX + 1] = {0x41b01687,
5760                                 0x183cfd8c, 0xce880440, 0x580cbc3c, 0x35897377,
5761                                 0x328b25e1, 0x4fa98922, 0xb7d90c14, 0xd5bad70d,
5762                                 0xcd15a2c1, 0xe8580225, 0x4a1e9d11, 0xfe5731be};
5763         struct i40e_hw *hw = &pf->hw;
5764         u32 lut = 0;
5765         int i, j;
5766         u64 hena;
5767
5768         /* Fill out hash function seed */
5769         for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
5770                 wr32(hw, I40E_PFQF_HKEY(i), seed[i]);
5771
5772         /* By default we enable TCP/UDP with IPv4/IPv6 ptypes */
5773         hena = (u64)rd32(hw, I40E_PFQF_HENA(0)) |
5774                 ((u64)rd32(hw, I40E_PFQF_HENA(1)) << 32);
5775         hena |= I40E_DEFAULT_RSS_HENA;
5776         wr32(hw, I40E_PFQF_HENA(0), (u32)hena);
5777         wr32(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32));
5778
5779         /* Populate the LUT with max no. of queues in round robin fashion */
5780         for (i = 0, j = 0; i < pf->hw.func_caps.rss_table_size; i++, j++) {
5781
5782                 /* The assumption is that lan qp count will be the highest
5783                  * qp count for any PF VSI that needs RSS.
5784                  * If multiple VSIs need RSS support, all the qp counts
5785                  * for those VSIs should be a power of 2 for RSS to work.
5786                  * If LAN VSI is the only consumer for RSS then this requirement
5787                  * is not necessary.
5788                  */
5789                 if (j == pf->rss_size)
5790                         j = 0;
5791                 /* lut = 4-byte sliding window of 4 lut entries */
5792                 lut = (lut << 8) | (j &
5793                          ((0x1 << pf->hw.func_caps.rss_table_entry_width) - 1));
5794                 /* On i = 3, we have 4 entries in lut; write to the register */
5795                 if ((i & 3) == 3)
5796                         wr32(hw, I40E_PFQF_HLUT(i >> 2), lut);
5797         }
5798         i40e_flush(hw);
5799
5800         return 0;
5801 }
5802
5803 /**
5804  * i40e_reconfig_rss_queues - change number of queues for rss and rebuild
5805  * @pf: board private structure
5806  * @queue_count: the requested queue count for rss.
5807  *
5808  * returns 0 if rss is not enabled, if enabled returns the final rss queue
5809  * count which may be different from the requested queue count.
5810  **/
5811 int i40e_reconfig_rss_queues(struct i40e_pf *pf, int queue_count)
5812 {
5813         if (!(pf->flags & I40E_FLAG_RSS_ENABLED))
5814                 return 0;
5815
5816         queue_count = min_t(int, queue_count, pf->rss_size_max);
5817         queue_count = rounddown_pow_of_two(queue_count);
5818
5819         if (queue_count != pf->rss_size) {
5820                 i40e_prep_for_reset(pf);
5821
5822                 pf->rss_size = queue_count;
5823
5824                 i40e_reset_and_rebuild(pf, true);
5825                 i40e_config_rss(pf);
5826         }
5827         dev_info(&pf->pdev->dev, "RSS count:  %d\n", pf->rss_size);
5828         return pf->rss_size;
5829 }
5830
5831 /**
5832  * i40e_sw_init - Initialize general software structures (struct i40e_pf)
5833  * @pf: board private structure to initialize
5834  *
5835  * i40e_sw_init initializes the Adapter private data structure.
5836  * Fields are initialized based on PCI device information and
5837  * OS network device settings (MTU size).
5838  **/
5839 static int i40e_sw_init(struct i40e_pf *pf)
5840 {
5841         int err = 0;
5842         int size;
5843
5844         pf->msg_enable = netif_msg_init(I40E_DEFAULT_MSG_ENABLE,
5845                                 (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK));
5846         pf->hw.debug_mask = pf->msg_enable | I40E_DEBUG_DIAG;
5847         if (debug != -1 && debug != I40E_DEFAULT_MSG_ENABLE) {
5848                 if (I40E_DEBUG_USER & debug)
5849                         pf->hw.debug_mask = debug;
5850                 pf->msg_enable = netif_msg_init((debug & ~I40E_DEBUG_USER),
5851                                                 I40E_DEFAULT_MSG_ENABLE);
5852         }
5853
5854         /* Set default capability flags */
5855         pf->flags = I40E_FLAG_RX_CSUM_ENABLED |
5856                     I40E_FLAG_MSI_ENABLED     |
5857                     I40E_FLAG_MSIX_ENABLED    |
5858                     I40E_FLAG_RX_1BUF_ENABLED;
5859
5860         /* Depending on PF configurations, it is possible that the RSS
5861          * maximum might end up larger than the available queues
5862          */
5863         pf->rss_size_max = 0x1 << pf->hw.func_caps.rss_table_entry_width;
5864         pf->rss_size_max = min_t(int, pf->rss_size_max,
5865                                  pf->hw.func_caps.num_tx_qp);
5866         if (pf->hw.func_caps.rss) {
5867                 pf->flags |= I40E_FLAG_RSS_ENABLED;
5868                 pf->rss_size = min_t(int, pf->rss_size_max, num_online_cpus());
5869         } else {
5870                 pf->rss_size = 1;
5871         }
5872
5873         /* MFP mode enabled */
5874         if (pf->hw.func_caps.npar_enable || pf->hw.func_caps.mfp_mode_1) {
5875                 pf->flags |= I40E_FLAG_MFP_ENABLED;
5876                 dev_info(&pf->pdev->dev, "MFP mode Enabled\n");
5877         }
5878
5879         if (pf->hw.func_caps.dcb)
5880                 pf->num_tc_qps = I40E_DEFAULT_QUEUES_PER_TC;
5881         else
5882                 pf->num_tc_qps = 0;
5883
5884         if (pf->hw.func_caps.fd) {
5885                 /* FW/NVM is not yet fixed in this regard */
5886                 if ((pf->hw.func_caps.fd_filters_guaranteed > 0) ||
5887                     (pf->hw.func_caps.fd_filters_best_effort > 0)) {
5888                         pf->flags |= I40E_FLAG_FDIR_ATR_ENABLED;
5889                         dev_info(&pf->pdev->dev,
5890                                  "Flow Director ATR mode Enabled\n");
5891                         pf->flags |= I40E_FLAG_FDIR_ENABLED;
5892                         dev_info(&pf->pdev->dev,
5893                                  "Flow Director Side Band mode Enabled\n");
5894                         pf->fdir_pf_filter_count =
5895                                          pf->hw.func_caps.fd_filters_guaranteed;
5896                 }
5897         } else {
5898                 pf->fdir_pf_filter_count = 0;
5899         }
5900
5901         if (pf->hw.func_caps.vmdq) {
5902                 pf->flags |= I40E_FLAG_VMDQ_ENABLED;
5903                 pf->num_vmdq_vsis = I40E_DEFAULT_NUM_VMDQ_VSI;
5904                 pf->num_vmdq_qps = I40E_DEFAULT_QUEUES_PER_VMDQ;
5905         }
5906
5907 #ifdef CONFIG_PCI_IOV
5908         if (pf->hw.func_caps.num_vfs) {
5909                 pf->num_vf_qps = I40E_DEFAULT_QUEUES_PER_VF;
5910                 pf->flags |= I40E_FLAG_SRIOV_ENABLED;
5911                 pf->num_req_vfs = min_t(int,
5912                                         pf->hw.func_caps.num_vfs,
5913                                         I40E_MAX_VF_COUNT);
5914                 dev_info(&pf->pdev->dev,
5915                          "Number of VFs being requested for PF[%d] = %d\n",
5916                          pf->hw.pf_id, pf->num_req_vfs);
5917         }
5918 #endif /* CONFIG_PCI_IOV */
5919         pf->eeprom_version = 0xDEAD;
5920         pf->lan_veb = I40E_NO_VEB;
5921         pf->lan_vsi = I40E_NO_VSI;
5922
5923         /* set up queue assignment tracking */
5924         size = sizeof(struct i40e_lump_tracking)
5925                 + (sizeof(u16) * pf->hw.func_caps.num_tx_qp);
5926         pf->qp_pile = kzalloc(size, GFP_KERNEL);
5927         if (!pf->qp_pile) {
5928                 err = -ENOMEM;
5929                 goto sw_init_done;
5930         }
5931         pf->qp_pile->num_entries = pf->hw.func_caps.num_tx_qp;
5932         pf->qp_pile->search_hint = 0;
5933
5934         /* set up vector assignment tracking */
5935         size = sizeof(struct i40e_lump_tracking)
5936                 + (sizeof(u16) * pf->hw.func_caps.num_msix_vectors);
5937         pf->irq_pile = kzalloc(size, GFP_KERNEL);
5938         if (!pf->irq_pile) {
5939                 kfree(pf->qp_pile);
5940                 err = -ENOMEM;
5941                 goto sw_init_done;
5942         }
5943         pf->irq_pile->num_entries = pf->hw.func_caps.num_msix_vectors;
5944         pf->irq_pile->search_hint = 0;
5945
5946         mutex_init(&pf->switch_mutex);
5947
5948 sw_init_done:
5949         return err;
5950 }
5951
5952 /**
5953  * i40e_set_features - set the netdev feature flags
5954  * @netdev: ptr to the netdev being adjusted
5955  * @features: the feature set that the stack is suggesting
5956  **/
5957 static int i40e_set_features(struct net_device *netdev,
5958                              netdev_features_t features)
5959 {
5960         struct i40e_netdev_priv *np = netdev_priv(netdev);
5961         struct i40e_vsi *vsi = np->vsi;
5962
5963         if (features & NETIF_F_HW_VLAN_CTAG_RX)
5964                 i40e_vlan_stripping_enable(vsi);
5965         else
5966                 i40e_vlan_stripping_disable(vsi);
5967
5968         return 0;
5969 }
5970
5971 #ifdef CONFIG_I40E_VXLAN
5972 /**
5973  * i40e_get_vxlan_port_idx - Lookup a possibly offloaded for Rx UDP port
5974  * @pf: board private structure
5975  * @port: The UDP port to look up
5976  *
5977  * Returns the index number or I40E_MAX_PF_UDP_OFFLOAD_PORTS if port not found
5978  **/
5979 static u8 i40e_get_vxlan_port_idx(struct i40e_pf *pf, __be16 port)
5980 {
5981         u8 i;
5982
5983         for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) {
5984                 if (pf->vxlan_ports[i] == port)
5985                         return i;
5986         }
5987
5988         return i;
5989 }
5990
5991 /**
5992  * i40e_add_vxlan_port - Get notifications about VXLAN ports that come up
5993  * @netdev: This physical port's netdev
5994  * @sa_family: Socket Family that VXLAN is notifying us about
5995  * @port: New UDP port number that VXLAN started listening to
5996  **/
5997 static void i40e_add_vxlan_port(struct net_device *netdev,
5998                                 sa_family_t sa_family, __be16 port)
5999 {
6000         struct i40e_netdev_priv *np = netdev_priv(netdev);
6001         struct i40e_vsi *vsi = np->vsi;
6002         struct i40e_pf *pf = vsi->back;
6003         u8 next_idx;
6004         u8 idx;
6005
6006         if (sa_family == AF_INET6)
6007                 return;
6008
6009         idx = i40e_get_vxlan_port_idx(pf, port);
6010
6011         /* Check if port already exists */
6012         if (idx < I40E_MAX_PF_UDP_OFFLOAD_PORTS) {
6013                 netdev_info(netdev, "Port %d already offloaded\n", ntohs(port));
6014                 return;
6015         }
6016
6017         /* Now check if there is space to add the new port */
6018         next_idx = i40e_get_vxlan_port_idx(pf, 0);
6019
6020         if (next_idx == I40E_MAX_PF_UDP_OFFLOAD_PORTS) {
6021                 netdev_info(netdev, "Maximum number of UDP ports reached, not adding port %d\n",
6022                             ntohs(port));
6023                 return;
6024         }
6025
6026         /* New port: add it and mark its index in the bitmap */
6027         pf->vxlan_ports[next_idx] = port;
6028         pf->pending_vxlan_bitmap |= (1 << next_idx);
6029
6030         pf->flags |= I40E_FLAG_VXLAN_FILTER_SYNC;
6031 }
6032
6033 /**
6034  * i40e_del_vxlan_port - Get notifications about VXLAN ports that go away
6035  * @netdev: This physical port's netdev
6036  * @sa_family: Socket Family that VXLAN is notifying us about
6037  * @port: UDP port number that VXLAN stopped listening to
6038  **/
6039 static void i40e_del_vxlan_port(struct net_device *netdev,
6040                                 sa_family_t sa_family, __be16 port)
6041 {
6042         struct i40e_netdev_priv *np = netdev_priv(netdev);
6043         struct i40e_vsi *vsi = np->vsi;
6044         struct i40e_pf *pf = vsi->back;
6045         u8 idx;
6046
6047         if (sa_family == AF_INET6)
6048                 return;
6049
6050         idx = i40e_get_vxlan_port_idx(pf, port);
6051
6052         /* Check if port already exists */
6053         if (idx < I40E_MAX_PF_UDP_OFFLOAD_PORTS) {
6054                 /* if port exists, set it to 0 (mark for deletion)
6055                  * and make it pending
6056                  */
6057                 pf->vxlan_ports[idx] = 0;
6058
6059                 pf->pending_vxlan_bitmap |= (1 << idx);
6060
6061                 pf->flags |= I40E_FLAG_VXLAN_FILTER_SYNC;
6062         } else {
6063                 netdev_warn(netdev, "Port %d was not found, not deleting\n",
6064                             ntohs(port));
6065         }
6066 }
6067
6068 #endif
6069 static const struct net_device_ops i40e_netdev_ops = {
6070         .ndo_open               = i40e_open,
6071         .ndo_stop               = i40e_close,
6072         .ndo_start_xmit         = i40e_lan_xmit_frame,
6073         .ndo_get_stats64        = i40e_get_netdev_stats_struct,
6074         .ndo_set_rx_mode        = i40e_set_rx_mode,
6075         .ndo_validate_addr      = eth_validate_addr,
6076         .ndo_set_mac_address    = i40e_set_mac,
6077         .ndo_change_mtu         = i40e_change_mtu,
6078         .ndo_do_ioctl           = i40e_ioctl,
6079         .ndo_tx_timeout         = i40e_tx_timeout,
6080         .ndo_vlan_rx_add_vid    = i40e_vlan_rx_add_vid,
6081         .ndo_vlan_rx_kill_vid   = i40e_vlan_rx_kill_vid,
6082 #ifdef CONFIG_NET_POLL_CONTROLLER
6083         .ndo_poll_controller    = i40e_netpoll,
6084 #endif
6085         .ndo_setup_tc           = i40e_setup_tc,
6086         .ndo_set_features       = i40e_set_features,
6087         .ndo_set_vf_mac         = i40e_ndo_set_vf_mac,
6088         .ndo_set_vf_vlan        = i40e_ndo_set_vf_port_vlan,
6089         .ndo_set_vf_tx_rate     = i40e_ndo_set_vf_bw,
6090         .ndo_get_vf_config      = i40e_ndo_get_vf_config,
6091 #ifdef CONFIG_I40E_VXLAN
6092         .ndo_add_vxlan_port     = i40e_add_vxlan_port,
6093         .ndo_del_vxlan_port     = i40e_del_vxlan_port,
6094 #endif
6095 };
6096
6097 /**
6098  * i40e_config_netdev - Setup the netdev flags
6099  * @vsi: the VSI being configured
6100  *
6101  * Returns 0 on success, negative value on failure
6102  **/
6103 static int i40e_config_netdev(struct i40e_vsi *vsi)
6104 {
6105         u8 brdcast[ETH_ALEN] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
6106         struct i40e_pf *pf = vsi->back;
6107         struct i40e_hw *hw = &pf->hw;
6108         struct i40e_netdev_priv *np;
6109         struct net_device *netdev;
6110         u8 mac_addr[ETH_ALEN];
6111         int etherdev_size;
6112
6113         etherdev_size = sizeof(struct i40e_netdev_priv);
6114         netdev = alloc_etherdev_mq(etherdev_size, vsi->alloc_queue_pairs);
6115         if (!netdev)
6116                 return -ENOMEM;
6117
6118         vsi->netdev = netdev;
6119         np = netdev_priv(netdev);
6120         np->vsi = vsi;
6121
6122         netdev->hw_enc_features = NETIF_F_IP_CSUM        |
6123                                   NETIF_F_GSO_UDP_TUNNEL |
6124                                   NETIF_F_TSO            |
6125                                   NETIF_F_SG;
6126
6127         netdev->features = NETIF_F_SG                  |
6128                            NETIF_F_IP_CSUM             |
6129                            NETIF_F_SCTP_CSUM           |
6130                            NETIF_F_HIGHDMA             |
6131                            NETIF_F_GSO_UDP_TUNNEL      |
6132                            NETIF_F_HW_VLAN_CTAG_TX     |
6133                            NETIF_F_HW_VLAN_CTAG_RX     |
6134                            NETIF_F_HW_VLAN_CTAG_FILTER |
6135                            NETIF_F_IPV6_CSUM           |
6136                            NETIF_F_TSO                 |
6137                            NETIF_F_TSO6                |
6138                            NETIF_F_RXCSUM              |
6139                            NETIF_F_RXHASH              |
6140                            0;
6141
6142         /* copy netdev features into list of user selectable features */
6143         netdev->hw_features |= netdev->features;
6144
6145         if (vsi->type == I40E_VSI_MAIN) {
6146                 SET_NETDEV_DEV(netdev, &pf->pdev->dev);
6147                 memcpy(mac_addr, hw->mac.perm_addr, ETH_ALEN);
6148         } else {
6149                 /* relate the VSI_VMDQ name to the VSI_MAIN name */
6150                 snprintf(netdev->name, IFNAMSIZ, "%sv%%d",
6151                          pf->vsi[pf->lan_vsi]->netdev->name);
6152                 random_ether_addr(mac_addr);
6153                 i40e_add_filter(vsi, mac_addr, I40E_VLAN_ANY, false, false);
6154         }
6155         i40e_add_filter(vsi, brdcast, I40E_VLAN_ANY, false, false);
6156
6157         memcpy(netdev->dev_addr, mac_addr, ETH_ALEN);
6158         memcpy(netdev->perm_addr, mac_addr, ETH_ALEN);
6159         /* vlan gets same features (except vlan offload)
6160          * after any tweaks for specific VSI types
6161          */
6162         netdev->vlan_features = netdev->features & ~(NETIF_F_HW_VLAN_CTAG_TX |
6163                                                      NETIF_F_HW_VLAN_CTAG_RX |
6164                                                    NETIF_F_HW_VLAN_CTAG_FILTER);
6165         netdev->priv_flags |= IFF_UNICAST_FLT;
6166         netdev->priv_flags |= IFF_SUPP_NOFCS;
6167         /* Setup netdev TC information */
6168         i40e_vsi_config_netdev_tc(vsi, vsi->tc_config.enabled_tc);
6169
6170         netdev->netdev_ops = &i40e_netdev_ops;
6171         netdev->watchdog_timeo = 5 * HZ;
6172         i40e_set_ethtool_ops(netdev);
6173
6174         return 0;
6175 }
6176
6177 /**
6178  * i40e_vsi_delete - Delete a VSI from the switch
6179  * @vsi: the VSI being removed
6180  *
6181  * Returns 0 on success, negative value on failure
6182  **/
6183 static void i40e_vsi_delete(struct i40e_vsi *vsi)
6184 {
6185         /* remove default VSI is not allowed */
6186         if (vsi == vsi->back->vsi[vsi->back->lan_vsi])
6187                 return;
6188
6189         /* there is no HW VSI for FDIR */
6190         if (vsi->type == I40E_VSI_FDIR)
6191                 return;
6192
6193         i40e_aq_delete_element(&vsi->back->hw, vsi->seid, NULL);
6194         return;
6195 }
6196
6197 /**
6198  * i40e_add_vsi - Add a VSI to the switch
6199  * @vsi: the VSI being configured
6200  *
6201  * This initializes a VSI context depending on the VSI type to be added and
6202  * passes it down to the add_vsi aq command.
6203  **/
6204 static int i40e_add_vsi(struct i40e_vsi *vsi)
6205 {
6206         int ret = -ENODEV;
6207         struct i40e_mac_filter *f, *ftmp;
6208         struct i40e_pf *pf = vsi->back;
6209         struct i40e_hw *hw = &pf->hw;
6210         struct i40e_vsi_context ctxt;
6211         u8 enabled_tc = 0x1; /* TC0 enabled */
6212         int f_count = 0;
6213
6214         memset(&ctxt, 0, sizeof(ctxt));
6215         switch (vsi->type) {
6216         case I40E_VSI_MAIN:
6217                 /* The PF's main VSI is already setup as part of the
6218                  * device initialization, so we'll not bother with
6219                  * the add_vsi call, but we will retrieve the current
6220                  * VSI context.
6221                  */
6222                 ctxt.seid = pf->main_vsi_seid;
6223                 ctxt.pf_num = pf->hw.pf_id;
6224                 ctxt.vf_num = 0;
6225                 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
6226                 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
6227                 if (ret) {
6228                         dev_info(&pf->pdev->dev,
6229                                  "couldn't get pf vsi config, err %d, aq_err %d\n",
6230                                  ret, pf->hw.aq.asq_last_status);
6231                         return -ENOENT;
6232                 }
6233                 memcpy(&vsi->info, &ctxt.info, sizeof(ctxt.info));
6234                 vsi->info.valid_sections = 0;
6235
6236                 vsi->seid = ctxt.seid;
6237                 vsi->id = ctxt.vsi_number;
6238
6239                 enabled_tc = i40e_pf_get_tc_map(pf);
6240
6241                 /* MFP mode setup queue map and update VSI */
6242                 if (pf->flags & I40E_FLAG_MFP_ENABLED) {
6243                         memset(&ctxt, 0, sizeof(ctxt));
6244                         ctxt.seid = pf->main_vsi_seid;
6245                         ctxt.pf_num = pf->hw.pf_id;
6246                         ctxt.vf_num = 0;
6247                         i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);
6248                         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
6249                         if (ret) {
6250                                 dev_info(&pf->pdev->dev,
6251                                          "update vsi failed, aq_err=%d\n",
6252                                          pf->hw.aq.asq_last_status);
6253                                 ret = -ENOENT;
6254                                 goto err;
6255                         }
6256                         /* update the local VSI info queue map */
6257                         i40e_vsi_update_queue_map(vsi, &ctxt);
6258                         vsi->info.valid_sections = 0;
6259                 } else {
6260                         /* Default/Main VSI is only enabled for TC0
6261                          * reconfigure it to enable all TCs that are
6262                          * available on the port in SFP mode.
6263                          */
6264                         ret = i40e_vsi_config_tc(vsi, enabled_tc);
6265                         if (ret) {
6266                                 dev_info(&pf->pdev->dev,
6267                                          "failed to configure TCs for main VSI tc_map 0x%08x, err %d, aq_err %d\n",
6268                                          enabled_tc, ret,
6269                                          pf->hw.aq.asq_last_status);
6270                                 ret = -ENOENT;
6271                         }
6272                 }
6273                 break;
6274
6275         case I40E_VSI_FDIR:
6276                 /* no queue mapping or actual HW VSI needed */
6277                 vsi->info.valid_sections = 0;
6278                 vsi->seid = 0;
6279                 vsi->id = 0;
6280                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
6281                 return 0;
6282                 break;
6283
6284         case I40E_VSI_VMDQ2:
6285                 ctxt.pf_num = hw->pf_id;
6286                 ctxt.vf_num = 0;
6287                 ctxt.uplink_seid = vsi->uplink_seid;
6288                 ctxt.connection_type = 0x1;     /* regular data port */
6289                 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2;
6290
6291                 ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
6292
6293                 /* This VSI is connected to VEB so the switch_id
6294                  * should be set to zero by default.
6295                  */
6296                 ctxt.info.switch_id = 0;
6297                 ctxt.info.switch_id |= cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_LOCAL_LB);
6298                 ctxt.info.switch_id |= cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
6299
6300                 /* Setup the VSI tx/rx queue map for TC0 only for now */
6301                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
6302                 break;
6303
6304         case I40E_VSI_SRIOV:
6305                 ctxt.pf_num = hw->pf_id;
6306                 ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id;
6307                 ctxt.uplink_seid = vsi->uplink_seid;
6308                 ctxt.connection_type = 0x1;     /* regular data port */
6309                 ctxt.flags = I40E_AQ_VSI_TYPE_VF;
6310
6311                 ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
6312
6313                 /* This VSI is connected to VEB so the switch_id
6314                  * should be set to zero by default.
6315                  */
6316                 ctxt.info.switch_id = cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
6317
6318                 ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
6319                 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL;
6320                 /* Setup the VSI tx/rx queue map for TC0 only for now */
6321                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
6322                 break;
6323
6324         default:
6325                 return -ENODEV;
6326         }
6327
6328         if (vsi->type != I40E_VSI_MAIN) {
6329                 ret = i40e_aq_add_vsi(hw, &ctxt, NULL);
6330                 if (ret) {
6331                         dev_info(&vsi->back->pdev->dev,
6332                                  "add vsi failed, aq_err=%d\n",
6333                                  vsi->back->hw.aq.asq_last_status);
6334                         ret = -ENOENT;
6335                         goto err;
6336                 }
6337                 memcpy(&vsi->info, &ctxt.info, sizeof(ctxt.info));
6338                 vsi->info.valid_sections = 0;
6339                 vsi->seid = ctxt.seid;
6340                 vsi->id = ctxt.vsi_number;
6341         }
6342
6343         /* If macvlan filters already exist, force them to get loaded */
6344         list_for_each_entry_safe(f, ftmp, &vsi->mac_filter_list, list) {
6345                 f->changed = true;
6346                 f_count++;
6347         }
6348         if (f_count) {
6349                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
6350                 pf->flags |= I40E_FLAG_FILTER_SYNC;
6351         }
6352
6353         /* Update VSI BW information */
6354         ret = i40e_vsi_get_bw_info(vsi);
6355         if (ret) {
6356                 dev_info(&pf->pdev->dev,
6357                          "couldn't get vsi bw info, err %d, aq_err %d\n",
6358                          ret, pf->hw.aq.asq_last_status);
6359                 /* VSI is already added so not tearing that up */
6360                 ret = 0;
6361         }
6362
6363 err:
6364         return ret;
6365 }
6366
6367 /**
6368  * i40e_vsi_release - Delete a VSI and free its resources
6369  * @vsi: the VSI being removed
6370  *
6371  * Returns 0 on success or < 0 on error
6372  **/
6373 int i40e_vsi_release(struct i40e_vsi *vsi)
6374 {
6375         struct i40e_mac_filter *f, *ftmp;
6376         struct i40e_veb *veb = NULL;
6377         struct i40e_pf *pf;
6378         u16 uplink_seid;
6379         int i, n;
6380
6381         pf = vsi->back;
6382
6383         /* release of a VEB-owner or last VSI is not allowed */
6384         if (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) {
6385                 dev_info(&pf->pdev->dev, "VSI %d has existing VEB %d\n",
6386                          vsi->seid, vsi->uplink_seid);
6387                 return -ENODEV;
6388         }
6389         if (vsi == pf->vsi[pf->lan_vsi] &&
6390             !test_bit(__I40E_DOWN, &pf->state)) {
6391                 dev_info(&pf->pdev->dev, "Can't remove PF VSI\n");
6392                 return -ENODEV;
6393         }
6394
6395         uplink_seid = vsi->uplink_seid;
6396         if (vsi->type != I40E_VSI_SRIOV) {
6397                 if (vsi->netdev_registered) {
6398                         vsi->netdev_registered = false;
6399                         if (vsi->netdev) {
6400                                 /* results in a call to i40e_close() */
6401                                 unregister_netdev(vsi->netdev);
6402                                 free_netdev(vsi->netdev);
6403                                 vsi->netdev = NULL;
6404                         }
6405                 } else {
6406                         if (!test_and_set_bit(__I40E_DOWN, &vsi->state))
6407                                 i40e_down(vsi);
6408                         i40e_vsi_free_irq(vsi);
6409                         i40e_vsi_free_tx_resources(vsi);
6410                         i40e_vsi_free_rx_resources(vsi);
6411                 }
6412                 i40e_vsi_disable_irq(vsi);
6413         }
6414
6415         list_for_each_entry_safe(f, ftmp, &vsi->mac_filter_list, list)
6416                 i40e_del_filter(vsi, f->macaddr, f->vlan,
6417                                 f->is_vf, f->is_netdev);
6418         i40e_sync_vsi_filters(vsi);
6419
6420         i40e_vsi_delete(vsi);
6421         i40e_vsi_free_q_vectors(vsi);
6422         i40e_vsi_clear_rings(vsi);
6423         i40e_vsi_clear(vsi);
6424
6425         /* If this was the last thing on the VEB, except for the
6426          * controlling VSI, remove the VEB, which puts the controlling
6427          * VSI onto the next level down in the switch.
6428          *
6429          * Well, okay, there's one more exception here: don't remove
6430          * the orphan VEBs yet.  We'll wait for an explicit remove request
6431          * from up the network stack.
6432          */
6433         for (n = 0, i = 0; i < pf->hw.func_caps.num_vsis; i++) {
6434                 if (pf->vsi[i] &&
6435                     pf->vsi[i]->uplink_seid == uplink_seid &&
6436                     (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) {
6437                         n++;      /* count the VSIs */
6438                 }
6439         }
6440         for (i = 0; i < I40E_MAX_VEB; i++) {
6441                 if (!pf->veb[i])
6442                         continue;
6443                 if (pf->veb[i]->uplink_seid == uplink_seid)
6444                         n++;     /* count the VEBs */
6445                 if (pf->veb[i]->seid == uplink_seid)
6446                         veb = pf->veb[i];
6447         }
6448         if (n == 0 && veb && veb->uplink_seid != 0)
6449                 i40e_veb_release(veb);
6450
6451         return 0;
6452 }
6453
6454 /**
6455  * i40e_vsi_setup_vectors - Set up the q_vectors for the given VSI
6456  * @vsi: ptr to the VSI
6457  *
6458  * This should only be called after i40e_vsi_mem_alloc() which allocates the
6459  * corresponding SW VSI structure and initializes num_queue_pairs for the
6460  * newly allocated VSI.
6461  *
6462  * Returns 0 on success or negative on failure
6463  **/
6464 static int i40e_vsi_setup_vectors(struct i40e_vsi *vsi)
6465 {
6466         int ret = -ENOENT;
6467         struct i40e_pf *pf = vsi->back;
6468
6469         if (vsi->q_vectors[0]) {
6470                 dev_info(&pf->pdev->dev, "VSI %d has existing q_vectors\n",
6471                          vsi->seid);
6472                 return -EEXIST;
6473         }
6474
6475         if (vsi->base_vector) {
6476                 dev_info(&pf->pdev->dev,
6477                          "VSI %d has non-zero base vector %d\n",
6478                          vsi->seid, vsi->base_vector);
6479                 return -EEXIST;
6480         }
6481
6482         ret = i40e_alloc_q_vectors(vsi);
6483         if (ret) {
6484                 dev_info(&pf->pdev->dev,
6485                          "failed to allocate %d q_vector for VSI %d, ret=%d\n",
6486                          vsi->num_q_vectors, vsi->seid, ret);
6487                 vsi->num_q_vectors = 0;
6488                 goto vector_setup_out;
6489         }
6490
6491         if (vsi->num_q_vectors)
6492                 vsi->base_vector = i40e_get_lump(pf, pf->irq_pile,
6493                                                  vsi->num_q_vectors, vsi->idx);
6494         if (vsi->base_vector < 0) {
6495                 dev_info(&pf->pdev->dev,
6496                          "failed to get q tracking for VSI %d, err=%d\n",
6497                          vsi->seid, vsi->base_vector);
6498                 i40e_vsi_free_q_vectors(vsi);
6499                 ret = -ENOENT;
6500                 goto vector_setup_out;
6501         }
6502
6503 vector_setup_out:
6504         return ret;
6505 }
6506
6507 /**
6508  * i40e_vsi_reinit_setup - return and reallocate resources for a VSI
6509  * @vsi: pointer to the vsi.
6510  *
6511  * This re-allocates a vsi's queue resources.
6512  *
6513  * Returns pointer to the successfully allocated and configured VSI sw struct
6514  * on success, otherwise returns NULL on failure.
6515  **/
6516 static struct i40e_vsi *i40e_vsi_reinit_setup(struct i40e_vsi *vsi)
6517 {
6518         struct i40e_pf *pf = vsi->back;
6519         u8 enabled_tc;
6520         int ret;
6521
6522         i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);
6523         i40e_vsi_clear_rings(vsi);
6524
6525         i40e_vsi_free_arrays(vsi, false);
6526         i40e_set_num_rings_in_vsi(vsi);
6527         ret = i40e_vsi_alloc_arrays(vsi, false);
6528         if (ret)
6529                 goto err_vsi;
6530
6531         ret = i40e_get_lump(pf, pf->qp_pile, vsi->alloc_queue_pairs, vsi->idx);
6532         if (ret < 0) {
6533                 dev_info(&pf->pdev->dev, "VSI %d get_lump failed %d\n",
6534                          vsi->seid, ret);
6535                 goto err_vsi;
6536         }
6537         vsi->base_queue = ret;
6538
6539         /* Update the FW view of the VSI. Force a reset of TC and queue
6540          * layout configurations.
6541          */
6542         enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc;
6543         pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0;
6544         pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid;
6545         i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc);
6546
6547         /* assign it some queues */
6548         ret = i40e_alloc_rings(vsi);
6549         if (ret)
6550                 goto err_rings;
6551
6552         /* map all of the rings to the q_vectors */
6553         i40e_vsi_map_rings_to_vectors(vsi);
6554         return vsi;
6555
6556 err_rings:
6557         i40e_vsi_free_q_vectors(vsi);
6558         if (vsi->netdev_registered) {
6559                 vsi->netdev_registered = false;
6560                 unregister_netdev(vsi->netdev);
6561                 free_netdev(vsi->netdev);
6562                 vsi->netdev = NULL;
6563         }
6564         i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);
6565 err_vsi:
6566         i40e_vsi_clear(vsi);
6567         return NULL;
6568 }
6569
6570 /**
6571  * i40e_vsi_setup - Set up a VSI by a given type
6572  * @pf: board private structure
6573  * @type: VSI type
6574  * @uplink_seid: the switch element to link to
6575  * @param1: usage depends upon VSI type. For VF types, indicates VF id
6576  *
6577  * This allocates the sw VSI structure and its queue resources, then add a VSI
6578  * to the identified VEB.
6579  *
6580  * Returns pointer to the successfully allocated and configure VSI sw struct on
6581  * success, otherwise returns NULL on failure.
6582  **/
6583 struct i40e_vsi *i40e_vsi_setup(struct i40e_pf *pf, u8 type,
6584                                 u16 uplink_seid, u32 param1)
6585 {
6586         struct i40e_vsi *vsi = NULL;
6587         struct i40e_veb *veb = NULL;
6588         int ret, i;
6589         int v_idx;
6590
6591         /* The requested uplink_seid must be either
6592          *     - the PF's port seid
6593          *              no VEB is needed because this is the PF
6594          *              or this is a Flow Director special case VSI
6595          *     - seid of an existing VEB
6596          *     - seid of a VSI that owns an existing VEB
6597          *     - seid of a VSI that doesn't own a VEB
6598          *              a new VEB is created and the VSI becomes the owner
6599          *     - seid of the PF VSI, which is what creates the first VEB
6600          *              this is a special case of the previous
6601          *
6602          * Find which uplink_seid we were given and create a new VEB if needed
6603          */
6604         for (i = 0; i < I40E_MAX_VEB; i++) {
6605                 if (pf->veb[i] && pf->veb[i]->seid == uplink_seid) {
6606                         veb = pf->veb[i];
6607                         break;
6608                 }
6609         }
6610
6611         if (!veb && uplink_seid != pf->mac_seid) {
6612
6613                 for (i = 0; i < pf->hw.func_caps.num_vsis; i++) {
6614                         if (pf->vsi[i] && pf->vsi[i]->seid == uplink_seid) {
6615                                 vsi = pf->vsi[i];
6616                                 break;
6617                         }
6618                 }
6619                 if (!vsi) {
6620                         dev_info(&pf->pdev->dev, "no such uplink_seid %d\n",
6621                                  uplink_seid);
6622                         return NULL;
6623                 }
6624
6625                 if (vsi->uplink_seid == pf->mac_seid)
6626                         veb = i40e_veb_setup(pf, 0, pf->mac_seid, vsi->seid,
6627                                              vsi->tc_config.enabled_tc);
6628                 else if ((vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0)
6629                         veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid,
6630                                              vsi->tc_config.enabled_tc);
6631
6632                 for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
6633                         if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
6634                                 veb = pf->veb[i];
6635                 }
6636                 if (!veb) {
6637                         dev_info(&pf->pdev->dev, "couldn't add VEB\n");
6638                         return NULL;
6639                 }
6640
6641                 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;
6642                 uplink_seid = veb->seid;
6643         }
6644
6645         /* get vsi sw struct */
6646         v_idx = i40e_vsi_mem_alloc(pf, type);
6647         if (v_idx < 0)
6648                 goto err_alloc;
6649         vsi = pf->vsi[v_idx];
6650         vsi->type = type;
6651         vsi->veb_idx = (veb ? veb->idx : I40E_NO_VEB);
6652
6653         if (type == I40E_VSI_MAIN)
6654                 pf->lan_vsi = v_idx;
6655         else if (type == I40E_VSI_SRIOV)
6656                 vsi->vf_id = param1;
6657         /* assign it some queues */
6658         ret = i40e_get_lump(pf, pf->qp_pile, vsi->alloc_queue_pairs, vsi->idx);
6659         if (ret < 0) {
6660                 dev_info(&pf->pdev->dev, "VSI %d get_lump failed %d\n",
6661                          vsi->seid, ret);
6662                 goto err_vsi;
6663         }
6664         vsi->base_queue = ret;
6665
6666         /* get a VSI from the hardware */
6667         vsi->uplink_seid = uplink_seid;
6668         ret = i40e_add_vsi(vsi);
6669         if (ret)
6670                 goto err_vsi;
6671
6672         switch (vsi->type) {
6673         /* setup the netdev if needed */
6674         case I40E_VSI_MAIN:
6675         case I40E_VSI_VMDQ2:
6676                 ret = i40e_config_netdev(vsi);
6677                 if (ret)
6678                         goto err_netdev;
6679                 ret = register_netdev(vsi->netdev);
6680                 if (ret)
6681                         goto err_netdev;
6682                 vsi->netdev_registered = true;
6683                 netif_carrier_off(vsi->netdev);
6684                 /* fall through */
6685
6686         case I40E_VSI_FDIR:
6687                 /* set up vectors and rings if needed */
6688                 ret = i40e_vsi_setup_vectors(vsi);
6689                 if (ret)
6690                         goto err_msix;
6691
6692                 ret = i40e_alloc_rings(vsi);
6693                 if (ret)
6694                         goto err_rings;
6695
6696                 /* map all of the rings to the q_vectors */
6697                 i40e_vsi_map_rings_to_vectors(vsi);
6698
6699                 i40e_vsi_reset_stats(vsi);
6700                 break;
6701
6702         default:
6703                 /* no netdev or rings for the other VSI types */
6704                 break;
6705         }
6706
6707         return vsi;
6708
6709 err_rings:
6710         i40e_vsi_free_q_vectors(vsi);
6711 err_msix:
6712         if (vsi->netdev_registered) {
6713                 vsi->netdev_registered = false;
6714                 unregister_netdev(vsi->netdev);
6715                 free_netdev(vsi->netdev);
6716                 vsi->netdev = NULL;
6717         }
6718 err_netdev:
6719         i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);
6720 err_vsi:
6721         i40e_vsi_clear(vsi);
6722 err_alloc:
6723         return NULL;
6724 }
6725
6726 /**
6727  * i40e_veb_get_bw_info - Query VEB BW information
6728  * @veb: the veb to query
6729  *
6730  * Query the Tx scheduler BW configuration data for given VEB
6731  **/
6732 static int i40e_veb_get_bw_info(struct i40e_veb *veb)
6733 {
6734         struct i40e_aqc_query_switching_comp_ets_config_resp ets_data;
6735         struct i40e_aqc_query_switching_comp_bw_config_resp bw_data;
6736         struct i40e_pf *pf = veb->pf;
6737         struct i40e_hw *hw = &pf->hw;
6738         u32 tc_bw_max;
6739         int ret = 0;
6740         int i;
6741
6742         ret = i40e_aq_query_switch_comp_bw_config(hw, veb->seid,
6743                                                   &bw_data, NULL);
6744         if (ret) {
6745                 dev_info(&pf->pdev->dev,
6746                          "query veb bw config failed, aq_err=%d\n",
6747                          hw->aq.asq_last_status);
6748                 goto out;
6749         }
6750
6751         ret = i40e_aq_query_switch_comp_ets_config(hw, veb->seid,
6752                                                    &ets_data, NULL);
6753         if (ret) {
6754                 dev_info(&pf->pdev->dev,
6755                          "query veb bw ets config failed, aq_err=%d\n",
6756                          hw->aq.asq_last_status);
6757                 goto out;
6758         }
6759
6760         veb->bw_limit = le16_to_cpu(ets_data.port_bw_limit);
6761         veb->bw_max_quanta = ets_data.tc_bw_max;
6762         veb->is_abs_credits = bw_data.absolute_credits_enable;
6763         tc_bw_max = le16_to_cpu(bw_data.tc_bw_max[0]) |
6764                     (le16_to_cpu(bw_data.tc_bw_max[1]) << 16);
6765         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
6766                 veb->bw_tc_share_credits[i] = bw_data.tc_bw_share_credits[i];
6767                 veb->bw_tc_limit_credits[i] =
6768                                         le16_to_cpu(bw_data.tc_bw_limits[i]);
6769                 veb->bw_tc_max_quanta[i] = ((tc_bw_max >> (i*4)) & 0x7);
6770         }
6771
6772 out:
6773         return ret;
6774 }
6775
6776 /**
6777  * i40e_veb_mem_alloc - Allocates the next available struct veb in the PF
6778  * @pf: board private structure
6779  *
6780  * On error: returns error code (negative)
6781  * On success: returns vsi index in PF (positive)
6782  **/
6783 static int i40e_veb_mem_alloc(struct i40e_pf *pf)
6784 {
6785         int ret = -ENOENT;
6786         struct i40e_veb *veb;
6787         int i;
6788
6789         /* Need to protect the allocation of switch elements at the PF level */
6790         mutex_lock(&pf->switch_mutex);
6791
6792         /* VEB list may be fragmented if VEB creation/destruction has
6793          * been happening.  We can afford to do a quick scan to look
6794          * for any free slots in the list.
6795          *
6796          * find next empty veb slot, looping back around if necessary
6797          */
6798         i = 0;
6799         while ((i < I40E_MAX_VEB) && (pf->veb[i] != NULL))
6800                 i++;
6801         if (i >= I40E_MAX_VEB) {
6802                 ret = -ENOMEM;
6803                 goto err_alloc_veb;  /* out of VEB slots! */
6804         }
6805
6806         veb = kzalloc(sizeof(*veb), GFP_KERNEL);
6807         if (!veb) {
6808                 ret = -ENOMEM;
6809                 goto err_alloc_veb;
6810         }
6811         veb->pf = pf;
6812         veb->idx = i;
6813         veb->enabled_tc = 1;
6814
6815         pf->veb[i] = veb;
6816         ret = i;
6817 err_alloc_veb:
6818         mutex_unlock(&pf->switch_mutex);
6819         return ret;
6820 }
6821
6822 /**
6823  * i40e_switch_branch_release - Delete a branch of the switch tree
6824  * @branch: where to start deleting
6825  *
6826  * This uses recursion to find the tips of the branch to be
6827  * removed, deleting until we get back to and can delete this VEB.
6828  **/
6829 static void i40e_switch_branch_release(struct i40e_veb *branch)
6830 {
6831         struct i40e_pf *pf = branch->pf;
6832         u16 branch_seid = branch->seid;
6833         u16 veb_idx = branch->idx;
6834         int i;
6835
6836         /* release any VEBs on this VEB - RECURSION */
6837         for (i = 0; i < I40E_MAX_VEB; i++) {
6838                 if (!pf->veb[i])
6839                         continue;
6840                 if (pf->veb[i]->uplink_seid == branch->seid)
6841                         i40e_switch_branch_release(pf->veb[i]);
6842         }
6843
6844         /* Release the VSIs on this VEB, but not the owner VSI.
6845          *
6846          * NOTE: Removing the last VSI on a VEB has the SIDE EFFECT of removing
6847          *       the VEB itself, so don't use (*branch) after this loop.
6848          */
6849         for (i = 0; i < pf->hw.func_caps.num_vsis; i++) {
6850                 if (!pf->vsi[i])
6851                         continue;
6852                 if (pf->vsi[i]->uplink_seid == branch_seid &&
6853                    (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) {
6854                         i40e_vsi_release(pf->vsi[i]);
6855                 }
6856         }
6857
6858         /* There's one corner case where the VEB might not have been
6859          * removed, so double check it here and remove it if needed.
6860          * This case happens if the veb was created from the debugfs
6861          * commands and no VSIs were added to it.
6862          */
6863         if (pf->veb[veb_idx])
6864                 i40e_veb_release(pf->veb[veb_idx]);
6865 }
6866
6867 /**
6868  * i40e_veb_clear - remove veb struct
6869  * @veb: the veb to remove
6870  **/
6871 static void i40e_veb_clear(struct i40e_veb *veb)
6872 {
6873         if (!veb)
6874                 return;
6875
6876         if (veb->pf) {
6877                 struct i40e_pf *pf = veb->pf;
6878
6879                 mutex_lock(&pf->switch_mutex);
6880                 if (pf->veb[veb->idx] == veb)
6881                         pf->veb[veb->idx] = NULL;
6882                 mutex_unlock(&pf->switch_mutex);
6883         }
6884
6885         kfree(veb);
6886 }
6887
6888 /**
6889  * i40e_veb_release - Delete a VEB and free its resources
6890  * @veb: the VEB being removed
6891  **/
6892 void i40e_veb_release(struct i40e_veb *veb)
6893 {
6894         struct i40e_vsi *vsi = NULL;
6895         struct i40e_pf *pf;
6896         int i, n = 0;
6897
6898         pf = veb->pf;
6899
6900         /* find the remaining VSI and check for extras */
6901         for (i = 0; i < pf->hw.func_caps.num_vsis; i++) {
6902                 if (pf->vsi[i] && pf->vsi[i]->uplink_seid == veb->seid) {
6903                         n++;
6904                         vsi = pf->vsi[i];
6905                 }
6906         }
6907         if (n != 1) {
6908                 dev_info(&pf->pdev->dev,
6909                          "can't remove VEB %d with %d VSIs left\n",
6910                          veb->seid, n);
6911                 return;
6912         }
6913
6914         /* move the remaining VSI to uplink veb */
6915         vsi->flags &= ~I40E_VSI_FLAG_VEB_OWNER;
6916         if (veb->uplink_seid) {
6917                 vsi->uplink_seid = veb->uplink_seid;
6918                 if (veb->uplink_seid == pf->mac_seid)
6919                         vsi->veb_idx = I40E_NO_VEB;
6920                 else
6921                         vsi->veb_idx = veb->veb_idx;
6922         } else {
6923                 /* floating VEB */
6924                 vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid;
6925                 vsi->veb_idx = pf->vsi[pf->lan_vsi]->veb_idx;
6926         }
6927
6928         i40e_aq_delete_element(&pf->hw, veb->seid, NULL);
6929         i40e_veb_clear(veb);
6930
6931         return;
6932 }
6933
6934 /**
6935  * i40e_add_veb - create the VEB in the switch
6936  * @veb: the VEB to be instantiated
6937  * @vsi: the controlling VSI
6938  **/
6939 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi)
6940 {
6941         bool is_default = false;
6942         bool is_cloud = false;
6943         int ret;
6944
6945         /* get a VEB from the hardware */
6946         ret = i40e_aq_add_veb(&veb->pf->hw, veb->uplink_seid, vsi->seid,
6947                               veb->enabled_tc, is_default,
6948                               is_cloud, &veb->seid, NULL);
6949         if (ret) {
6950                 dev_info(&veb->pf->pdev->dev,
6951                          "couldn't add VEB, err %d, aq_err %d\n",
6952                          ret, veb->pf->hw.aq.asq_last_status);
6953                 return -EPERM;
6954         }
6955
6956         /* get statistics counter */
6957         ret = i40e_aq_get_veb_parameters(&veb->pf->hw, veb->seid, NULL, NULL,
6958                                          &veb->stats_idx, NULL, NULL, NULL);
6959         if (ret) {
6960                 dev_info(&veb->pf->pdev->dev,
6961                          "couldn't get VEB statistics idx, err %d, aq_err %d\n",
6962                          ret, veb->pf->hw.aq.asq_last_status);
6963                 return -EPERM;
6964         }
6965         ret = i40e_veb_get_bw_info(veb);
6966         if (ret) {
6967                 dev_info(&veb->pf->pdev->dev,
6968                          "couldn't get VEB bw info, err %d, aq_err %d\n",
6969                          ret, veb->pf->hw.aq.asq_last_status);
6970                 i40e_aq_delete_element(&veb->pf->hw, veb->seid, NULL);
6971                 return -ENOENT;
6972         }
6973
6974         vsi->uplink_seid = veb->seid;
6975         vsi->veb_idx = veb->idx;
6976         vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;
6977
6978         return 0;
6979 }
6980
6981 /**
6982  * i40e_veb_setup - Set up a VEB
6983  * @pf: board private structure
6984  * @flags: VEB setup flags
6985  * @uplink_seid: the switch element to link to
6986  * @vsi_seid: the initial VSI seid
6987  * @enabled_tc: Enabled TC bit-map
6988  *
6989  * This allocates the sw VEB structure and links it into the switch
6990  * It is possible and legal for this to be a duplicate of an already
6991  * existing VEB.  It is also possible for both uplink and vsi seids
6992  * to be zero, in order to create a floating VEB.
6993  *
6994  * Returns pointer to the successfully allocated VEB sw struct on
6995  * success, otherwise returns NULL on failure.
6996  **/
6997 struct i40e_veb *i40e_veb_setup(struct i40e_pf *pf, u16 flags,
6998                                 u16 uplink_seid, u16 vsi_seid,
6999                                 u8 enabled_tc)
7000 {
7001         struct i40e_veb *veb, *uplink_veb = NULL;
7002         int vsi_idx, veb_idx;
7003         int ret;
7004
7005         /* if one seid is 0, the other must be 0 to create a floating relay */
7006         if ((uplink_seid == 0 || vsi_seid == 0) &&
7007             (uplink_seid + vsi_seid != 0)) {
7008                 dev_info(&pf->pdev->dev,
7009                          "one, not both seid's are 0: uplink=%d vsi=%d\n",
7010                          uplink_seid, vsi_seid);
7011                 return NULL;
7012         }
7013
7014         /* make sure there is such a vsi and uplink */
7015         for (vsi_idx = 0; vsi_idx < pf->hw.func_caps.num_vsis; vsi_idx++)
7016                 if (pf->vsi[vsi_idx] && pf->vsi[vsi_idx]->seid == vsi_seid)
7017                         break;
7018         if (vsi_idx >= pf->hw.func_caps.num_vsis && vsi_seid != 0) {
7019                 dev_info(&pf->pdev->dev, "vsi seid %d not found\n",
7020                          vsi_seid);
7021                 return NULL;
7022         }
7023
7024         if (uplink_seid && uplink_seid != pf->mac_seid) {
7025                 for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) {
7026                         if (pf->veb[veb_idx] &&
7027                             pf->veb[veb_idx]->seid == uplink_seid) {
7028                                 uplink_veb = pf->veb[veb_idx];
7029                                 break;
7030                         }
7031                 }
7032                 if (!uplink_veb) {
7033                         dev_info(&pf->pdev->dev,
7034                                  "uplink seid %d not found\n", uplink_seid);
7035                         return NULL;
7036                 }
7037         }
7038
7039         /* get veb sw struct */
7040         veb_idx = i40e_veb_mem_alloc(pf);
7041         if (veb_idx < 0)
7042                 goto err_alloc;
7043         veb = pf->veb[veb_idx];
7044         veb->flags = flags;
7045         veb->uplink_seid = uplink_seid;
7046         veb->veb_idx = (uplink_veb ? uplink_veb->idx : I40E_NO_VEB);
7047         veb->enabled_tc = (enabled_tc ? enabled_tc : 0x1);
7048
7049         /* create the VEB in the switch */
7050         ret = i40e_add_veb(veb, pf->vsi[vsi_idx]);
7051         if (ret)
7052                 goto err_veb;
7053
7054         return veb;
7055
7056 err_veb:
7057         i40e_veb_clear(veb);
7058 err_alloc:
7059         return NULL;
7060 }
7061
7062 /**
7063  * i40e_setup_pf_switch_element - set pf vars based on switch type
7064  * @pf: board private structure
7065  * @ele: element we are building info from
7066  * @num_reported: total number of elements
7067  * @printconfig: should we print the contents
7068  *
7069  * helper function to assist in extracting a few useful SEID values.
7070  **/
7071 static void i40e_setup_pf_switch_element(struct i40e_pf *pf,
7072                                 struct i40e_aqc_switch_config_element_resp *ele,
7073                                 u16 num_reported, bool printconfig)
7074 {
7075         u16 downlink_seid = le16_to_cpu(ele->downlink_seid);
7076         u16 uplink_seid = le16_to_cpu(ele->uplink_seid);
7077         u8 element_type = ele->element_type;
7078         u16 seid = le16_to_cpu(ele->seid);
7079
7080         if (printconfig)
7081                 dev_info(&pf->pdev->dev,
7082                          "type=%d seid=%d uplink=%d downlink=%d\n",
7083                          element_type, seid, uplink_seid, downlink_seid);
7084
7085         switch (element_type) {
7086         case I40E_SWITCH_ELEMENT_TYPE_MAC:
7087                 pf->mac_seid = seid;
7088                 break;
7089         case I40E_SWITCH_ELEMENT_TYPE_VEB:
7090                 /* Main VEB? */
7091                 if (uplink_seid != pf->mac_seid)
7092                         break;
7093                 if (pf->lan_veb == I40E_NO_VEB) {
7094                         int v;
7095
7096                         /* find existing or else empty VEB */
7097                         for (v = 0; v < I40E_MAX_VEB; v++) {
7098                                 if (pf->veb[v] && (pf->veb[v]->seid == seid)) {
7099                                         pf->lan_veb = v;
7100                                         break;
7101                                 }
7102                         }
7103                         if (pf->lan_veb == I40E_NO_VEB) {
7104                                 v = i40e_veb_mem_alloc(pf);
7105                                 if (v < 0)
7106                                         break;
7107                                 pf->lan_veb = v;
7108                         }
7109                 }
7110
7111                 pf->veb[pf->lan_veb]->seid = seid;
7112                 pf->veb[pf->lan_veb]->uplink_seid = pf->mac_seid;
7113                 pf->veb[pf->lan_veb]->pf = pf;
7114                 pf->veb[pf->lan_veb]->veb_idx = I40E_NO_VEB;
7115                 break;
7116         case I40E_SWITCH_ELEMENT_TYPE_VSI:
7117                 if (num_reported != 1)
7118                         break;
7119                 /* This is immediately after a reset so we can assume this is
7120                  * the PF's VSI
7121                  */
7122                 pf->mac_seid = uplink_seid;
7123                 pf->pf_seid = downlink_seid;
7124                 pf->main_vsi_seid = seid;
7125                 if (printconfig)
7126                         dev_info(&pf->pdev->dev,
7127                                  "pf_seid=%d main_vsi_seid=%d\n",
7128                                  pf->pf_seid, pf->main_vsi_seid);
7129                 break;
7130         case I40E_SWITCH_ELEMENT_TYPE_PF:
7131         case I40E_SWITCH_ELEMENT_TYPE_VF:
7132         case I40E_SWITCH_ELEMENT_TYPE_EMP:
7133         case I40E_SWITCH_ELEMENT_TYPE_BMC:
7134         case I40E_SWITCH_ELEMENT_TYPE_PE:
7135         case I40E_SWITCH_ELEMENT_TYPE_PA:
7136                 /* ignore these for now */
7137                 break;
7138         default:
7139                 dev_info(&pf->pdev->dev, "unknown element type=%d seid=%d\n",
7140                          element_type, seid);
7141                 break;
7142         }
7143 }
7144
7145 /**
7146  * i40e_fetch_switch_configuration - Get switch config from firmware
7147  * @pf: board private structure
7148  * @printconfig: should we print the contents
7149  *
7150  * Get the current switch configuration from the device and
7151  * extract a few useful SEID values.
7152  **/
7153 int i40e_fetch_switch_configuration(struct i40e_pf *pf, bool printconfig)
7154 {
7155         struct i40e_aqc_get_switch_config_resp *sw_config;
7156         u16 next_seid = 0;
7157         int ret = 0;
7158         u8 *aq_buf;
7159         int i;
7160
7161         aq_buf = kzalloc(I40E_AQ_LARGE_BUF, GFP_KERNEL);
7162         if (!aq_buf)
7163                 return -ENOMEM;
7164
7165         sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf;
7166         do {
7167                 u16 num_reported, num_total;
7168
7169                 ret = i40e_aq_get_switch_config(&pf->hw, sw_config,
7170                                                 I40E_AQ_LARGE_BUF,
7171                                                 &next_seid, NULL);
7172                 if (ret) {
7173                         dev_info(&pf->pdev->dev,
7174                                  "get switch config failed %d aq_err=%x\n",
7175                                  ret, pf->hw.aq.asq_last_status);
7176                         kfree(aq_buf);
7177                         return -ENOENT;
7178                 }
7179
7180                 num_reported = le16_to_cpu(sw_config->header.num_reported);
7181                 num_total = le16_to_cpu(sw_config->header.num_total);
7182
7183                 if (printconfig)
7184                         dev_info(&pf->pdev->dev,
7185                                  "header: %d reported %d total\n",
7186                                  num_reported, num_total);
7187
7188                 if (num_reported) {
7189                         int sz = sizeof(*sw_config) * num_reported;
7190
7191                         kfree(pf->sw_config);
7192                         pf->sw_config = kzalloc(sz, GFP_KERNEL);
7193                         if (pf->sw_config)
7194                                 memcpy(pf->sw_config, sw_config, sz);
7195                 }
7196
7197                 for (i = 0; i < num_reported; i++) {
7198                         struct i40e_aqc_switch_config_element_resp *ele =
7199                                 &sw_config->element[i];
7200
7201                         i40e_setup_pf_switch_element(pf, ele, num_reported,
7202                                                      printconfig);
7203                 }
7204         } while (next_seid != 0);
7205
7206         kfree(aq_buf);
7207         return ret;
7208 }
7209
7210 /**
7211  * i40e_setup_pf_switch - Setup the HW switch on startup or after reset
7212  * @pf: board private structure
7213  * @reinit: if the Main VSI needs to re-initialized.
7214  *
7215  * Returns 0 on success, negative value on failure
7216  **/
7217 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit)
7218 {
7219         u32 rxfc = 0, txfc = 0, rxfc_reg;
7220         int ret;
7221
7222         /* find out what's out there already */
7223         ret = i40e_fetch_switch_configuration(pf, false);
7224         if (ret) {
7225                 dev_info(&pf->pdev->dev,
7226                          "couldn't fetch switch config, err %d, aq_err %d\n",
7227                          ret, pf->hw.aq.asq_last_status);
7228                 return ret;
7229         }
7230         i40e_pf_reset_stats(pf);
7231
7232         /* fdir VSI must happen first to be sure it gets queue 0, but only
7233          * if there is enough room for the fdir VSI
7234          */
7235         if (pf->num_lan_qps > 1)
7236                 i40e_fdir_setup(pf);
7237
7238         /* first time setup */
7239         if (pf->lan_vsi == I40E_NO_VSI || reinit) {
7240                 struct i40e_vsi *vsi = NULL;
7241                 u16 uplink_seid;
7242
7243                 /* Set up the PF VSI associated with the PF's main VSI
7244                  * that is already in the HW switch
7245                  */
7246                 if (pf->lan_veb != I40E_NO_VEB && pf->veb[pf->lan_veb])
7247                         uplink_seid = pf->veb[pf->lan_veb]->seid;
7248                 else
7249                         uplink_seid = pf->mac_seid;
7250                 if (pf->lan_vsi == I40E_NO_VSI)
7251                         vsi = i40e_vsi_setup(pf, I40E_VSI_MAIN, uplink_seid, 0);
7252                 else if (reinit)
7253                         vsi = i40e_vsi_reinit_setup(pf->vsi[pf->lan_vsi]);
7254                 if (!vsi) {
7255                         dev_info(&pf->pdev->dev, "setup of MAIN VSI failed\n");
7256                         i40e_fdir_teardown(pf);
7257                         return -EAGAIN;
7258                 }
7259         } else {
7260                 /* force a reset of TC and queue layout configurations */
7261                 u8 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc;
7262                 pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0;
7263                 pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid;
7264                 i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc);
7265         }
7266         i40e_vlan_stripping_disable(pf->vsi[pf->lan_vsi]);
7267
7268         /* Setup static PF queue filter control settings */
7269         ret = i40e_setup_pf_filter_control(pf);
7270         if (ret) {
7271                 dev_info(&pf->pdev->dev, "setup_pf_filter_control failed: %d\n",
7272                          ret);
7273                 /* Failure here should not stop continuing other steps */
7274         }
7275
7276         /* enable RSS in the HW, even for only one queue, as the stack can use
7277          * the hash
7278          */
7279         if ((pf->flags & I40E_FLAG_RSS_ENABLED))
7280                 i40e_config_rss(pf);
7281
7282         /* fill in link information and enable LSE reporting */
7283         i40e_aq_get_link_info(&pf->hw, true, NULL, NULL);
7284         i40e_link_event(pf);
7285
7286         /* Initialize user-specific link properties */
7287         pf->fc_autoneg_status = ((pf->hw.phy.link_info.an_info &
7288                                   I40E_AQ_AN_COMPLETED) ? true : false);
7289         /* requested_mode is set in probe or by ethtool */
7290         if (!pf->fc_autoneg_status)
7291                 goto no_autoneg;
7292
7293         if ((pf->hw.phy.link_info.an_info & I40E_AQ_LINK_PAUSE_TX) &&
7294             (pf->hw.phy.link_info.an_info & I40E_AQ_LINK_PAUSE_RX))
7295                 pf->hw.fc.current_mode = I40E_FC_FULL;
7296         else if (pf->hw.phy.link_info.an_info & I40E_AQ_LINK_PAUSE_TX)
7297                 pf->hw.fc.current_mode = I40E_FC_TX_PAUSE;
7298         else if (pf->hw.phy.link_info.an_info & I40E_AQ_LINK_PAUSE_RX)
7299                 pf->hw.fc.current_mode = I40E_FC_RX_PAUSE;
7300         else
7301                 pf->hw.fc.current_mode = I40E_FC_NONE;
7302
7303         /* sync the flow control settings with the auto-neg values */
7304         switch (pf->hw.fc.current_mode) {
7305         case I40E_FC_FULL:
7306                 txfc = 1;
7307                 rxfc = 1;
7308                 break;
7309         case I40E_FC_TX_PAUSE:
7310                 txfc = 1;
7311                 rxfc = 0;
7312                 break;
7313         case I40E_FC_RX_PAUSE:
7314                 txfc = 0;
7315                 rxfc = 1;
7316                 break;
7317         case I40E_FC_NONE:
7318         case I40E_FC_DEFAULT:
7319                 txfc = 0;
7320                 rxfc = 0;
7321                 break;
7322         case I40E_FC_PFC:
7323                 /* TBD */
7324                 break;
7325         /* no default case, we have to handle all possibilities here */
7326         }
7327
7328         wr32(&pf->hw, I40E_PRTDCB_FCCFG, txfc << I40E_PRTDCB_FCCFG_TFCE_SHIFT);
7329
7330         rxfc_reg = rd32(&pf->hw, I40E_PRTDCB_MFLCN) &
7331                    ~I40E_PRTDCB_MFLCN_RFCE_MASK;
7332         rxfc_reg |= (rxfc << I40E_PRTDCB_MFLCN_RFCE_SHIFT);
7333
7334         wr32(&pf->hw, I40E_PRTDCB_MFLCN, rxfc_reg);
7335
7336         goto fc_complete;
7337
7338 no_autoneg:
7339         /* disable L2 flow control, user can turn it on if they wish */
7340         wr32(&pf->hw, I40E_PRTDCB_FCCFG, 0);
7341         wr32(&pf->hw, I40E_PRTDCB_MFLCN, rd32(&pf->hw, I40E_PRTDCB_MFLCN) &
7342                                          ~I40E_PRTDCB_MFLCN_RFCE_MASK);
7343
7344 fc_complete:
7345         i40e_ptp_init(pf);
7346
7347         return ret;
7348 }
7349
7350 /**
7351  * i40e_set_rss_size - helper to set rss_size
7352  * @pf: board private structure
7353  * @queues_left: how many queues
7354  */
7355 static u16 i40e_set_rss_size(struct i40e_pf *pf, int queues_left)
7356 {
7357         int num_tc0;
7358
7359         num_tc0 = min_t(int, queues_left, pf->rss_size_max);
7360         num_tc0 = min_t(int, num_tc0, num_online_cpus());
7361         num_tc0 = rounddown_pow_of_two(num_tc0);
7362
7363         return num_tc0;
7364 }
7365
7366 /**
7367  * i40e_determine_queue_usage - Work out queue distribution
7368  * @pf: board private structure
7369  **/
7370 static void i40e_determine_queue_usage(struct i40e_pf *pf)
7371 {
7372         int accum_tc_size;
7373         int queues_left;
7374
7375         pf->num_lan_qps = 0;
7376         pf->num_tc_qps = rounddown_pow_of_two(pf->num_tc_qps);
7377         accum_tc_size = (I40E_MAX_TRAFFIC_CLASS - 1) * pf->num_tc_qps;
7378
7379         /* Find the max queues to be put into basic use.  We'll always be
7380          * using TC0, whether or not DCB is running, and TC0 will get the
7381          * big RSS set.
7382          */
7383         queues_left = pf->hw.func_caps.num_tx_qp;
7384
7385         if   (!(pf->flags & I40E_FLAG_MSIX_ENABLED) ||
7386                 !(pf->flags & (I40E_FLAG_RSS_ENABLED |
7387                 I40E_FLAG_FDIR_ENABLED | I40E_FLAG_DCB_ENABLED)) ||
7388                 (queues_left == 1)) {
7389
7390                 /* one qp for PF, no queues for anything else */
7391                 queues_left = 0;
7392                 pf->rss_size = pf->num_lan_qps = 1;
7393
7394                 /* make sure all the fancies are disabled */
7395                 pf->flags &= ~(I40E_FLAG_RSS_ENABLED       |
7396                                 I40E_FLAG_FDIR_ENABLED     |
7397                                 I40E_FLAG_FDIR_ATR_ENABLED |
7398                                 I40E_FLAG_DCB_ENABLED      |
7399                                 I40E_FLAG_SRIOV_ENABLED    |
7400                                 I40E_FLAG_VMDQ_ENABLED);
7401
7402         } else if (pf->flags & I40E_FLAG_RSS_ENABLED      &&
7403                    !(pf->flags & I40E_FLAG_FDIR_ENABLED)  &&
7404                    !(pf->flags & I40E_FLAG_DCB_ENABLED)) {
7405
7406                 pf->rss_size = i40e_set_rss_size(pf, queues_left);
7407
7408                 queues_left -= pf->rss_size;
7409                 pf->num_lan_qps = pf->rss_size_max;
7410
7411         } else if (pf->flags & I40E_FLAG_RSS_ENABLED      &&
7412                    !(pf->flags & I40E_FLAG_FDIR_ENABLED)  &&
7413                    (pf->flags & I40E_FLAG_DCB_ENABLED)) {
7414
7415                 /* save num_tc_qps queues for TCs 1 thru 7 and the rest
7416                  * are set up for RSS in TC0
7417                  */
7418                 queues_left -= accum_tc_size;
7419
7420                 pf->rss_size = i40e_set_rss_size(pf, queues_left);
7421
7422                 queues_left -= pf->rss_size;
7423                 if (queues_left < 0) {
7424                         dev_info(&pf->pdev->dev, "not enough queues for DCB\n");
7425                         return;
7426                 }
7427
7428                 pf->num_lan_qps = pf->rss_size_max + accum_tc_size;
7429
7430         } else if (pf->flags & I40E_FLAG_RSS_ENABLED   &&
7431                   (pf->flags & I40E_FLAG_FDIR_ENABLED) &&
7432                   !(pf->flags & I40E_FLAG_DCB_ENABLED)) {
7433
7434                 queues_left -= 1; /* save 1 queue for FD */
7435
7436                 pf->rss_size = i40e_set_rss_size(pf, queues_left);
7437
7438                 queues_left -= pf->rss_size;
7439                 if (queues_left < 0) {
7440                         dev_info(&pf->pdev->dev, "not enough queues for Flow Director\n");
7441                         return;
7442                 }
7443
7444                 pf->num_lan_qps = pf->rss_size_max;
7445
7446         } else if (pf->flags & I40E_FLAG_RSS_ENABLED   &&
7447                   (pf->flags & I40E_FLAG_FDIR_ENABLED) &&
7448                   (pf->flags & I40E_FLAG_DCB_ENABLED)) {
7449
7450                 /* save 1 queue for TCs 1 thru 7,
7451                  * 1 queue for flow director,
7452                  * and the rest are set up for RSS in TC0
7453                  */
7454                 queues_left -= 1;
7455                 queues_left -= accum_tc_size;
7456
7457                 pf->rss_size = i40e_set_rss_size(pf, queues_left);
7458                 queues_left -= pf->rss_size;
7459                 if (queues_left < 0) {
7460                         dev_info(&pf->pdev->dev, "not enough queues for DCB and Flow Director\n");
7461                         return;
7462                 }
7463
7464                 pf->num_lan_qps = pf->rss_size_max + accum_tc_size;
7465
7466         } else {
7467                 dev_info(&pf->pdev->dev,
7468                          "Invalid configuration, flags=0x%08llx\n", pf->flags);
7469                 return;
7470         }
7471
7472         if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
7473             pf->num_vf_qps && pf->num_req_vfs && queues_left) {
7474                 pf->num_req_vfs = min_t(int, pf->num_req_vfs, (queues_left /
7475                                                                pf->num_vf_qps));
7476                 queues_left -= (pf->num_req_vfs * pf->num_vf_qps);
7477         }
7478
7479         if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) &&
7480             pf->num_vmdq_vsis && pf->num_vmdq_qps && queues_left) {
7481                 pf->num_vmdq_vsis = min_t(int, pf->num_vmdq_vsis,
7482                                           (queues_left / pf->num_vmdq_qps));
7483                 queues_left -= (pf->num_vmdq_vsis * pf->num_vmdq_qps);
7484         }
7485
7486         pf->queues_left = queues_left;
7487         return;
7488 }
7489
7490 /**
7491  * i40e_setup_pf_filter_control - Setup PF static filter control
7492  * @pf: PF to be setup
7493  *
7494  * i40e_setup_pf_filter_control sets up a pf's initial filter control
7495  * settings. If PE/FCoE are enabled then it will also set the per PF
7496  * based filter sizes required for them. It also enables Flow director,
7497  * ethertype and macvlan type filter settings for the pf.
7498  *
7499  * Returns 0 on success, negative on failure
7500  **/
7501 static int i40e_setup_pf_filter_control(struct i40e_pf *pf)
7502 {
7503         struct i40e_filter_control_settings *settings = &pf->filter_settings;
7504
7505         settings->hash_lut_size = I40E_HASH_LUT_SIZE_128;
7506
7507         /* Flow Director is enabled */
7508         if (pf->flags & (I40E_FLAG_FDIR_ENABLED | I40E_FLAG_FDIR_ATR_ENABLED))
7509                 settings->enable_fdir = true;
7510
7511         /* Ethtype and MACVLAN filters enabled for PF */
7512         settings->enable_ethtype = true;
7513         settings->enable_macvlan = true;
7514
7515         if (i40e_set_filter_control(&pf->hw, settings))
7516                 return -ENOENT;
7517
7518         return 0;
7519 }
7520
7521 /**
7522  * i40e_probe - Device initialization routine
7523  * @pdev: PCI device information struct
7524  * @ent: entry in i40e_pci_tbl
7525  *
7526  * i40e_probe initializes a pf identified by a pci_dev structure.
7527  * The OS initialization, configuring of the pf private structure,
7528  * and a hardware reset occur.
7529  *
7530  * Returns 0 on success, negative on failure
7531  **/
7532 static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
7533 {
7534         struct i40e_driver_version dv;
7535         struct i40e_pf *pf;
7536         struct i40e_hw *hw;
7537         static u16 pfs_found;
7538         u16 link_status;
7539         int err = 0;
7540         u32 len;
7541
7542         err = pci_enable_device_mem(pdev);
7543         if (err)
7544                 return err;
7545
7546         /* set up for high or low dma */
7547         if (!dma_set_mask(&pdev->dev, DMA_BIT_MASK(64))) {
7548                 /* coherent mask for the same size will always succeed if
7549                  * dma_set_mask does
7550                  */
7551                 dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
7552         } else if (!dma_set_mask(&pdev->dev, DMA_BIT_MASK(32))) {
7553                 dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
7554         } else {
7555                 dev_err(&pdev->dev, "DMA configuration failed: %d\n", err);
7556                 err = -EIO;
7557                 goto err_dma;
7558         }
7559
7560         /* set up pci connections */
7561         err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
7562                                            IORESOURCE_MEM), i40e_driver_name);
7563         if (err) {
7564                 dev_info(&pdev->dev,
7565                          "pci_request_selected_regions failed %d\n", err);
7566                 goto err_pci_reg;
7567         }
7568
7569         pci_enable_pcie_error_reporting(pdev);
7570         pci_set_master(pdev);
7571
7572         /* Now that we have a PCI connection, we need to do the
7573          * low level device setup.  This is primarily setting up
7574          * the Admin Queue structures and then querying for the
7575          * device's current profile information.
7576          */
7577         pf = kzalloc(sizeof(*pf), GFP_KERNEL);
7578         if (!pf) {
7579                 err = -ENOMEM;
7580                 goto err_pf_alloc;
7581         }
7582         pf->next_vsi = 0;
7583         pf->pdev = pdev;
7584         set_bit(__I40E_DOWN, &pf->state);
7585
7586         hw = &pf->hw;
7587         hw->back = pf;
7588         hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
7589                               pci_resource_len(pdev, 0));
7590         if (!hw->hw_addr) {
7591                 err = -EIO;
7592                 dev_info(&pdev->dev, "ioremap(0x%04x, 0x%04x) failed: 0x%x\n",
7593                          (unsigned int)pci_resource_start(pdev, 0),
7594                          (unsigned int)pci_resource_len(pdev, 0), err);
7595                 goto err_ioremap;
7596         }
7597         hw->vendor_id = pdev->vendor;
7598         hw->device_id = pdev->device;
7599         pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id);
7600         hw->subsystem_vendor_id = pdev->subsystem_vendor;
7601         hw->subsystem_device_id = pdev->subsystem_device;
7602         hw->bus.device = PCI_SLOT(pdev->devfn);
7603         hw->bus.func = PCI_FUNC(pdev->devfn);
7604         pf->instance = pfs_found;
7605
7606         /* do a special CORER for clearing PXE mode once at init */
7607         if (hw->revision_id == 0 &&
7608             (rd32(hw, I40E_GLLAN_RCTL_0) & I40E_GLLAN_RCTL_0_PXE_MODE_MASK)) {
7609                 wr32(hw, I40E_GLGEN_RTRIG, I40E_GLGEN_RTRIG_CORER_MASK);
7610                 i40e_flush(hw);
7611                 msleep(200);
7612                 pf->corer_count++;
7613
7614                 i40e_clear_pxe_mode(hw);
7615         }
7616
7617         /* Reset here to make sure all is clean and to define PF 'n' */
7618         err = i40e_pf_reset(hw);
7619         if (err) {
7620                 dev_info(&pdev->dev, "Initial pf_reset failed: %d\n", err);
7621                 goto err_pf_reset;
7622         }
7623         pf->pfr_count++;
7624
7625         hw->aq.num_arq_entries = I40E_AQ_LEN;
7626         hw->aq.num_asq_entries = I40E_AQ_LEN;
7627         hw->aq.arq_buf_size = I40E_MAX_AQ_BUF_SIZE;
7628         hw->aq.asq_buf_size = I40E_MAX_AQ_BUF_SIZE;
7629         pf->adminq_work_limit = I40E_AQ_WORK_LIMIT;
7630         snprintf(pf->misc_int_name, sizeof(pf->misc_int_name) - 1,
7631                  "%s-pf%d:misc",
7632                  dev_driver_string(&pf->pdev->dev), pf->hw.pf_id);
7633
7634         err = i40e_init_shared_code(hw);
7635         if (err) {
7636                 dev_info(&pdev->dev, "init_shared_code failed: %d\n", err);
7637                 goto err_pf_reset;
7638         }
7639
7640         /* set up a default setting for link flow control */
7641         pf->hw.fc.requested_mode = I40E_FC_NONE;
7642
7643         err = i40e_init_adminq(hw);
7644         dev_info(&pdev->dev, "%s\n", i40e_fw_version_str(hw));
7645         if (((hw->nvm.version & I40E_NVM_VERSION_HI_MASK)
7646                  >> I40E_NVM_VERSION_HI_SHIFT) != I40E_CURRENT_NVM_VERSION_HI) {
7647                 dev_info(&pdev->dev,
7648                          "warning: NVM version not supported, supported version: %02x.%02x\n",
7649                          I40E_CURRENT_NVM_VERSION_HI,
7650                          I40E_CURRENT_NVM_VERSION_LO);
7651         }
7652         if (err) {
7653                 dev_info(&pdev->dev,
7654                          "init_adminq failed: %d expecting API %02x.%02x\n",
7655                          err,
7656                          I40E_FW_API_VERSION_MAJOR, I40E_FW_API_VERSION_MINOR);
7657                 goto err_pf_reset;
7658         }
7659
7660         i40e_clear_pxe_mode(hw);
7661         err = i40e_get_capabilities(pf);
7662         if (err)
7663                 goto err_adminq_setup;
7664
7665         err = i40e_sw_init(pf);
7666         if (err) {
7667                 dev_info(&pdev->dev, "sw_init failed: %d\n", err);
7668                 goto err_sw_init;
7669         }
7670
7671         err = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
7672                                 hw->func_caps.num_rx_qp,
7673                                 pf->fcoe_hmc_cntx_num, pf->fcoe_hmc_filt_num);
7674         if (err) {
7675                 dev_info(&pdev->dev, "init_lan_hmc failed: %d\n", err);
7676                 goto err_init_lan_hmc;
7677         }
7678
7679         err = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
7680         if (err) {
7681                 dev_info(&pdev->dev, "configure_lan_hmc failed: %d\n", err);
7682                 err = -ENOENT;
7683                 goto err_configure_lan_hmc;
7684         }
7685
7686         i40e_get_mac_addr(hw, hw->mac.addr);
7687         if (!is_valid_ether_addr(hw->mac.addr)) {
7688                 dev_info(&pdev->dev, "invalid MAC address %pM\n", hw->mac.addr);
7689                 err = -EIO;
7690                 goto err_mac_addr;
7691         }
7692         dev_info(&pdev->dev, "MAC address: %pM\n", hw->mac.addr);
7693         memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
7694
7695         pci_set_drvdata(pdev, pf);
7696         pci_save_state(pdev);
7697
7698         /* set up periodic task facility */
7699         setup_timer(&pf->service_timer, i40e_service_timer, (unsigned long)pf);
7700         pf->service_timer_period = HZ;
7701
7702         INIT_WORK(&pf->service_task, i40e_service_task);
7703         clear_bit(__I40E_SERVICE_SCHED, &pf->state);
7704         pf->flags |= I40E_FLAG_NEED_LINK_UPDATE;
7705         pf->link_check_timeout = jiffies;
7706
7707         /* WoL defaults to disabled */
7708         pf->wol_en = false;
7709         device_set_wakeup_enable(&pf->pdev->dev, pf->wol_en);
7710
7711         /* set up the main switch operations */
7712         i40e_determine_queue_usage(pf);
7713         i40e_init_interrupt_scheme(pf);
7714
7715         /* Set up the *vsi struct based on the number of VSIs in the HW,
7716          * and set up our local tracking of the MAIN PF vsi.
7717          */
7718         len = sizeof(struct i40e_vsi *) * pf->hw.func_caps.num_vsis;
7719         pf->vsi = kzalloc(len, GFP_KERNEL);
7720         if (!pf->vsi) {
7721                 err = -ENOMEM;
7722                 goto err_switch_setup;
7723         }
7724
7725         err = i40e_setup_pf_switch(pf, false);
7726         if (err) {
7727                 dev_info(&pdev->dev, "setup_pf_switch failed: %d\n", err);
7728                 goto err_vsis;
7729         }
7730
7731         /* The main driver is (mostly) up and happy. We need to set this state
7732          * before setting up the misc vector or we get a race and the vector
7733          * ends up disabled forever.
7734          */
7735         clear_bit(__I40E_DOWN, &pf->state);
7736
7737         /* In case of MSIX we are going to setup the misc vector right here
7738          * to handle admin queue events etc. In case of legacy and MSI
7739          * the misc functionality and queue processing is combined in
7740          * the same vector and that gets setup at open.
7741          */
7742         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
7743                 err = i40e_setup_misc_vector(pf);
7744                 if (err) {
7745                         dev_info(&pdev->dev,
7746                                  "setup of misc vector failed: %d\n", err);
7747                         goto err_vsis;
7748                 }
7749         }
7750
7751         /* prep for VF support */
7752         if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
7753             (pf->flags & I40E_FLAG_MSIX_ENABLED)) {
7754                 u32 val;
7755
7756                 /* disable link interrupts for VFs */
7757                 val = rd32(hw, I40E_PFGEN_PORTMDIO_NUM);
7758                 val &= ~I40E_PFGEN_PORTMDIO_NUM_VFLINK_STAT_ENA_MASK;
7759                 wr32(hw, I40E_PFGEN_PORTMDIO_NUM, val);
7760                 i40e_flush(hw);
7761         }
7762
7763         pfs_found++;
7764
7765         i40e_dbg_pf_init(pf);
7766
7767         /* tell the firmware that we're starting */
7768         dv.major_version = DRV_VERSION_MAJOR;
7769         dv.minor_version = DRV_VERSION_MINOR;
7770         dv.build_version = DRV_VERSION_BUILD;
7771         dv.subbuild_version = 0;
7772         i40e_aq_send_driver_version(&pf->hw, &dv, NULL);
7773
7774         /* since everything's happy, start the service_task timer */
7775         mod_timer(&pf->service_timer,
7776                   round_jiffies(jiffies + pf->service_timer_period));
7777
7778         /* Get the negotiated link width and speed from PCI config space */
7779         pcie_capability_read_word(pf->pdev, PCI_EXP_LNKSTA, &link_status);
7780
7781         i40e_set_pci_config_data(hw, link_status);
7782
7783         dev_info(&pdev->dev, "PCI Express: %s %s\n",
7784                 (hw->bus.speed == i40e_bus_speed_8000 ? "Speed 8.0GT/s" :
7785                  hw->bus.speed == i40e_bus_speed_5000 ? "Speed 5.0GT/s" :
7786                  hw->bus.speed == i40e_bus_speed_2500 ? "Speed 2.5GT/s" :
7787                  "Unknown"),
7788                 (hw->bus.width == i40e_bus_width_pcie_x8 ? "Width x8" :
7789                  hw->bus.width == i40e_bus_width_pcie_x4 ? "Width x4" :
7790                  hw->bus.width == i40e_bus_width_pcie_x2 ? "Width x2" :
7791                  hw->bus.width == i40e_bus_width_pcie_x1 ? "Width x1" :
7792                  "Unknown"));
7793
7794         if (hw->bus.width < i40e_bus_width_pcie_x8 ||
7795             hw->bus.speed < i40e_bus_speed_8000) {
7796                 dev_warn(&pdev->dev, "PCI-Express bandwidth available for this device may be insufficient for optimal performance.\n");
7797                 dev_warn(&pdev->dev, "Please move the device to a different PCI-e link with more lanes and/or higher transfer rate.\n");
7798         }
7799
7800         return 0;
7801
7802         /* Unwind what we've done if something failed in the setup */
7803 err_vsis:
7804         set_bit(__I40E_DOWN, &pf->state);
7805         i40e_clear_interrupt_scheme(pf);
7806         kfree(pf->vsi);
7807 err_switch_setup:
7808         i40e_reset_interrupt_capability(pf);
7809         del_timer_sync(&pf->service_timer);
7810 err_mac_addr:
7811 err_configure_lan_hmc:
7812         (void)i40e_shutdown_lan_hmc(hw);
7813 err_init_lan_hmc:
7814         kfree(pf->qp_pile);
7815         kfree(pf->irq_pile);
7816 err_sw_init:
7817 err_adminq_setup:
7818         (void)i40e_shutdown_adminq(hw);
7819 err_pf_reset:
7820         iounmap(hw->hw_addr);
7821 err_ioremap:
7822         kfree(pf);
7823 err_pf_alloc:
7824         pci_disable_pcie_error_reporting(pdev);
7825         pci_release_selected_regions(pdev,
7826                                      pci_select_bars(pdev, IORESOURCE_MEM));
7827 err_pci_reg:
7828 err_dma:
7829         pci_disable_device(pdev);
7830         return err;
7831 }
7832
7833 /**
7834  * i40e_remove - Device removal routine
7835  * @pdev: PCI device information struct
7836  *
7837  * i40e_remove is called by the PCI subsystem to alert the driver
7838  * that is should release a PCI device.  This could be caused by a
7839  * Hot-Plug event, or because the driver is going to be removed from
7840  * memory.
7841  **/
7842 static void i40e_remove(struct pci_dev *pdev)
7843 {
7844         struct i40e_pf *pf = pci_get_drvdata(pdev);
7845         i40e_status ret_code;
7846         u32 reg;
7847         int i;
7848
7849         i40e_dbg_pf_exit(pf);
7850
7851         i40e_ptp_stop(pf);
7852
7853         if (pf->flags & I40E_FLAG_SRIOV_ENABLED) {
7854                 i40e_free_vfs(pf);
7855                 pf->flags &= ~I40E_FLAG_SRIOV_ENABLED;
7856         }
7857
7858         /* no more scheduling of any task */
7859         set_bit(__I40E_DOWN, &pf->state);
7860         del_timer_sync(&pf->service_timer);
7861         cancel_work_sync(&pf->service_task);
7862
7863         i40e_fdir_teardown(pf);
7864
7865         /* If there is a switch structure or any orphans, remove them.
7866          * This will leave only the PF's VSI remaining.
7867          */
7868         for (i = 0; i < I40E_MAX_VEB; i++) {
7869                 if (!pf->veb[i])
7870                         continue;
7871
7872                 if (pf->veb[i]->uplink_seid == pf->mac_seid ||
7873                     pf->veb[i]->uplink_seid == 0)
7874                         i40e_switch_branch_release(pf->veb[i]);
7875         }
7876
7877         /* Now we can shutdown the PF's VSI, just before we kill
7878          * adminq and hmc.
7879          */
7880         if (pf->vsi[pf->lan_vsi])
7881                 i40e_vsi_release(pf->vsi[pf->lan_vsi]);
7882
7883         i40e_stop_misc_vector(pf);
7884         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
7885                 synchronize_irq(pf->msix_entries[0].vector);
7886                 free_irq(pf->msix_entries[0].vector, pf);
7887         }
7888
7889         /* shutdown and destroy the HMC */
7890         ret_code = i40e_shutdown_lan_hmc(&pf->hw);
7891         if (ret_code)
7892                 dev_warn(&pdev->dev,
7893                          "Failed to destroy the HMC resources: %d\n", ret_code);
7894
7895         /* shutdown the adminq */
7896         ret_code = i40e_shutdown_adminq(&pf->hw);
7897         if (ret_code)
7898                 dev_warn(&pdev->dev,
7899                          "Failed to destroy the Admin Queue resources: %d\n",
7900                          ret_code);
7901
7902         /* Clear all dynamic memory lists of rings, q_vectors, and VSIs */
7903         i40e_clear_interrupt_scheme(pf);
7904         for (i = 0; i < pf->hw.func_caps.num_vsis; i++) {
7905                 if (pf->vsi[i]) {
7906                         i40e_vsi_clear_rings(pf->vsi[i]);
7907                         i40e_vsi_clear(pf->vsi[i]);
7908                         pf->vsi[i] = NULL;
7909                 }
7910         }
7911
7912         for (i = 0; i < I40E_MAX_VEB; i++) {
7913                 kfree(pf->veb[i]);
7914                 pf->veb[i] = NULL;
7915         }
7916
7917         kfree(pf->qp_pile);
7918         kfree(pf->irq_pile);
7919         kfree(pf->sw_config);
7920         kfree(pf->vsi);
7921
7922         /* force a PF reset to clean anything leftover */
7923         reg = rd32(&pf->hw, I40E_PFGEN_CTRL);
7924         wr32(&pf->hw, I40E_PFGEN_CTRL, (reg | I40E_PFGEN_CTRL_PFSWR_MASK));
7925         i40e_flush(&pf->hw);
7926
7927         iounmap(pf->hw.hw_addr);
7928         kfree(pf);
7929         pci_release_selected_regions(pdev,
7930                                      pci_select_bars(pdev, IORESOURCE_MEM));
7931
7932         pci_disable_pcie_error_reporting(pdev);
7933         pci_disable_device(pdev);
7934 }
7935
7936 /**
7937  * i40e_pci_error_detected - warning that something funky happened in PCI land
7938  * @pdev: PCI device information struct
7939  *
7940  * Called to warn that something happened and the error handling steps
7941  * are in progress.  Allows the driver to quiesce things, be ready for
7942  * remediation.
7943  **/
7944 static pci_ers_result_t i40e_pci_error_detected(struct pci_dev *pdev,
7945                                                 enum pci_channel_state error)
7946 {
7947         struct i40e_pf *pf = pci_get_drvdata(pdev);
7948
7949         dev_info(&pdev->dev, "%s: error %d\n", __func__, error);
7950
7951         /* shutdown all operations */
7952         if (!test_bit(__I40E_SUSPENDED, &pf->state)) {
7953                 rtnl_lock();
7954                 i40e_prep_for_reset(pf);
7955                 rtnl_unlock();
7956         }
7957
7958         /* Request a slot reset */
7959         return PCI_ERS_RESULT_NEED_RESET;
7960 }
7961
7962 /**
7963  * i40e_pci_error_slot_reset - a PCI slot reset just happened
7964  * @pdev: PCI device information struct
7965  *
7966  * Called to find if the driver can work with the device now that
7967  * the pci slot has been reset.  If a basic connection seems good
7968  * (registers are readable and have sane content) then return a
7969  * happy little PCI_ERS_RESULT_xxx.
7970  **/
7971 static pci_ers_result_t i40e_pci_error_slot_reset(struct pci_dev *pdev)
7972 {
7973         struct i40e_pf *pf = pci_get_drvdata(pdev);
7974         pci_ers_result_t result;
7975         int err;
7976         u32 reg;
7977
7978         dev_info(&pdev->dev, "%s\n", __func__);
7979         if (pci_enable_device_mem(pdev)) {
7980                 dev_info(&pdev->dev,
7981                          "Cannot re-enable PCI device after reset.\n");
7982                 result = PCI_ERS_RESULT_DISCONNECT;
7983         } else {
7984                 pci_set_master(pdev);
7985                 pci_restore_state(pdev);
7986                 pci_save_state(pdev);
7987                 pci_wake_from_d3(pdev, false);
7988
7989                 reg = rd32(&pf->hw, I40E_GLGEN_RTRIG);
7990                 if (reg == 0)
7991                         result = PCI_ERS_RESULT_RECOVERED;
7992                 else
7993                         result = PCI_ERS_RESULT_DISCONNECT;
7994         }
7995
7996         err = pci_cleanup_aer_uncorrect_error_status(pdev);
7997         if (err) {
7998                 dev_info(&pdev->dev,
7999                          "pci_cleanup_aer_uncorrect_error_status failed 0x%0x\n",
8000                          err);
8001                 /* non-fatal, continue */
8002         }
8003
8004         return result;
8005 }
8006
8007 /**
8008  * i40e_pci_error_resume - restart operations after PCI error recovery
8009  * @pdev: PCI device information struct
8010  *
8011  * Called to allow the driver to bring things back up after PCI error
8012  * and/or reset recovery has finished.
8013  **/
8014 static void i40e_pci_error_resume(struct pci_dev *pdev)
8015 {
8016         struct i40e_pf *pf = pci_get_drvdata(pdev);
8017
8018         dev_info(&pdev->dev, "%s\n", __func__);
8019         if (test_bit(__I40E_SUSPENDED, &pf->state))
8020                 return;
8021
8022         rtnl_lock();
8023         i40e_handle_reset_warning(pf);
8024         rtnl_lock();
8025 }
8026
8027 /**
8028  * i40e_shutdown - PCI callback for shutting down
8029  * @pdev: PCI device information struct
8030  **/
8031 static void i40e_shutdown(struct pci_dev *pdev)
8032 {
8033         struct i40e_pf *pf = pci_get_drvdata(pdev);
8034         struct i40e_hw *hw = &pf->hw;
8035
8036         set_bit(__I40E_SUSPENDED, &pf->state);
8037         set_bit(__I40E_DOWN, &pf->state);
8038         rtnl_lock();
8039         i40e_prep_for_reset(pf);
8040         rtnl_unlock();
8041
8042         wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
8043         wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
8044
8045         if (system_state == SYSTEM_POWER_OFF) {
8046                 pci_wake_from_d3(pdev, pf->wol_en);
8047                 pci_set_power_state(pdev, PCI_D3hot);
8048         }
8049 }
8050
8051 #ifdef CONFIG_PM
8052 /**
8053  * i40e_suspend - PCI callback for moving to D3
8054  * @pdev: PCI device information struct
8055  **/
8056 static int i40e_suspend(struct pci_dev *pdev, pm_message_t state)
8057 {
8058         struct i40e_pf *pf = pci_get_drvdata(pdev);
8059         struct i40e_hw *hw = &pf->hw;
8060
8061         set_bit(__I40E_SUSPENDED, &pf->state);
8062         set_bit(__I40E_DOWN, &pf->state);
8063         rtnl_lock();
8064         i40e_prep_for_reset(pf);
8065         rtnl_unlock();
8066
8067         wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
8068         wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
8069
8070         pci_wake_from_d3(pdev, pf->wol_en);
8071         pci_set_power_state(pdev, PCI_D3hot);
8072
8073         return 0;
8074 }
8075
8076 /**
8077  * i40e_resume - PCI callback for waking up from D3
8078  * @pdev: PCI device information struct
8079  **/
8080 static int i40e_resume(struct pci_dev *pdev)
8081 {
8082         struct i40e_pf *pf = pci_get_drvdata(pdev);
8083         u32 err;
8084
8085         pci_set_power_state(pdev, PCI_D0);
8086         pci_restore_state(pdev);
8087         /* pci_restore_state() clears dev->state_saves, so
8088          * call pci_save_state() again to restore it.
8089          */
8090         pci_save_state(pdev);
8091
8092         err = pci_enable_device_mem(pdev);
8093         if (err) {
8094                 dev_err(&pdev->dev,
8095                         "%s: Cannot enable PCI device from suspend\n",
8096                         __func__);
8097                 return err;
8098         }
8099         pci_set_master(pdev);
8100
8101         /* no wakeup events while running */
8102         pci_wake_from_d3(pdev, false);
8103
8104         /* handling the reset will rebuild the device state */
8105         if (test_and_clear_bit(__I40E_SUSPENDED, &pf->state)) {
8106                 clear_bit(__I40E_DOWN, &pf->state);
8107                 rtnl_lock();
8108                 i40e_reset_and_rebuild(pf, false);
8109                 rtnl_unlock();
8110         }
8111
8112         return 0;
8113 }
8114
8115 #endif
8116 static const struct pci_error_handlers i40e_err_handler = {
8117         .error_detected = i40e_pci_error_detected,
8118         .slot_reset = i40e_pci_error_slot_reset,
8119         .resume = i40e_pci_error_resume,
8120 };
8121
8122 static struct pci_driver i40e_driver = {
8123         .name     = i40e_driver_name,
8124         .id_table = i40e_pci_tbl,
8125         .probe    = i40e_probe,
8126         .remove   = i40e_remove,
8127 #ifdef CONFIG_PM
8128         .suspend  = i40e_suspend,
8129         .resume   = i40e_resume,
8130 #endif
8131         .shutdown = i40e_shutdown,
8132         .err_handler = &i40e_err_handler,
8133         .sriov_configure = i40e_pci_sriov_configure,
8134 };
8135
8136 /**
8137  * i40e_init_module - Driver registration routine
8138  *
8139  * i40e_init_module is the first routine called when the driver is
8140  * loaded. All it does is register with the PCI subsystem.
8141  **/
8142 static int __init i40e_init_module(void)
8143 {
8144         pr_info("%s: %s - version %s\n", i40e_driver_name,
8145                 i40e_driver_string, i40e_driver_version_str);
8146         pr_info("%s: %s\n", i40e_driver_name, i40e_copyright);
8147         i40e_dbg_init();
8148         return pci_register_driver(&i40e_driver);
8149 }
8150 module_init(i40e_init_module);
8151
8152 /**
8153  * i40e_exit_module - Driver exit cleanup routine
8154  *
8155  * i40e_exit_module is called just before the driver is removed
8156  * from memory.
8157  **/
8158 static void __exit i40e_exit_module(void)
8159 {
8160         pci_unregister_driver(&i40e_driver);
8161         i40e_dbg_exit();
8162 }
8163 module_exit(i40e_exit_module);