Pileus Git - ~andy/linux/blob - drivers/net/ethernet/intel/i40e/i40e_main.c

   1 /*******************************************************************************
   2  *
   3  * Intel Ethernet Controller XL710 Family Linux Driver
   4  * Copyright(c) 2013 Intel Corporation.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms and conditions of the GNU General Public License,
   8  * version 2, as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13  * more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along with
  16  * this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * The full GNU General Public License is included in this distribution in
  20  * the file called "COPYING".
  21  *
  22  * Contact Information:
  23  * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
  24  * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  25  *
  26  ******************************************************************************/
  27
  28 /* Local includes */
  29 #include "i40e.h"
  30 #ifdef CONFIG_I40E_VXLAN
  31 #include <net/vxlan.h>
  32 #endif
  33
  34 const char i40e_driver_name[] = "i40e";
  35 static const char i40e_driver_string[] =
  36                         "Intel(R) Ethernet Connection XL710 Network Driver";
  37
  38 #define DRV_KERN "-k"
  39
  40 #define DRV_VERSION_MAJOR 0
  41 #define DRV_VERSION_MINOR 3
  42 #define DRV_VERSION_BUILD 25
  43 #define DRV_VERSION __stringify(DRV_VERSION_MAJOR) "." \
  44              __stringify(DRV_VERSION_MINOR) "." \
  45              __stringify(DRV_VERSION_BUILD)    DRV_KERN
  46 const char i40e_driver_version_str[] = DRV_VERSION;
  47 static const char i40e_copyright[] = "Copyright (c) 2013 Intel Corporation.";
  48
  49 /* a bit of forward declarations */
  50 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi);
  51 static void i40e_handle_reset_warning(struct i40e_pf *pf);
  52 static int i40e_add_vsi(struct i40e_vsi *vsi);
  53 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi);
  54 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit);
  55 static int i40e_setup_misc_vector(struct i40e_pf *pf);
  56 static void i40e_determine_queue_usage(struct i40e_pf *pf);
  57 static int i40e_setup_pf_filter_control(struct i40e_pf *pf);
  58
  59 /* i40e_pci_tbl - PCI Device ID Table
  60  *
  61  * Last entry must be all 0s
  62  *
  63  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
  64  *   Class, Class Mask, private data (not used) }
  65  */
  66 static DEFINE_PCI_DEVICE_TABLE(i40e_pci_tbl) = {
  67         {PCI_VDEVICE(INTEL, I40E_SFP_XL710_DEVICE_ID), 0},
  68         {PCI_VDEVICE(INTEL, I40E_SFP_X710_DEVICE_ID), 0},
  69         {PCI_VDEVICE(INTEL, I40E_QEMU_DEVICE_ID), 0},
  70         {PCI_VDEVICE(INTEL, I40E_KX_A_DEVICE_ID), 0},
  71         {PCI_VDEVICE(INTEL, I40E_KX_B_DEVICE_ID), 0},
  72         {PCI_VDEVICE(INTEL, I40E_KX_C_DEVICE_ID), 0},
  73         {PCI_VDEVICE(INTEL, I40E_KX_D_DEVICE_ID), 0},
  74         {PCI_VDEVICE(INTEL, I40E_QSFP_A_DEVICE_ID), 0},
  75         {PCI_VDEVICE(INTEL, I40E_QSFP_B_DEVICE_ID), 0},
  76         {PCI_VDEVICE(INTEL, I40E_QSFP_C_DEVICE_ID), 0},
  77         /* required last entry */
  78         {0, }
  79 };
  80 MODULE_DEVICE_TABLE(pci, i40e_pci_tbl);
  81
  82 #define I40E_MAX_VF_COUNT 128
  83 static int debug = -1;
  84 module_param(debug, int, 0);
  85 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
  86
  87 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
  88 MODULE_DESCRIPTION("Intel(R) Ethernet Connection XL710 Network Driver");
  89 MODULE_LICENSE("GPL");
  90 MODULE_VERSION(DRV_VERSION);
  91
  92 /**
  93  * i40e_allocate_dma_mem_d - OS specific memory alloc for shared code
  94  * @hw:   pointer to the HW structure
  95  * @mem:  ptr to mem struct to fill out
  96  * @size: size of memory requested
  97  * @alignment: what to align the allocation to
  98  **/
  99 int i40e_allocate_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem,
 100                             u64 size, u32 alignment)
 101 {
 102         struct i40e_pf *pf = (struct i40e_pf *)hw->back;
 103
 104         mem->size = ALIGN(size, alignment);
 105         mem->va = dma_zalloc_coherent(&pf->pdev->dev, mem->size,
 106                                       &mem->pa, GFP_KERNEL);
 107         if (!mem->va)
 108                 return -ENOMEM;
 109
 110         return 0;
 111 }
 112
 113 /**
 114  * i40e_free_dma_mem_d - OS specific memory free for shared code
 115  * @hw:   pointer to the HW structure
 116  * @mem:  ptr to mem struct to free
 117  **/
 118 int i40e_free_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem)
 119 {
 120         struct i40e_pf *pf = (struct i40e_pf *)hw->back;
 121
 122         dma_free_coherent(&pf->pdev->dev, mem->size, mem->va, mem->pa);
 123         mem->va = NULL;
 124         mem->pa = 0;
 125         mem->size = 0;
 126
 127         return 0;
 128 }
 129
 130 /**
 131  * i40e_allocate_virt_mem_d - OS specific memory alloc for shared code
 132  * @hw:   pointer to the HW structure
 133  * @mem:  ptr to mem struct to fill out
 134  * @size: size of memory requested
 135  **/
 136 int i40e_allocate_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem,
 137                              u32 size)
 138 {
 139         mem->size = size;
 140         mem->va = kzalloc(size, GFP_KERNEL);
 141
 142         if (!mem->va)
 143                 return -ENOMEM;
 144
 145         return 0;
 146 }
 147
 148 /**
 149  * i40e_free_virt_mem_d - OS specific memory free for shared code
 150  * @hw:   pointer to the HW structure
 151  * @mem:  ptr to mem struct to free
 152  **/
 153 int i40e_free_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem)
 154 {
 155         /* it's ok to kfree a NULL pointer */
 156         kfree(mem->va);
 157         mem->va = NULL;
 158         mem->size = 0;
 159
 160         return 0;
 161 }
 162
 163 /**
 164  * i40e_get_lump - find a lump of free generic resource
 165  * @pf: board private structure
 166  * @pile: the pile of resource to search
 167  * @needed: the number of items needed
 168  * @id: an owner id to stick on the items assigned
 169  *
 170  * Returns the base item index of the lump, or negative for error
 171  *
 172  * The search_hint trick and lack of advanced fit-finding only work
 173  * because we're highly likely to have all the same size lump requests.
 174  * Linear search time and any fragmentation should be minimal.
 175  **/
 176 static int i40e_get_lump(struct i40e_pf *pf, struct i40e_lump_tracking *pile,
 177                          u16 needed, u16 id)
 178 {
 179         int ret = -ENOMEM;
 180         int i, j;
 181
 182         if (!pile || needed == 0 || id >= I40E_PILE_VALID_BIT) {
 183                 dev_info(&pf->pdev->dev,
 184                          "param err: pile=%p needed=%d id=0x%04x\n",
 185                          pile, needed, id);
 186                 return -EINVAL;
 187         }
 188
 189         /* start the linear search with an imperfect hint */
 190         i = pile->search_hint;
 191         while (i < pile->num_entries) {
 192                 /* skip already allocated entries */
 193                 if (pile->list[i] & I40E_PILE_VALID_BIT) {
 194                         i++;
 195                         continue;
 196                 }
 197
 198                 /* do we have enough in this lump? */
 199                 for (j = 0; (j < needed) && ((i+j) < pile->num_entries); j++) {
 200                         if (pile->list[i+j] & I40E_PILE_VALID_BIT)
 201                                 break;
 202                 }
 203
 204                 if (j == needed) {
 205                         /* there was enough, so assign it to the requestor */
 206                         for (j = 0; j < needed; j++)
 207                                 pile->list[i+j] = id | I40E_PILE_VALID_BIT;
 208                         ret = i;
 209                         pile->search_hint = i + j;
 210                         break;
 211                 } else {
 212                         /* not enough, so skip over it and continue looking */
 213                         i += j;
 214                 }
 215         }
 216
 217         return ret;
 218 }
 219
 220 /**
 221  * i40e_put_lump - return a lump of generic resource
 222  * @pile: the pile of resource to search
 223  * @index: the base item index
 224  * @id: the owner id of the items assigned
 225  *
 226  * Returns the count of items in the lump
 227  **/
 228 static int i40e_put_lump(struct i40e_lump_tracking *pile, u16 index, u16 id)
 229 {
 230         int valid_id = (id | I40E_PILE_VALID_BIT);
 231         int count = 0;
 232         int i;
 233
 234         if (!pile || index >= pile->num_entries)
 235                 return -EINVAL;
 236
 237         for (i = index;
 238              i < pile->num_entries && pile->list[i] == valid_id;
 239              i++) {
 240                 pile->list[i] = 0;
 241                 count++;
 242         }
 243
 244         if (count && index < pile->search_hint)
 245                 pile->search_hint = index;
 246
 247         return count;
 248 }
 249
 250 /**
 251  * i40e_service_event_schedule - Schedule the service task to wake up
 252  * @pf: board private structure
 253  *
 254  * If not already scheduled, this puts the task into the work queue
 255  **/
 256 static void i40e_service_event_schedule(struct i40e_pf *pf)
 257 {
 258         if (!test_bit(__I40E_DOWN, &pf->state) &&
 259             !test_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state) &&
 260             !test_and_set_bit(__I40E_SERVICE_SCHED, &pf->state))
 261                 schedule_work(&pf->service_task);
 262 }
 263
 264 /**
 265  * i40e_tx_timeout - Respond to a Tx Hang
 266  * @netdev: network interface device structure
 267  *
 268  * If any port has noticed a Tx timeout, it is likely that the whole
 269  * device is munged, not just the one netdev port, so go for the full
 270  * reset.
 271  **/
 272 static void i40e_tx_timeout(struct net_device *netdev)
 273 {
 274         struct i40e_netdev_priv *np = netdev_priv(netdev);
 275         struct i40e_vsi *vsi = np->vsi;
 276         struct i40e_pf *pf = vsi->back;
 277
 278         pf->tx_timeout_count++;
 279
 280         if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ*20)))
 281                 pf->tx_timeout_recovery_level = 0;
 282         pf->tx_timeout_last_recovery = jiffies;
 283         netdev_info(netdev, "tx_timeout recovery level %d\n",
 284                     pf->tx_timeout_recovery_level);
 285
 286         switch (pf->tx_timeout_recovery_level) {
 287         case 0:
 288                 /* disable and re-enable queues for the VSI */
 289                 if (in_interrupt()) {
 290                         set_bit(__I40E_REINIT_REQUESTED, &pf->state);
 291                         set_bit(__I40E_REINIT_REQUESTED, &vsi->state);
 292                 } else {
 293                         i40e_vsi_reinit_locked(vsi);
 294                 }
 295                 break;
 296         case 1:
 297                 set_bit(__I40E_PF_RESET_REQUESTED, &pf->state);
 298                 break;
 299         case 2:
 300                 set_bit(__I40E_CORE_RESET_REQUESTED, &pf->state);
 301                 break;
 302         case 3:
 303                 set_bit(__I40E_GLOBAL_RESET_REQUESTED, &pf->state);
 304                 break;
 305         default:
 306                 netdev_err(netdev, "tx_timeout recovery unsuccessful\n");
 307                 i40e_down(vsi);
 308                 break;
 309         }
 310         i40e_service_event_schedule(pf);
 311         pf->tx_timeout_recovery_level++;
 312 }
 313
 314 /**
 315  * i40e_release_rx_desc - Store the new tail and head values
 316  * @rx_ring: ring to bump
 317  * @val: new head index
 318  **/
 319 static inline void i40e_release_rx_desc(struct i40e_ring *rx_ring, u32 val)
 320 {
 321         rx_ring->next_to_use = val;
 322
 323         /* Force memory writes to complete before letting h/w
 324          * know there are new descriptors to fetch.  (Only
 325          * applicable for weak-ordered memory model archs,
 326          * such as IA-64).
 327          */
 328         wmb();
 329         writel(val, rx_ring->tail);
 330 }
 331
 332 /**
 333  * i40e_get_vsi_stats_struct - Get System Network Statistics
 334  * @vsi: the VSI we care about
 335  *
 336  * Returns the address of the device statistics structure.
 337  * The statistics are actually updated from the service task.
 338  **/
 339 struct rtnl_link_stats64 *i40e_get_vsi_stats_struct(struct i40e_vsi *vsi)
 340 {
 341         return &vsi->net_stats;
 342 }
 343
 344 /**
 345  * i40e_get_netdev_stats_struct - Get statistics for netdev interface
 346  * @netdev: network interface device structure
 347  *
 348  * Returns the address of the device statistics structure.
 349  * The statistics are actually updated from the service task.
 350  **/
 351 static struct rtnl_link_stats64 *i40e_get_netdev_stats_struct(
 352                                              struct net_device *netdev,
 353                                              struct rtnl_link_stats64 *stats)
 354 {
 355         struct i40e_netdev_priv *np = netdev_priv(netdev);
 356         struct i40e_vsi *vsi = np->vsi;
 357         struct rtnl_link_stats64 *vsi_stats = i40e_get_vsi_stats_struct(vsi);
 358         int i;
 359
 360
 361         if (test_bit(__I40E_DOWN, &vsi->state))
 362                 return stats;
 363
 364         if (!vsi->tx_rings)
 365                 return stats;
 366
 367         rcu_read_lock();
 368         for (i = 0; i < vsi->num_queue_pairs; i++) {
 369                 struct i40e_ring *tx_ring, *rx_ring;
 370                 u64 bytes, packets;
 371                 unsigned int start;
 372
 373                 tx_ring = ACCESS_ONCE(vsi->tx_rings[i]);
 374                 if (!tx_ring)
 375                         continue;
 376
 377                 do {
 378                         start = u64_stats_fetch_begin_bh(&tx_ring->syncp);
 379                         packets = tx_ring->stats.packets;
 380                         bytes   = tx_ring->stats.bytes;
 381                 } while (u64_stats_fetch_retry_bh(&tx_ring->syncp, start));
 382
 383                 stats->tx_packets += packets;
 384                 stats->tx_bytes   += bytes;
 385                 rx_ring = &tx_ring[1];
 386
 387                 do {
 388                         start = u64_stats_fetch_begin_bh(&rx_ring->syncp);
 389                         packets = rx_ring->stats.packets;
 390                         bytes   = rx_ring->stats.bytes;
 391                 } while (u64_stats_fetch_retry_bh(&rx_ring->syncp, start));
 392
 393                 stats->rx_packets += packets;
 394                 stats->rx_bytes   += bytes;
 395         }
 396         rcu_read_unlock();
 397
 398         /* following stats updated by ixgbe_watchdog_task() */
 399         stats->multicast        = vsi_stats->multicast;
 400         stats->tx_errors        = vsi_stats->tx_errors;
 401         stats->tx_dropped       = vsi_stats->tx_dropped;
 402         stats->rx_errors        = vsi_stats->rx_errors;
 403         stats->rx_crc_errors    = vsi_stats->rx_crc_errors;
 404         stats->rx_length_errors = vsi_stats->rx_length_errors;
 405
 406         return stats;
 407 }
 408
 409 /**
 410  * i40e_vsi_reset_stats - Resets all stats of the given vsi
 411  * @vsi: the VSI to have its stats reset
 412  **/
 413 void i40e_vsi_reset_stats(struct i40e_vsi *vsi)
 414 {
 415         struct rtnl_link_stats64 *ns;
 416         int i;
 417
 418         if (!vsi)
 419                 return;
 420
 421         ns = i40e_get_vsi_stats_struct(vsi);
 422         memset(ns, 0, sizeof(*ns));
 423         memset(&vsi->net_stats_offsets, 0, sizeof(vsi->net_stats_offsets));
 424         memset(&vsi->eth_stats, 0, sizeof(vsi->eth_stats));
 425         memset(&vsi->eth_stats_offsets, 0, sizeof(vsi->eth_stats_offsets));
 426         if (vsi->rx_rings)
 427                 for (i = 0; i < vsi->num_queue_pairs; i++) {
 428                         memset(&vsi->rx_rings[i]->stats, 0 ,
 429                                sizeof(vsi->rx_rings[i]->stats));
 430                         memset(&vsi->rx_rings[i]->rx_stats, 0 ,
 431                                sizeof(vsi->rx_rings[i]->rx_stats));
 432                         memset(&vsi->tx_rings[i]->stats, 0 ,
 433                                sizeof(vsi->tx_rings[i]->stats));
 434                         memset(&vsi->tx_rings[i]->tx_stats, 0,
 435                                sizeof(vsi->tx_rings[i]->tx_stats));
 436                 }
 437         vsi->stat_offsets_loaded = false;
 438 }
 439
 440 /**
 441  * i40e_pf_reset_stats - Reset all of the stats for the given pf
 442  * @pf: the PF to be reset
 443  **/
 444 void i40e_pf_reset_stats(struct i40e_pf *pf)
 445 {
 446         memset(&pf->stats, 0, sizeof(pf->stats));
 447         memset(&pf->stats_offsets, 0, sizeof(pf->stats_offsets));
 448         pf->stat_offsets_loaded = false;
 449 }
 450
 451 /**
 452  * i40e_stat_update48 - read and update a 48 bit stat from the chip
 453  * @hw: ptr to the hardware info
 454  * @hireg: the high 32 bit reg to read
 455  * @loreg: the low 32 bit reg to read
 456  * @offset_loaded: has the initial offset been loaded yet
 457  * @offset: ptr to current offset value
 458  * @stat: ptr to the stat
 459  *
 460  * Since the device stats are not reset at PFReset, they likely will not
 461  * be zeroed when the driver starts.  We'll save the first values read
 462  * and use them as offsets to be subtracted from the raw values in order
 463  * to report stats that count from zero.  In the process, we also manage
 464  * the potential roll-over.
 465  **/
 466 static void i40e_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg,
 467                                bool offset_loaded, u64 *offset, u64 *stat)
 468 {
 469         u64 new_data;
 470
 471         if (hw->device_id == I40E_QEMU_DEVICE_ID) {
 472                 new_data = rd32(hw, loreg);
 473                 new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32;
 474         } else {
 475                 new_data = rd64(hw, loreg);
 476         }
 477         if (!offset_loaded)
 478                 *offset = new_data;
 479         if (likely(new_data >= *offset))
 480                 *stat = new_data - *offset;
 481         else
 482                 *stat = (new_data + ((u64)1 << 48)) - *offset;
 483         *stat &= 0xFFFFFFFFFFFFULL;
 484 }
 485
 486 /**
 487  * i40e_stat_update32 - read and update a 32 bit stat from the chip
 488  * @hw: ptr to the hardware info
 489  * @reg: the hw reg to read
 490  * @offset_loaded: has the initial offset been loaded yet
 491  * @offset: ptr to current offset value
 492  * @stat: ptr to the stat
 493  **/
 494 static void i40e_stat_update32(struct i40e_hw *hw, u32 reg,
 495                                bool offset_loaded, u64 *offset, u64 *stat)
 496 {
 497         u32 new_data;
 498
 499         new_data = rd32(hw, reg);
 500         if (!offset_loaded)
 501                 *offset = new_data;
 502         if (likely(new_data >= *offset))
 503                 *stat = (u32)(new_data - *offset);
 504         else
 505                 *stat = (u32)((new_data + ((u64)1 << 32)) - *offset);
 506 }
 507
 508 /**
 509  * i40e_update_eth_stats - Update VSI-specific ethernet statistics counters.
 510  * @vsi: the VSI to be updated
 511  **/
 512 void i40e_update_eth_stats(struct i40e_vsi *vsi)
 513 {
 514         int stat_idx = le16_to_cpu(vsi->info.stat_counter_idx);
 515         struct i40e_pf *pf = vsi->back;
 516         struct i40e_hw *hw = &pf->hw;
 517         struct i40e_eth_stats *oes;
 518         struct i40e_eth_stats *es;     /* device's eth stats */
 519
 520         es = &vsi->eth_stats;
 521         oes = &vsi->eth_stats_offsets;
 522
 523         /* Gather up the stats that the hw collects */
 524         i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx),
 525                            vsi->stat_offsets_loaded,
 526                            &oes->tx_errors, &es->tx_errors);
 527         i40e_stat_update32(hw, I40E_GLV_RDPC(stat_idx),
 528                            vsi->stat_offsets_loaded,
 529                            &oes->rx_discards, &es->rx_discards);
 530
 531         i40e_stat_update48(hw, I40E_GLV_GORCH(stat_idx),
 532                            I40E_GLV_GORCL(stat_idx),
 533                            vsi->stat_offsets_loaded,
 534                            &oes->rx_bytes, &es->rx_bytes);
 535         i40e_stat_update48(hw, I40E_GLV_UPRCH(stat_idx),
 536                            I40E_GLV_UPRCL(stat_idx),
 537                            vsi->stat_offsets_loaded,
 538                            &oes->rx_unicast, &es->rx_unicast);
 539         i40e_stat_update48(hw, I40E_GLV_MPRCH(stat_idx),
 540                            I40E_GLV_MPRCL(stat_idx),
 541                            vsi->stat_offsets_loaded,
 542                            &oes->rx_multicast, &es->rx_multicast);
 543         i40e_stat_update48(hw, I40E_GLV_BPRCH(stat_idx),
 544                            I40E_GLV_BPRCL(stat_idx),
 545                            vsi->stat_offsets_loaded,
 546                            &oes->rx_broadcast, &es->rx_broadcast);
 547
 548         i40e_stat_update48(hw, I40E_GLV_GOTCH(stat_idx),
 549                            I40E_GLV_GOTCL(stat_idx),
 550                            vsi->stat_offsets_loaded,
 551                            &oes->tx_bytes, &es->tx_bytes);
 552         i40e_stat_update48(hw, I40E_GLV_UPTCH(stat_idx),
 553                            I40E_GLV_UPTCL(stat_idx),
 554                            vsi->stat_offsets_loaded,
 555                            &oes->tx_unicast, &es->tx_unicast);
 556         i40e_stat_update48(hw, I40E_GLV_MPTCH(stat_idx),
 557                            I40E_GLV_MPTCL(stat_idx),
 558                            vsi->stat_offsets_loaded,
 559                            &oes->tx_multicast, &es->tx_multicast);
 560         i40e_stat_update48(hw, I40E_GLV_BPTCH(stat_idx),
 561                            I40E_GLV_BPTCL(stat_idx),
 562                            vsi->stat_offsets_loaded,
 563                            &oes->tx_broadcast, &es->tx_broadcast);
 564         vsi->stat_offsets_loaded = true;
 565 }
 566
 567 /**
 568  * i40e_update_veb_stats - Update Switch component statistics
 569  * @veb: the VEB being updated
 570  **/
 571 static void i40e_update_veb_stats(struct i40e_veb *veb)
 572 {
 573         struct i40e_pf *pf = veb->pf;
 574         struct i40e_hw *hw = &pf->hw;
 575         struct i40e_eth_stats *oes;
 576         struct i40e_eth_stats *es;     /* device's eth stats */
 577         int idx = 0;
 578
 579         idx = veb->stats_idx;
 580         es = &veb->stats;
 581         oes = &veb->stats_offsets;
 582
 583         /* Gather up the stats that the hw collects */
 584         i40e_stat_update32(hw, I40E_GLSW_TDPC(idx),
 585                            veb->stat_offsets_loaded,
 586                            &oes->tx_discards, &es->tx_discards);
 587         if (hw->revision_id > 0)
 588                 i40e_stat_update32(hw, I40E_GLSW_RUPP(idx),
 589                                    veb->stat_offsets_loaded,
 590                                    &oes->rx_unknown_protocol,
 591                                    &es->rx_unknown_protocol);
 592         i40e_stat_update48(hw, I40E_GLSW_GORCH(idx), I40E_GLSW_GORCL(idx),
 593                            veb->stat_offsets_loaded,
 594                            &oes->rx_bytes, &es->rx_bytes);
 595         i40e_stat_update48(hw, I40E_GLSW_UPRCH(idx), I40E_GLSW_UPRCL(idx),
 596                            veb->stat_offsets_loaded,
 597                            &oes->rx_unicast, &es->rx_unicast);
 598         i40e_stat_update48(hw, I40E_GLSW_MPRCH(idx), I40E_GLSW_MPRCL(idx),
 599                            veb->stat_offsets_loaded,
 600                            &oes->rx_multicast, &es->rx_multicast);
 601         i40e_stat_update48(hw, I40E_GLSW_BPRCH(idx), I40E_GLSW_BPRCL(idx),
 602                            veb->stat_offsets_loaded,
 603                            &oes->rx_broadcast, &es->rx_broadcast);
 604
 605         i40e_stat_update48(hw, I40E_GLSW_GOTCH(idx), I40E_GLSW_GOTCL(idx),
 606                            veb->stat_offsets_loaded,
 607                            &oes->tx_bytes, &es->tx_bytes);
 608         i40e_stat_update48(hw, I40E_GLSW_UPTCH(idx), I40E_GLSW_UPTCL(idx),
 609                            veb->stat_offsets_loaded,
 610                            &oes->tx_unicast, &es->tx_unicast);
 611         i40e_stat_update48(hw, I40E_GLSW_MPTCH(idx), I40E_GLSW_MPTCL(idx),
 612                            veb->stat_offsets_loaded,
 613                            &oes->tx_multicast, &es->tx_multicast);
 614         i40e_stat_update48(hw, I40E_GLSW_BPTCH(idx), I40E_GLSW_BPTCL(idx),
 615                            veb->stat_offsets_loaded,
 616                            &oes->tx_broadcast, &es->tx_broadcast);
 617         veb->stat_offsets_loaded = true;
 618 }
 619
 620 /**
 621  * i40e_update_link_xoff_rx - Update XOFF received in link flow control mode
 622  * @pf: the corresponding PF
 623  *
 624  * Update the Rx XOFF counter (PAUSE frames) in link flow control mode
 625  **/
 626 static void i40e_update_link_xoff_rx(struct i40e_pf *pf)
 627 {
 628         struct i40e_hw_port_stats *osd = &pf->stats_offsets;
 629         struct i40e_hw_port_stats *nsd = &pf->stats;
 630         struct i40e_hw *hw = &pf->hw;
 631         u64 xoff = 0;
 632         u16 i, v;
 633
 634         if ((hw->fc.current_mode != I40E_FC_FULL) &&
 635             (hw->fc.current_mode != I40E_FC_RX_PAUSE))
 636                 return;
 637
 638         xoff = nsd->link_xoff_rx;
 639         i40e_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port),
 640                            pf->stat_offsets_loaded,
 641                            &osd->link_xoff_rx, &nsd->link_xoff_rx);
 642
 643         /* No new LFC xoff rx */
 644         if (!(nsd->link_xoff_rx - xoff))
 645                 return;
 646
 647         /* Clear the __I40E_HANG_CHECK_ARMED bit for all Tx rings */
 648         for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
 649                 struct i40e_vsi *vsi = pf->vsi[v];
 650
 651                 if (!vsi)
 652                         continue;
 653
 654                 for (i = 0; i < vsi->num_queue_pairs; i++) {
 655                         struct i40e_ring *ring = vsi->tx_rings[i];
 656                         clear_bit(__I40E_HANG_CHECK_ARMED, &ring->state);
 657                 }
 658         }
 659 }
 660
 661 /**
 662  * i40e_update_prio_xoff_rx - Update XOFF received in PFC mode
 663  * @pf: the corresponding PF
 664  *
 665  * Update the Rx XOFF counter (PAUSE frames) in PFC mode
 666  **/
 667 static void i40e_update_prio_xoff_rx(struct i40e_pf *pf)
 668 {
 669         struct i40e_hw_port_stats *osd = &pf->stats_offsets;
 670         struct i40e_hw_port_stats *nsd = &pf->stats;
 671         bool xoff[I40E_MAX_TRAFFIC_CLASS] = {false};
 672         struct i40e_dcbx_config *dcb_cfg;
 673         struct i40e_hw *hw = &pf->hw;
 674         u16 i, v;
 675         u8 tc;
 676
 677         dcb_cfg = &hw->local_dcbx_config;
 678
 679         /* See if DCB enabled with PFC TC */
 680         if (!(pf->flags & I40E_FLAG_DCB_ENABLED) ||
 681             !(dcb_cfg->pfc.pfcenable)) {
 682                 i40e_update_link_xoff_rx(pf);
 683                 return;
 684         }
 685
 686         for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) {
 687                 u64 prio_xoff = nsd->priority_xoff_rx[i];
 688                 i40e_stat_update32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i),
 689                                    pf->stat_offsets_loaded,
 690                                    &osd->priority_xoff_rx[i],
 691                                    &nsd->priority_xoff_rx[i]);
 692
 693                 /* No new PFC xoff rx */
 694                 if (!(nsd->priority_xoff_rx[i] - prio_xoff))
 695                         continue;
 696                 /* Get the TC for given priority */
 697                 tc = dcb_cfg->etscfg.prioritytable[i];
 698                 xoff[tc] = true;
 699         }
 700
 701         /* Clear the __I40E_HANG_CHECK_ARMED bit for Tx rings */
 702         for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
 703                 struct i40e_vsi *vsi = pf->vsi[v];
 704
 705                 if (!vsi)
 706                         continue;
 707
 708                 for (i = 0; i < vsi->num_queue_pairs; i++) {
 709                         struct i40e_ring *ring = vsi->tx_rings[i];
 710
 711                         tc = ring->dcb_tc;
 712                         if (xoff[tc])
 713                                 clear_bit(__I40E_HANG_CHECK_ARMED,
 714                                           &ring->state);
 715                 }
 716         }
 717 }
 718
 719 /**
 720  * i40e_update_stats - Update the board statistics counters.
 721  * @vsi: the VSI to be updated
 722  *
 723  * There are a few instances where we store the same stat in a
 724  * couple of different structs.  This is partly because we have
 725  * the netdev stats that need to be filled out, which is slightly
 726  * different from the "eth_stats" defined by the chip and used in
 727  * VF communications.  We sort it all out here in a central place.
 728  **/
 729 void i40e_update_stats(struct i40e_vsi *vsi)
 730 {
 731         struct i40e_pf *pf = vsi->back;
 732         struct i40e_hw *hw = &pf->hw;
 733         struct rtnl_link_stats64 *ons;
 734         struct rtnl_link_stats64 *ns;   /* netdev stats */
 735         struct i40e_eth_stats *oes;
 736         struct i40e_eth_stats *es;     /* device's eth stats */
 737         u32 tx_restart, tx_busy;
 738         u32 rx_page, rx_buf;
 739         u64 rx_p, rx_b;
 740         u64 tx_p, tx_b;
 741         int i;
 742         u16 q;
 743
 744         if (test_bit(__I40E_DOWN, &vsi->state) ||
 745             test_bit(__I40E_CONFIG_BUSY, &pf->state))
 746                 return;
 747
 748         ns = i40e_get_vsi_stats_struct(vsi);
 749         ons = &vsi->net_stats_offsets;
 750         es = &vsi->eth_stats;
 751         oes = &vsi->eth_stats_offsets;
 752
 753         /* Gather up the netdev and vsi stats that the driver collects
 754          * on the fly during packet processing
 755          */
 756         rx_b = rx_p = 0;
 757         tx_b = tx_p = 0;
 758         tx_restart = tx_busy = 0;
 759         rx_page = 0;
 760         rx_buf = 0;
 761         rcu_read_lock();
 762         for (q = 0; q < vsi->num_queue_pairs; q++) {
 763                 struct i40e_ring *p;
 764                 u64 bytes, packets;
 765                 unsigned int start;
 766
 767                 /* locate Tx ring */
 768                 p = ACCESS_ONCE(vsi->tx_rings[q]);
 769
 770                 do {
 771                         start = u64_stats_fetch_begin_bh(&p->syncp);
 772                         packets = p->stats.packets;
 773                         bytes = p->stats.bytes;
 774                 } while (u64_stats_fetch_retry_bh(&p->syncp, start));
 775                 tx_b += bytes;
 776                 tx_p += packets;
 777                 tx_restart += p->tx_stats.restart_queue;
 778                 tx_busy += p->tx_stats.tx_busy;
 779
 780                 /* Rx queue is part of the same block as Tx queue */
 781                 p = &p[1];
 782                 do {
 783                         start = u64_stats_fetch_begin_bh(&p->syncp);
 784                         packets = p->stats.packets;
 785                         bytes = p->stats.bytes;
 786                 } while (u64_stats_fetch_retry_bh(&p->syncp, start));
 787                 rx_b += bytes;
 788                 rx_p += packets;
 789                 rx_buf += p->rx_stats.alloc_rx_buff_failed;
 790                 rx_page += p->rx_stats.alloc_rx_page_failed;
 791         }
 792         rcu_read_unlock();
 793         vsi->tx_restart = tx_restart;
 794         vsi->tx_busy = tx_busy;
 795         vsi->rx_page_failed = rx_page;
 796         vsi->rx_buf_failed = rx_buf;
 797
 798         ns->rx_packets = rx_p;
 799         ns->rx_bytes = rx_b;
 800         ns->tx_packets = tx_p;
 801         ns->tx_bytes = tx_b;
 802
 803         i40e_update_eth_stats(vsi);
 804         /* update netdev stats from eth stats */
 805         ons->rx_errors = oes->rx_errors;
 806         ns->rx_errors = es->rx_errors;
 807         ons->tx_errors = oes->tx_errors;
 808         ns->tx_errors = es->tx_errors;
 809         ons->multicast = oes->rx_multicast;
 810         ns->multicast = es->rx_multicast;
 811         ons->tx_dropped = oes->tx_discards;
 812         ns->tx_dropped = es->tx_discards;
 813
 814         /* Get the port data only if this is the main PF VSI */
 815         if (vsi == pf->vsi[pf->lan_vsi]) {
 816                 struct i40e_hw_port_stats *nsd = &pf->stats;
 817                 struct i40e_hw_port_stats *osd = &pf->stats_offsets;
 818
 819                 i40e_stat_update48(hw, I40E_GLPRT_GORCH(hw->port),
 820                                    I40E_GLPRT_GORCL(hw->port),
 821                                    pf->stat_offsets_loaded,
 822                                    &osd->eth.rx_bytes, &nsd->eth.rx_bytes);
 823                 i40e_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port),
 824                                    I40E_GLPRT_GOTCL(hw->port),
 825                                    pf->stat_offsets_loaded,
 826                                    &osd->eth.tx_bytes, &nsd->eth.tx_bytes);
 827                 i40e_stat_update32(hw, I40E_GLPRT_RDPC(hw->port),
 828                                    pf->stat_offsets_loaded,
 829                                    &osd->eth.rx_discards,
 830                                    &nsd->eth.rx_discards);
 831                 i40e_stat_update32(hw, I40E_GLPRT_TDPC(hw->port),
 832                                    pf->stat_offsets_loaded,
 833                                    &osd->eth.tx_discards,
 834                                    &nsd->eth.tx_discards);
 835                 i40e_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port),
 836                                    I40E_GLPRT_MPRCL(hw->port),
 837                                    pf->stat_offsets_loaded,
 838                                    &osd->eth.rx_multicast,
 839                                    &nsd->eth.rx_multicast);
 840
 841                 i40e_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port),
 842                                    pf->stat_offsets_loaded,
 843                                    &osd->tx_dropped_link_down,
 844                                    &nsd->tx_dropped_link_down);
 845
 846                 i40e_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port),
 847                                    pf->stat_offsets_loaded,
 848                                    &osd->crc_errors, &nsd->crc_errors);
 849                 ns->rx_crc_errors = nsd->crc_errors;
 850
 851                 i40e_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port),
 852                                    pf->stat_offsets_loaded,
 853                                    &osd->illegal_bytes, &nsd->illegal_bytes);
 854                 ns->rx_errors = nsd->crc_errors
 855                                 + nsd->illegal_bytes;
 856
 857                 i40e_stat_update32(hw, I40E_GLPRT_MLFC(hw->port),
 858                                    pf->stat_offsets_loaded,
 859                                    &osd->mac_local_faults,
 860                                    &nsd->mac_local_faults);
 861                 i40e_stat_update32(hw, I40E_GLPRT_MRFC(hw->port),
 862                                    pf->stat_offsets_loaded,
 863                                    &osd->mac_remote_faults,
 864                                    &nsd->mac_remote_faults);
 865
 866                 i40e_stat_update32(hw, I40E_GLPRT_RLEC(hw->port),
 867                                    pf->stat_offsets_loaded,
 868                                    &osd->rx_length_errors,
 869                                    &nsd->rx_length_errors);
 870                 ns->rx_length_errors = nsd->rx_length_errors;
 871
 872                 i40e_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port),
 873                                    pf->stat_offsets_loaded,
 874                                    &osd->link_xon_rx, &nsd->link_xon_rx);
 875                 i40e_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port),
 876                                    pf->stat_offsets_loaded,
 877                                    &osd->link_xon_tx, &nsd->link_xon_tx);
 878                 i40e_update_prio_xoff_rx(pf);  /* handles I40E_GLPRT_LXOFFRXC */
 879                 i40e_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port),
 880                                    pf->stat_offsets_loaded,
 881                                    &osd->link_xoff_tx, &nsd->link_xoff_tx);
 882
 883                 for (i = 0; i < 8; i++) {
 884                         i40e_stat_update32(hw, I40E_GLPRT_PXONRXC(hw->port, i),
 885                                            pf->stat_offsets_loaded,
 886                                            &osd->priority_xon_rx[i],
 887                                            &nsd->priority_xon_rx[i]);
 888                         i40e_stat_update32(hw, I40E_GLPRT_PXONTXC(hw->port, i),
 889                                            pf->stat_offsets_loaded,
 890                                            &osd->priority_xon_tx[i],
 891                                            &nsd->priority_xon_tx[i]);
 892                         i40e_stat_update32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i),
 893                                            pf->stat_offsets_loaded,
 894                                            &osd->priority_xoff_tx[i],
 895                                            &nsd->priority_xoff_tx[i]);
 896                         i40e_stat_update32(hw,
 897                                            I40E_GLPRT_RXON2OFFCNT(hw->port, i),
 898                                            pf->stat_offsets_loaded,
 899                                            &osd->priority_xon_2_xoff[i],
 900                                            &nsd->priority_xon_2_xoff[i]);
 901                 }
 902
 903                 i40e_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port),
 904                                    I40E_GLPRT_PRC64L(hw->port),
 905                                    pf->stat_offsets_loaded,
 906                                    &osd->rx_size_64, &nsd->rx_size_64);
 907                 i40e_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port),
 908                                    I40E_GLPRT_PRC127L(hw->port),
 909                                    pf->stat_offsets_loaded,
 910                                    &osd->rx_size_127, &nsd->rx_size_127);
 911                 i40e_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port),
 912                                    I40E_GLPRT_PRC255L(hw->port),
 913                                    pf->stat_offsets_loaded,
 914                                    &osd->rx_size_255, &nsd->rx_size_255);
 915                 i40e_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port),
 916                                    I40E_GLPRT_PRC511L(hw->port),
 917                                    pf->stat_offsets_loaded,
 918                                    &osd->rx_size_511, &nsd->rx_size_511);
 919                 i40e_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port),
 920                                    I40E_GLPRT_PRC1023L(hw->port),
 921                                    pf->stat_offsets_loaded,
 922                                    &osd->rx_size_1023, &nsd->rx_size_1023);
 923                 i40e_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port),
 924                                    I40E_GLPRT_PRC1522L(hw->port),
 925                                    pf->stat_offsets_loaded,
 926                                    &osd->rx_size_1522, &nsd->rx_size_1522);
 927                 i40e_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port),
 928                                    I40E_GLPRT_PRC9522L(hw->port),
 929                                    pf->stat_offsets_loaded,
 930                                    &osd->rx_size_big, &nsd->rx_size_big);
 931
 932                 i40e_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port),
 933                                    I40E_GLPRT_PTC64L(hw->port),
 934                                    pf->stat_offsets_loaded,
 935                                    &osd->tx_size_64, &nsd->tx_size_64);
 936                 i40e_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port),
 937                                    I40E_GLPRT_PTC127L(hw->port),
 938                                    pf->stat_offsets_loaded,
 939                                    &osd->tx_size_127, &nsd->tx_size_127);
 940                 i40e_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port),
 941                                    I40E_GLPRT_PTC255L(hw->port),
 942                                    pf->stat_offsets_loaded,
 943                                    &osd->tx_size_255, &nsd->tx_size_255);
 944                 i40e_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port),
 945                                    I40E_GLPRT_PTC511L(hw->port),
 946                                    pf->stat_offsets_loaded,
 947                                    &osd->tx_size_511, &nsd->tx_size_511);
 948                 i40e_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port),
 949                                    I40E_GLPRT_PTC1023L(hw->port),
 950                                    pf->stat_offsets_loaded,
 951                                    &osd->tx_size_1023, &nsd->tx_size_1023);
 952                 i40e_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port),
 953                                    I40E_GLPRT_PTC1522L(hw->port),
 954                                    pf->stat_offsets_loaded,
 955                                    &osd->tx_size_1522, &nsd->tx_size_1522);
 956                 i40e_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port),
 957                                    I40E_GLPRT_PTC9522L(hw->port),
 958                                    pf->stat_offsets_loaded,
 959                                    &osd->tx_size_big, &nsd->tx_size_big);
 960
 961                 i40e_stat_update32(hw, I40E_GLPRT_RUC(hw->port),
 962                                    pf->stat_offsets_loaded,
 963                                    &osd->rx_undersize, &nsd->rx_undersize);
 964                 i40e_stat_update32(hw, I40E_GLPRT_RFC(hw->port),
 965                                    pf->stat_offsets_loaded,
 966                                    &osd->rx_fragments, &nsd->rx_fragments);
 967                 i40e_stat_update32(hw, I40E_GLPRT_ROC(hw->port),
 968                                    pf->stat_offsets_loaded,
 969                                    &osd->rx_oversize, &nsd->rx_oversize);
 970                 i40e_stat_update32(hw, I40E_GLPRT_RJC(hw->port),
 971                                    pf->stat_offsets_loaded,
 972                                    &osd->rx_jabber, &nsd->rx_jabber);
 973         }
 974
 975         pf->stat_offsets_loaded = true;
 976 }
 977
 978 /**
 979  * i40e_find_filter - Search VSI filter list for specific mac/vlan filter
 980  * @vsi: the VSI to be searched
 981  * @macaddr: the MAC address
 982  * @vlan: the vlan
 983  * @is_vf: make sure its a vf filter, else doesn't matter
 984  * @is_netdev: make sure its a netdev filter, else doesn't matter
 985  *
 986  * Returns ptr to the filter object or NULL
 987  **/
 988 static struct i40e_mac_filter *i40e_find_filter(struct i40e_vsi *vsi,
 989                                                 u8 *macaddr, s16 vlan,
 990                                                 bool is_vf, bool is_netdev)
 991 {
 992         struct i40e_mac_filter *f;
 993
 994         if (!vsi || !macaddr)
 995                 return NULL;
 996
 997         list_for_each_entry(f, &vsi->mac_filter_list, list) {
 998                 if ((ether_addr_equal(macaddr, f->macaddr)) &&
 999                     (vlan == f->vlan)    &&
1000                     (!is_vf || f->is_vf) &&
1001                     (!is_netdev || f->is_netdev))
1002                         return f;
1003         }
1004         return NULL;
1005 }
1006
1007 /**
1008  * i40e_find_mac - Find a mac addr in the macvlan filters list
1009  * @vsi: the VSI to be searched
1010  * @macaddr: the MAC address we are searching for
1011  * @is_vf: make sure its a vf filter, else doesn't matter
1012  * @is_netdev: make sure its a netdev filter, else doesn't matter
1013  *
1014  * Returns the first filter with the provided MAC address or NULL if
1015  * MAC address was not found
1016  **/
1017 struct i40e_mac_filter *i40e_find_mac(struct i40e_vsi *vsi, u8 *macaddr,
1018                                       bool is_vf, bool is_netdev)
1019 {
1020         struct i40e_mac_filter *f;
1021
1022         if (!vsi || !macaddr)
1023                 return NULL;
1024
1025         list_for_each_entry(f, &vsi->mac_filter_list, list) {
1026                 if ((ether_addr_equal(macaddr, f->macaddr)) &&
1027                     (!is_vf || f->is_vf) &&
1028                     (!is_netdev || f->is_netdev))
1029                         return f;
1030         }
1031         return NULL;
1032 }
1033
1034 /**
1035  * i40e_is_vsi_in_vlan - Check if VSI is in vlan mode
1036  * @vsi: the VSI to be searched
1037  *
1038  * Returns true if VSI is in vlan mode or false otherwise
1039  **/
1040 bool i40e_is_vsi_in_vlan(struct i40e_vsi *vsi)
1041 {
1042         struct i40e_mac_filter *f;
1043
1044         /* Only -1 for all the filters denotes not in vlan mode
1045          * so we have to go through all the list in order to make sure
1046          */
1047         list_for_each_entry(f, &vsi->mac_filter_list, list) {
1048                 if (f->vlan >= 0)
1049                         return true;
1050         }
1051
1052         return false;
1053 }
1054
1055 /**
1056  * i40e_put_mac_in_vlan - Make macvlan filters from macaddrs and vlans
1057  * @vsi: the VSI to be searched
1058  * @macaddr: the mac address to be filtered
1059  * @is_vf: true if it is a vf
1060  * @is_netdev: true if it is a netdev
1061  *
1062  * Goes through all the macvlan filters and adds a
1063  * macvlan filter for each unique vlan that already exists
1064  *
1065  * Returns first filter found on success, else NULL
1066  **/
1067 struct i40e_mac_filter *i40e_put_mac_in_vlan(struct i40e_vsi *vsi, u8 *macaddr,
1068                                              bool is_vf, bool is_netdev)
1069 {
1070         struct i40e_mac_filter *f;
1071
1072         list_for_each_entry(f, &vsi->mac_filter_list, list) {
1073                 if (!i40e_find_filter(vsi, macaddr, f->vlan,
1074                                       is_vf, is_netdev)) {
1075                         if (!i40e_add_filter(vsi, macaddr, f->vlan,
1076                                                 is_vf, is_netdev))
1077                                 return NULL;
1078                 }
1079         }
1080
1081         return list_first_entry_or_null(&vsi->mac_filter_list,
1082                                         struct i40e_mac_filter, list);
1083 }
1084
1085 /**
1086  * i40e_add_filter - Add a mac/vlan filter to the VSI
1087  * @vsi: the VSI to be searched
1088  * @macaddr: the MAC address
1089  * @vlan: the vlan
1090  * @is_vf: make sure its a vf filter, else doesn't matter
1091  * @is_netdev: make sure its a netdev filter, else doesn't matter
1092  *
1093  * Returns ptr to the filter object or NULL when no memory available.
1094  **/
1095 struct i40e_mac_filter *i40e_add_filter(struct i40e_vsi *vsi,
1096                                         u8 *macaddr, s16 vlan,
1097                                         bool is_vf, bool is_netdev)
1098 {
1099         struct i40e_mac_filter *f;
1100
1101         if (!vsi || !macaddr)
1102                 return NULL;
1103
1104         f = i40e_find_filter(vsi, macaddr, vlan, is_vf, is_netdev);
1105         if (!f) {
1106                 f = kzalloc(sizeof(*f), GFP_ATOMIC);
1107                 if (!f)
1108                         goto add_filter_out;
1109
1110                 memcpy(f->macaddr, macaddr, ETH_ALEN);
1111                 f->vlan = vlan;
1112                 f->changed = true;
1113
1114                 INIT_LIST_HEAD(&f->list);
1115                 list_add(&f->list, &vsi->mac_filter_list);
1116         }
1117
1118         /* increment counter and add a new flag if needed */
1119         if (is_vf) {
1120                 if (!f->is_vf) {
1121                         f->is_vf = true;
1122                         f->counter++;
1123                 }
1124         } else if (is_netdev) {
1125                 if (!f->is_netdev) {
1126                         f->is_netdev = true;
1127                         f->counter++;
1128                 }
1129         } else {
1130                 f->counter++;
1131         }
1132
1133         /* changed tells sync_filters_subtask to
1134          * push the filter down to the firmware
1135          */
1136         if (f->changed) {
1137                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1138                 vsi->back->flags |= I40E_FLAG_FILTER_SYNC;
1139         }
1140
1141 add_filter_out:
1142         return f;
1143 }
1144
1145 /**
1146  * i40e_del_filter - Remove a mac/vlan filter from the VSI
1147  * @vsi: the VSI to be searched
1148  * @macaddr: the MAC address
1149  * @vlan: the vlan
1150  * @is_vf: make sure it's a vf filter, else doesn't matter
1151  * @is_netdev: make sure it's a netdev filter, else doesn't matter
1152  **/
1153 void i40e_del_filter(struct i40e_vsi *vsi,
1154                      u8 *macaddr, s16 vlan,
1155                      bool is_vf, bool is_netdev)
1156 {
1157         struct i40e_mac_filter *f;
1158
1159         if (!vsi || !macaddr)
1160                 return;
1161
1162         f = i40e_find_filter(vsi, macaddr, vlan, is_vf, is_netdev);
1163         if (!f || f->counter == 0)
1164                 return;
1165
1166         if (is_vf) {
1167                 if (f->is_vf) {
1168                         f->is_vf = false;
1169                         f->counter--;
1170                 }
1171         } else if (is_netdev) {
1172                 if (f->is_netdev) {
1173                         f->is_netdev = false;
1174                         f->counter--;
1175                 }
1176         } else {
1177                 /* make sure we don't remove a filter in use by vf or netdev */
1178                 int min_f = 0;
1179                 min_f += (f->is_vf ? 1 : 0);
1180                 min_f += (f->is_netdev ? 1 : 0);
1181
1182                 if (f->counter > min_f)
1183                         f->counter--;
1184         }
1185
1186         /* counter == 0 tells sync_filters_subtask to
1187          * remove the filter from the firmware's list
1188          */
1189         if (f->counter == 0) {
1190                 f->changed = true;
1191                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1192                 vsi->back->flags |= I40E_FLAG_FILTER_SYNC;
1193         }
1194 }
1195
1196 /**
1197  * i40e_set_mac - NDO callback to set mac address
1198  * @netdev: network interface device structure
1199  * @p: pointer to an address structure
1200  *
1201  * Returns 0 on success, negative on failure
1202  **/
1203 static int i40e_set_mac(struct net_device *netdev, void *p)
1204 {
1205         struct i40e_netdev_priv *np = netdev_priv(netdev);
1206         struct i40e_vsi *vsi = np->vsi;
1207         struct sockaddr *addr = p;
1208         struct i40e_mac_filter *f;
1209
1210         if (!is_valid_ether_addr(addr->sa_data))
1211                 return -EADDRNOTAVAIL;
1212
1213         netdev_info(netdev, "set mac address=%pM\n", addr->sa_data);
1214
1215         if (ether_addr_equal(netdev->dev_addr, addr->sa_data))
1216                 return 0;
1217
1218         if (test_bit(__I40E_DOWN, &vsi->back->state) ||
1219             test_bit(__I40E_RESET_RECOVERY_PENDING, &vsi->back->state))
1220                 return -EADDRNOTAVAIL;
1221
1222         if (vsi->type == I40E_VSI_MAIN) {
1223                 i40e_status ret;
1224                 ret = i40e_aq_mac_address_write(&vsi->back->hw,
1225                                                 I40E_AQC_WRITE_TYPE_LAA_ONLY,
1226                                                 addr->sa_data, NULL);
1227                 if (ret) {
1228                         netdev_info(netdev,
1229                                     "Addr change for Main VSI failed: %d\n",
1230                                     ret);
1231                         return -EADDRNOTAVAIL;
1232                 }
1233
1234                 memcpy(vsi->back->hw.mac.addr, addr->sa_data, netdev->addr_len);
1235         }
1236
1237         /* In order to be sure to not drop any packets, add the new address
1238          * then delete the old one.
1239          */
1240         f = i40e_add_filter(vsi, addr->sa_data, I40E_VLAN_ANY, false, false);
1241         if (!f)
1242                 return -ENOMEM;
1243
1244         i40e_sync_vsi_filters(vsi);
1245         i40e_del_filter(vsi, netdev->dev_addr, I40E_VLAN_ANY, false, false);
1246         i40e_sync_vsi_filters(vsi);
1247
1248         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
1249
1250         return 0;
1251 }
1252
1253 /**
1254  * i40e_vsi_setup_queue_map - Setup a VSI queue map based on enabled_tc
1255  * @vsi: the VSI being setup
1256  * @ctxt: VSI context structure
1257  * @enabled_tc: Enabled TCs bitmap
1258  * @is_add: True if called before Add VSI
1259  *
1260  * Setup VSI queue mapping for enabled traffic classes.
1261  **/
1262 static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi,
1263                                      struct i40e_vsi_context *ctxt,
1264                                      u8 enabled_tc,
1265                                      bool is_add)
1266 {
1267         struct i40e_pf *pf = vsi->back;
1268         u16 sections = 0;
1269         u8 netdev_tc = 0;
1270         u16 numtc = 0;
1271         u16 qcount;
1272         u8 offset;
1273         u16 qmap;
1274         int i;
1275
1276         sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
1277         offset = 0;
1278
1279         if (enabled_tc && (vsi->back->flags & I40E_FLAG_DCB_ENABLED)) {
1280                 /* Find numtc from enabled TC bitmap */
1281                 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1282                         if (enabled_tc & (1 << i)) /* TC is enabled */
1283                                 numtc++;
1284                 }
1285                 if (!numtc) {
1286                         dev_warn(&pf->pdev->dev, "DCB is enabled but no TC enabled, forcing TC0\n");
1287                         numtc = 1;
1288                 }
1289         } else {
1290                 /* At least TC0 is enabled in case of non-DCB case */
1291                 numtc = 1;
1292         }
1293
1294         vsi->tc_config.numtc = numtc;
1295         vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1;
1296
1297         /* Setup queue offset/count for all TCs for given VSI */
1298         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1299                 /* See if the given TC is enabled for the given VSI */
1300                 if (vsi->tc_config.enabled_tc & (1 << i)) { /* TC is enabled */
1301                         int pow, num_qps;
1302
1303                         vsi->tc_config.tc_info[i].qoffset = offset;
1304                         switch (vsi->type) {
1305                         case I40E_VSI_MAIN:
1306                                 if (i == 0)
1307                                         qcount = pf->rss_size;
1308                                 else
1309                                         qcount = pf->num_tc_qps;
1310                                 vsi->tc_config.tc_info[i].qcount = qcount;
1311                                 break;
1312                         case I40E_VSI_FDIR:
1313                         case I40E_VSI_SRIOV:
1314                         case I40E_VSI_VMDQ2:
1315                         default:
1316                                 qcount = vsi->alloc_queue_pairs;
1317                                 vsi->tc_config.tc_info[i].qcount = qcount;
1318                                 WARN_ON(i != 0);
1319                                 break;
1320                         }
1321
1322                         /* find the power-of-2 of the number of queue pairs */
1323                         num_qps = vsi->tc_config.tc_info[i].qcount;
1324                         pow = 0;
1325                         while (num_qps &&
1326                               ((1 << pow) < vsi->tc_config.tc_info[i].qcount)) {
1327                                 pow++;
1328                                 num_qps >>= 1;
1329                         }
1330
1331                         vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++;
1332                         qmap =
1333                             (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
1334                             (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
1335
1336                         offset += vsi->tc_config.tc_info[i].qcount;
1337                 } else {
1338                         /* TC is not enabled so set the offset to
1339                          * default queue and allocate one queue
1340                          * for the given TC.
1341                          */
1342                         vsi->tc_config.tc_info[i].qoffset = 0;
1343                         vsi->tc_config.tc_info[i].qcount = 1;
1344                         vsi->tc_config.tc_info[i].netdev_tc = 0;
1345
1346                         qmap = 0;
1347                 }
1348                 ctxt->info.tc_mapping[i] = cpu_to_le16(qmap);
1349         }
1350
1351         /* Set actual Tx/Rx queue pairs */
1352         vsi->num_queue_pairs = offset;
1353
1354         /* Scheduler section valid can only be set for ADD VSI */
1355         if (is_add) {
1356                 sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
1357
1358                 ctxt->info.up_enable_bits = enabled_tc;
1359         }
1360         if (vsi->type == I40E_VSI_SRIOV) {
1361                 ctxt->info.mapping_flags |=
1362                                      cpu_to_le16(I40E_AQ_VSI_QUE_MAP_NONCONTIG);
1363                 for (i = 0; i < vsi->num_queue_pairs; i++)
1364                         ctxt->info.queue_mapping[i] =
1365                                                cpu_to_le16(vsi->base_queue + i);
1366         } else {
1367                 ctxt->info.mapping_flags |=
1368                                         cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
1369                 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
1370         }
1371         ctxt->info.valid_sections |= cpu_to_le16(sections);
1372 }
1373
1374 /**
1375  * i40e_set_rx_mode - NDO callback to set the netdev filters
1376  * @netdev: network interface device structure
1377  **/
1378 static void i40e_set_rx_mode(struct net_device *netdev)
1379 {
1380         struct i40e_netdev_priv *np = netdev_priv(netdev);
1381         struct i40e_mac_filter *f, *ftmp;
1382         struct i40e_vsi *vsi = np->vsi;
1383         struct netdev_hw_addr *uca;
1384         struct netdev_hw_addr *mca;
1385         struct netdev_hw_addr *ha;
1386
1387         /* add addr if not already in the filter list */
1388         netdev_for_each_uc_addr(uca, netdev) {
1389                 if (!i40e_find_mac(vsi, uca->addr, false, true)) {
1390                         if (i40e_is_vsi_in_vlan(vsi))
1391                                 i40e_put_mac_in_vlan(vsi, uca->addr,
1392                                                      false, true);
1393                         else
1394                                 i40e_add_filter(vsi, uca->addr, I40E_VLAN_ANY,
1395                                                 false, true);
1396                 }
1397         }
1398
1399         netdev_for_each_mc_addr(mca, netdev) {
1400                 if (!i40e_find_mac(vsi, mca->addr, false, true)) {
1401                         if (i40e_is_vsi_in_vlan(vsi))
1402                                 i40e_put_mac_in_vlan(vsi, mca->addr,
1403                                                      false, true);
1404                         else
1405                                 i40e_add_filter(vsi, mca->addr, I40E_VLAN_ANY,
1406                                                 false, true);
1407                 }
1408         }
1409
1410         /* remove filter if not in netdev list */
1411         list_for_each_entry_safe(f, ftmp, &vsi->mac_filter_list, list) {
1412                 bool found = false;
1413
1414                 if (!f->is_netdev)
1415                         continue;
1416
1417                 if (is_multicast_ether_addr(f->macaddr)) {
1418                         netdev_for_each_mc_addr(mca, netdev) {
1419                                 if (ether_addr_equal(mca->addr, f->macaddr)) {
1420                                         found = true;
1421                                         break;
1422                                 }
1423                         }
1424                 } else {
1425                         netdev_for_each_uc_addr(uca, netdev) {
1426                                 if (ether_addr_equal(uca->addr, f->macaddr)) {
1427                                         found = true;
1428                                         break;
1429                                 }
1430                         }
1431
1432                         for_each_dev_addr(netdev, ha) {
1433                                 if (ether_addr_equal(ha->addr, f->macaddr)) {
1434                                         found = true;
1435                                         break;
1436                                 }
1437                         }
1438                 }
1439                 if (!found)
1440                         i40e_del_filter(
1441                            vsi, f->macaddr, I40E_VLAN_ANY, false, true);
1442         }
1443
1444         /* check for other flag changes */
1445         if (vsi->current_netdev_flags != vsi->netdev->flags) {
1446                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1447                 vsi->back->flags |= I40E_FLAG_FILTER_SYNC;
1448         }
1449 }
1450
1451 /**
1452  * i40e_sync_vsi_filters - Update the VSI filter list to the HW
1453  * @vsi: ptr to the VSI
1454  *
1455  * Push any outstanding VSI filter changes through the AdminQ.
1456  *
1457  * Returns 0 or error value
1458  **/
1459 int i40e_sync_vsi_filters(struct i40e_vsi *vsi)
1460 {
1461         struct i40e_mac_filter *f, *ftmp;
1462         bool promisc_forced_on = false;
1463         bool add_happened = false;
1464         int filter_list_len = 0;
1465         u32 changed_flags = 0;
1466         i40e_status aq_ret = 0;
1467         struct i40e_pf *pf;
1468         int num_add = 0;
1469         int num_del = 0;
1470         u16 cmd_flags;
1471
1472         /* empty array typed pointers, kcalloc later */
1473         struct i40e_aqc_add_macvlan_element_data *add_list;
1474         struct i40e_aqc_remove_macvlan_element_data *del_list;
1475
1476         while (test_and_set_bit(__I40E_CONFIG_BUSY, &vsi->state))
1477                 usleep_range(1000, 2000);
1478         pf = vsi->back;
1479
1480         if (vsi->netdev) {
1481                 changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags;
1482                 vsi->current_netdev_flags = vsi->netdev->flags;
1483         }
1484
1485         if (vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) {
1486                 vsi->flags &= ~I40E_VSI_FLAG_FILTER_CHANGED;
1487
1488                 filter_list_len = pf->hw.aq.asq_buf_size /
1489                             sizeof(struct i40e_aqc_remove_macvlan_element_data);
1490                 del_list = kcalloc(filter_list_len,
1491                             sizeof(struct i40e_aqc_remove_macvlan_element_data),
1492                             GFP_KERNEL);
1493                 if (!del_list)
1494                         return -ENOMEM;
1495
1496                 list_for_each_entry_safe(f, ftmp, &vsi->mac_filter_list, list) {
1497                         if (!f->changed)
1498                                 continue;
1499
1500                         if (f->counter != 0)
1501                                 continue;
1502                         f->changed = false;
1503                         cmd_flags = 0;
1504
1505                         /* add to delete list */
1506                         memcpy(del_list[num_del].mac_addr,
1507                                f->macaddr, ETH_ALEN);
1508                         del_list[num_del].vlan_tag =
1509                                 cpu_to_le16((u16)(f->vlan ==
1510                                             I40E_VLAN_ANY ? 0 : f->vlan));
1511
1512                         cmd_flags |= I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
1513                         del_list[num_del].flags = cmd_flags;
1514                         num_del++;
1515
1516                         /* unlink from filter list */
1517                         list_del(&f->list);
1518                         kfree(f);
1519
1520                         /* flush a full buffer */
1521                         if (num_del == filter_list_len) {
1522                                 aq_ret = i40e_aq_remove_macvlan(&pf->hw,
1523                                             vsi->seid, del_list, num_del,
1524                                             NULL);
1525                                 num_del = 0;
1526                                 memset(del_list, 0, sizeof(*del_list));
1527
1528                                 if (aq_ret)
1529                                         dev_info(&pf->pdev->dev,
1530                                                  "ignoring delete macvlan error, err %d, aq_err %d while flushing a full buffer\n",
1531                                                  aq_ret,
1532                                                  pf->hw.aq.asq_last_status);
1533                         }
1534                 }
1535                 if (num_del) {
1536                         aq_ret = i40e_aq_remove_macvlan(&pf->hw, vsi->seid,
1537                                                      del_list, num_del, NULL);
1538                         num_del = 0;
1539
1540                         if (aq_ret)
1541                                 dev_info(&pf->pdev->dev,
1542                                          "ignoring delete macvlan error, err %d, aq_err %d\n",
1543                                          aq_ret, pf->hw.aq.asq_last_status);
1544                 }
1545
1546                 kfree(del_list);
1547                 del_list = NULL;
1548
1549                 /* do all the adds now */
1550                 filter_list_len = pf->hw.aq.asq_buf_size /
1551                                sizeof(struct i40e_aqc_add_macvlan_element_data),
1552                 add_list = kcalloc(filter_list_len,
1553                                sizeof(struct i40e_aqc_add_macvlan_element_data),
1554                                GFP_KERNEL);
1555                 if (!add_list)
1556                         return -ENOMEM;
1557
1558                 list_for_each_entry_safe(f, ftmp, &vsi->mac_filter_list, list) {
1559                         if (!f->changed)
1560                                 continue;
1561
1562                         if (f->counter == 0)
1563                                 continue;
1564                         f->changed = false;
1565                         add_happened = true;
1566                         cmd_flags = 0;
1567
1568                         /* add to add array */
1569                         memcpy(add_list[num_add].mac_addr,
1570                                f->macaddr, ETH_ALEN);
1571                         add_list[num_add].vlan_tag =
1572                                 cpu_to_le16(
1573                                  (u16)(f->vlan == I40E_VLAN_ANY ? 0 : f->vlan));
1574                         add_list[num_add].queue_number = 0;
1575
1576                         cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
1577                         add_list[num_add].flags = cpu_to_le16(cmd_flags);
1578                         num_add++;
1579
1580                         /* flush a full buffer */
1581                         if (num_add == filter_list_len) {
1582                                 aq_ret = i40e_aq_add_macvlan(&pf->hw, vsi->seid,
1583                                                              add_list, num_add,
1584                                                              NULL);
1585                                 num_add = 0;
1586
1587                                 if (aq_ret)
1588                                         break;
1589                                 memset(add_list, 0, sizeof(*add_list));
1590                         }
1591                 }
1592                 if (num_add) {
1593                         aq_ret = i40e_aq_add_macvlan(&pf->hw, vsi->seid,
1594                                                      add_list, num_add, NULL);
1595                         num_add = 0;
1596                 }
1597                 kfree(add_list);
1598                 add_list = NULL;
1599
1600                 if (add_happened && (!aq_ret)) {
1601                         /* do nothing */;
1602                 } else if (add_happened && (aq_ret)) {
1603                         dev_info(&pf->pdev->dev,
1604                                  "add filter failed, err %d, aq_err %d\n",
1605                                  aq_ret, pf->hw.aq.asq_last_status);
1606                         if ((pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOSPC) &&
1607                             !test_bit(__I40E_FILTER_OVERFLOW_PROMISC,
1608                                       &vsi->state)) {
1609                                 promisc_forced_on = true;
1610                                 set_bit(__I40E_FILTER_OVERFLOW_PROMISC,
1611                                         &vsi->state);
1612                                 dev_info(&pf->pdev->dev, "promiscuous mode forced on\n");
1613                         }
1614                 }
1615         }
1616
1617         /* check for changes in promiscuous modes */
1618         if (changed_flags & IFF_ALLMULTI) {
1619                 bool cur_multipromisc;
1620                 cur_multipromisc = !!(vsi->current_netdev_flags & IFF_ALLMULTI);
1621                 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(&vsi->back->hw,
1622                                                                vsi->seid,
1623                                                                cur_multipromisc,
1624                                                                NULL);
1625                 if (aq_ret)
1626                         dev_info(&pf->pdev->dev,
1627                                  "set multi promisc failed, err %d, aq_err %d\n",
1628                                  aq_ret, pf->hw.aq.asq_last_status);
1629         }
1630         if ((changed_flags & IFF_PROMISC) || promisc_forced_on) {
1631                 bool cur_promisc;
1632                 cur_promisc = (!!(vsi->current_netdev_flags & IFF_PROMISC) ||
1633                                test_bit(__I40E_FILTER_OVERFLOW_PROMISC,
1634                                         &vsi->state));
1635                 aq_ret = i40e_aq_set_vsi_unicast_promiscuous(&vsi->back->hw,
1636                                                              vsi->seid,
1637                                                              cur_promisc, NULL);
1638                 if (aq_ret)
1639                         dev_info(&pf->pdev->dev,
1640                                  "set uni promisc failed, err %d, aq_err %d\n",
1641                                  aq_ret, pf->hw.aq.asq_last_status);
1642                 aq_ret = i40e_aq_set_vsi_broadcast(&vsi->back->hw,
1643                                                    vsi->seid,
1644                                                    cur_promisc, NULL);
1645                 if (aq_ret)
1646                         dev_info(&pf->pdev->dev,
1647                                  "set brdcast promisc failed, err %d, aq_err %d\n",
1648                                  aq_ret, pf->hw.aq.asq_last_status);
1649         }
1650
1651         clear_bit(__I40E_CONFIG_BUSY, &vsi->state);
1652         return 0;
1653 }
1654
1655 /**
1656  * i40e_sync_filters_subtask - Sync the VSI filter list with HW
1657  * @pf: board private structure
1658  **/
1659 static void i40e_sync_filters_subtask(struct i40e_pf *pf)
1660 {
1661         int v;
1662
1663         if (!pf || !(pf->flags & I40E_FLAG_FILTER_SYNC))
1664                 return;
1665         pf->flags &= ~I40E_FLAG_FILTER_SYNC;
1666
1667         for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
1668                 if (pf->vsi[v] &&
1669                     (pf->vsi[v]->flags & I40E_VSI_FLAG_FILTER_CHANGED))
1670                         i40e_sync_vsi_filters(pf->vsi[v]);
1671         }
1672 }
1673
1674 /**
1675  * i40e_change_mtu - NDO callback to change the Maximum Transfer Unit
1676  * @netdev: network interface device structure
1677  * @new_mtu: new value for maximum frame size
1678  *
1679  * Returns 0 on success, negative on failure
1680  **/
1681 static int i40e_change_mtu(struct net_device *netdev, int new_mtu)
1682 {
1683         struct i40e_netdev_priv *np = netdev_priv(netdev);
1684         int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
1685         struct i40e_vsi *vsi = np->vsi;
1686
1687         /* MTU < 68 is an error and causes problems on some kernels */
1688         if ((new_mtu < 68) || (max_frame > I40E_MAX_RXBUFFER))
1689                 return -EINVAL;
1690
1691         netdev_info(netdev, "changing MTU from %d to %d\n",
1692                     netdev->mtu, new_mtu);
1693         netdev->mtu = new_mtu;
1694         if (netif_running(netdev))
1695                 i40e_vsi_reinit_locked(vsi);
1696
1697         return 0;
1698 }
1699
1700 /**
1701  * i40e_vlan_stripping_enable - Turn on vlan stripping for the VSI
1702  * @vsi: the vsi being adjusted
1703  **/
1704 void i40e_vlan_stripping_enable(struct i40e_vsi *vsi)
1705 {
1706         struct i40e_vsi_context ctxt;
1707         i40e_status ret;
1708
1709         if ((vsi->info.valid_sections &
1710              cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
1711             ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_MODE_MASK) == 0))
1712                 return;  /* already enabled */
1713
1714         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
1715         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
1716                                     I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
1717
1718         ctxt.seid = vsi->seid;
1719         memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info));
1720         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
1721         if (ret) {
1722                 dev_info(&vsi->back->pdev->dev,
1723                          "%s: update vsi failed, aq_err=%d\n",
1724                          __func__, vsi->back->hw.aq.asq_last_status);
1725         }
1726 }
1727
1728 /**
1729  * i40e_vlan_stripping_disable - Turn off vlan stripping for the VSI
1730  * @vsi: the vsi being adjusted
1731  **/
1732 void i40e_vlan_stripping_disable(struct i40e_vsi *vsi)
1733 {
1734         struct i40e_vsi_context ctxt;
1735         i40e_status ret;
1736
1737         if ((vsi->info.valid_sections &
1738              cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
1739             ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_EMOD_MASK) ==
1740              I40E_AQ_VSI_PVLAN_EMOD_MASK))
1741                 return;  /* already disabled */
1742
1743         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
1744         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
1745                                     I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
1746
1747         ctxt.seid = vsi->seid;
1748         memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info));
1749         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
1750         if (ret) {
1751                 dev_info(&vsi->back->pdev->dev,
1752                          "%s: update vsi failed, aq_err=%d\n",
1753                          __func__, vsi->back->hw.aq.asq_last_status);
1754         }
1755 }
1756
1757 /**
1758  * i40e_vlan_rx_register - Setup or shutdown vlan offload
1759  * @netdev: network interface to be adjusted
1760  * @features: netdev features to test if VLAN offload is enabled or not
1761  **/
1762 static void i40e_vlan_rx_register(struct net_device *netdev, u32 features)
1763 {
1764         struct i40e_netdev_priv *np = netdev_priv(netdev);
1765         struct i40e_vsi *vsi = np->vsi;
1766
1767         if (features & NETIF_F_HW_VLAN_CTAG_RX)
1768                 i40e_vlan_stripping_enable(vsi);
1769         else
1770                 i40e_vlan_stripping_disable(vsi);
1771 }
1772
1773 /**
1774  * i40e_vsi_add_vlan - Add vsi membership for given vlan
1775  * @vsi: the vsi being configured
1776  * @vid: vlan id to be added (0 = untagged only , -1 = any)
1777  **/
1778 int i40e_vsi_add_vlan(struct i40e_vsi *vsi, s16 vid)
1779 {
1780         struct i40e_mac_filter *f, *add_f;
1781         bool is_netdev, is_vf;
1782
1783         is_vf = (vsi->type == I40E_VSI_SRIOV);
1784         is_netdev = !!(vsi->netdev);
1785
1786         if (is_netdev) {
1787                 add_f = i40e_add_filter(vsi, vsi->netdev->dev_addr, vid,
1788                                         is_vf, is_netdev);
1789                 if (!add_f) {
1790                         dev_info(&vsi->back->pdev->dev,
1791                                  "Could not add vlan filter %d for %pM\n",
1792                                  vid, vsi->netdev->dev_addr);
1793                         return -ENOMEM;
1794                 }
1795         }
1796
1797         list_for_each_entry(f, &vsi->mac_filter_list, list) {
1798                 add_f = i40e_add_filter(vsi, f->macaddr, vid, is_vf, is_netdev);
1799                 if (!add_f) {
1800                         dev_info(&vsi->back->pdev->dev,
1801                                  "Could not add vlan filter %d for %pM\n",
1802                                  vid, f->macaddr);
1803                         return -ENOMEM;
1804                 }
1805         }
1806
1807         /* Now if we add a vlan tag, make sure to check if it is the first
1808          * tag (i.e. a "tag" -1 does exist) and if so replace the -1 "tag"
1809          * with 0, so we now accept untagged and specified tagged traffic
1810          * (and not any taged and untagged)
1811          */
1812         if (vid > 0) {
1813                 if (is_netdev && i40e_find_filter(vsi, vsi->netdev->dev_addr,
1814                                                   I40E_VLAN_ANY,
1815                                                   is_vf, is_netdev)) {
1816                         i40e_del_filter(vsi, vsi->netdev->dev_addr,
1817                                         I40E_VLAN_ANY, is_vf, is_netdev);
1818                         add_f = i40e_add_filter(vsi, vsi->netdev->dev_addr, 0,
1819                                                 is_vf, is_netdev);
1820                         if (!add_f) {
1821                                 dev_info(&vsi->back->pdev->dev,
1822                                          "Could not add filter 0 for %pM\n",
1823                                          vsi->netdev->dev_addr);
1824                                 return -ENOMEM;
1825                         }
1826                 }
1827
1828                 list_for_each_entry(f, &vsi->mac_filter_list, list) {
1829                         if (i40e_find_filter(vsi, f->macaddr, I40E_VLAN_ANY,
1830                                              is_vf, is_netdev)) {
1831                                 i40e_del_filter(vsi, f->macaddr, I40E_VLAN_ANY,
1832                                                 is_vf, is_netdev);
1833                                 add_f = i40e_add_filter(vsi, f->macaddr,
1834                                                         0, is_vf, is_netdev);
1835                                 if (!add_f) {
1836                                         dev_info(&vsi->back->pdev->dev,
1837                                                  "Could not add filter 0 for %pM\n",
1838                                                  f->macaddr);
1839                                         return -ENOMEM;
1840                                 }
1841                         }
1842                 }
1843         }
1844
1845         if (test_bit(__I40E_DOWN, &vsi->back->state) ||
1846             test_bit(__I40E_RESET_RECOVERY_PENDING, &vsi->back->state))
1847                 return 0;
1848
1849         return i40e_sync_vsi_filters(vsi);
1850 }
1851
1852 /**
1853  * i40e_vsi_kill_vlan - Remove vsi membership for given vlan
1854  * @vsi: the vsi being configured
1855  * @vid: vlan id to be removed (0 = untagged only , -1 = any)
1856  *
1857  * Return: 0 on success or negative otherwise
1858  **/
1859 int i40e_vsi_kill_vlan(struct i40e_vsi *vsi, s16 vid)
1860 {
1861         struct net_device *netdev = vsi->netdev;
1862         struct i40e_mac_filter *f, *add_f;
1863         bool is_vf, is_netdev;
1864         int filter_count = 0;
1865
1866         is_vf = (vsi->type == I40E_VSI_SRIOV);
1867         is_netdev = !!(netdev);
1868
1869         if (is_netdev)
1870                 i40e_del_filter(vsi, netdev->dev_addr, vid, is_vf, is_netdev);
1871
1872         list_for_each_entry(f, &vsi->mac_filter_list, list)
1873                 i40e_del_filter(vsi, f->macaddr, vid, is_vf, is_netdev);
1874
1875         /* go through all the filters for this VSI and if there is only
1876          * vid == 0 it means there are no other filters, so vid 0 must
1877          * be replaced with -1. This signifies that we should from now
1878          * on accept any traffic (with any tag present, or untagged)
1879          */
1880         list_for_each_entry(f, &vsi->mac_filter_list, list) {
1881                 if (is_netdev) {
1882                         if (f->vlan &&
1883                             ether_addr_equal(netdev->dev_addr, f->macaddr))
1884                                 filter_count++;
1885                 }
1886
1887                 if (f->vlan)
1888                         filter_count++;
1889         }
1890
1891         if (!filter_count && is_netdev) {
1892                 i40e_del_filter(vsi, netdev->dev_addr, 0, is_vf, is_netdev);
1893                 f = i40e_add_filter(vsi, netdev->dev_addr, I40E_VLAN_ANY,
1894                                     is_vf, is_netdev);
1895                 if (!f) {
1896                         dev_info(&vsi->back->pdev->dev,
1897                                  "Could not add filter %d for %pM\n",
1898                                  I40E_VLAN_ANY, netdev->dev_addr);
1899                         return -ENOMEM;
1900                 }
1901         }
1902
1903         if (!filter_count) {
1904                 list_for_each_entry(f, &vsi->mac_filter_list, list) {
1905                         i40e_del_filter(vsi, f->macaddr, 0, is_vf, is_netdev);
1906                         add_f = i40e_add_filter(vsi, f->macaddr, I40E_VLAN_ANY,
1907                                             is_vf, is_netdev);
1908                         if (!add_f) {
1909                                 dev_info(&vsi->back->pdev->dev,
1910                                          "Could not add filter %d for %pM\n",
1911                                          I40E_VLAN_ANY, f->macaddr);
1912                                 return -ENOMEM;
1913                         }
1914                 }
1915         }
1916
1917         if (test_bit(__I40E_DOWN, &vsi->back->state) ||
1918             test_bit(__I40E_RESET_RECOVERY_PENDING, &vsi->back->state))
1919                 return 0;
1920
1921         return i40e_sync_vsi_filters(vsi);
1922 }
1923
1924 /**
1925  * i40e_vlan_rx_add_vid - Add a vlan id filter to HW offload
1926  * @netdev: network interface to be adjusted
1927  * @vid: vlan id to be added
1928  *
1929  * net_device_ops implementation for adding vlan ids
1930  **/
1931 static int i40e_vlan_rx_add_vid(struct net_device *netdev,
1932                                 __always_unused __be16 proto, u16 vid)
1933 {
1934         struct i40e_netdev_priv *np = netdev_priv(netdev);
1935         struct i40e_vsi *vsi = np->vsi;
1936         int ret = 0;
1937
1938         if (vid > 4095)
1939                 return -EINVAL;
1940
1941         netdev_info(netdev, "adding %pM vid=%d\n", netdev->dev_addr, vid);
1942
1943         /* If the network stack called us with vid = 0, we should
1944          * indicate to i40e_vsi_add_vlan() that we want to receive
1945          * any traffic (i.e. with any vlan tag, or untagged)
1946          */
1947         ret = i40e_vsi_add_vlan(vsi, vid ? vid : I40E_VLAN_ANY);
1948
1949         if (!ret && (vid < VLAN_N_VID))
1950                 set_bit(vid, vsi->active_vlans);
1951
1952         return ret;
1953 }
1954
1955 /**
1956  * i40e_vlan_rx_kill_vid - Remove a vlan id filter from HW offload
1957  * @netdev: network interface to be adjusted
1958  * @vid: vlan id to be removed
1959  *
1960  * net_device_ops implementation for adding vlan ids
1961  **/
1962 static int i40e_vlan_rx_kill_vid(struct net_device *netdev,
1963                                  __always_unused __be16 proto, u16 vid)
1964 {
1965         struct i40e_netdev_priv *np = netdev_priv(netdev);
1966         struct i40e_vsi *vsi = np->vsi;
1967
1968         netdev_info(netdev, "removing %pM vid=%d\n", netdev->dev_addr, vid);
1969
1970         /* return code is ignored as there is nothing a user
1971          * can do about failure to remove and a log message was
1972          * already printed from the other function
1973          */
1974         i40e_vsi_kill_vlan(vsi, vid);
1975
1976         clear_bit(vid, vsi->active_vlans);
1977
1978         return 0;
1979 }
1980
1981 /**
1982  * i40e_restore_vlan - Reinstate vlans when vsi/netdev comes back up
1983  * @vsi: the vsi being brought back up
1984  **/
1985 static void i40e_restore_vlan(struct i40e_vsi *vsi)
1986 {
1987         u16 vid;
1988
1989         if (!vsi->netdev)
1990                 return;
1991
1992         i40e_vlan_rx_register(vsi->netdev, vsi->netdev->features);
1993
1994         for_each_set_bit(vid, vsi->active_vlans, VLAN_N_VID)
1995                 i40e_vlan_rx_add_vid(vsi->netdev, htons(ETH_P_8021Q),
1996                                      vid);
1997 }
1998
1999 /**
2000  * i40e_vsi_add_pvid - Add pvid for the VSI
2001  * @vsi: the vsi being adjusted
2002  * @vid: the vlan id to set as a PVID
2003  **/
2004 int i40e_vsi_add_pvid(struct i40e_vsi *vsi, u16 vid)
2005 {
2006         struct i40e_vsi_context ctxt;
2007         i40e_status aq_ret;
2008
2009         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
2010         vsi->info.pvid = cpu_to_le16(vid);
2011         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_TAGGED |
2012                                     I40E_AQ_VSI_PVLAN_INSERT_PVID |
2013                                     I40E_AQ_VSI_PVLAN_EMOD_STR;
2014
2015         ctxt.seid = vsi->seid;
2016         memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info));
2017         aq_ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
2018         if (aq_ret) {
2019                 dev_info(&vsi->back->pdev->dev,
2020                          "%s: update vsi failed, aq_err=%d\n",
2021                          __func__, vsi->back->hw.aq.asq_last_status);
2022                 return -ENOENT;
2023         }
2024
2025         return 0;
2026 }
2027
2028 /**
2029  * i40e_vsi_remove_pvid - Remove the pvid from the VSI
2030  * @vsi: the vsi being adjusted
2031  *
2032  * Just use the vlan_rx_register() service to put it back to normal
2033  **/
2034 void i40e_vsi_remove_pvid(struct i40e_vsi *vsi)
2035 {
2036         i40e_vlan_stripping_disable(vsi);
2037
2038         vsi->info.pvid = 0;
2039 }
2040
2041 /**
2042  * i40e_vsi_setup_tx_resources - Allocate VSI Tx queue resources
2043  * @vsi: ptr to the VSI
2044  *
2045  * If this function returns with an error, then it's possible one or
2046  * more of the rings is populated (while the rest are not).  It is the
2047  * callers duty to clean those orphaned rings.
2048  *
2049  * Return 0 on success, negative on failure
2050  **/
2051 static int i40e_vsi_setup_tx_resources(struct i40e_vsi *vsi)
2052 {
2053         int i, err = 0;
2054
2055         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
2056                 err = i40e_setup_tx_descriptors(vsi->tx_rings[i]);
2057
2058         return err;
2059 }
2060
2061 /**
2062  * i40e_vsi_free_tx_resources - Free Tx resources for VSI queues
2063  * @vsi: ptr to the VSI
2064  *
2065  * Free VSI's transmit software resources
2066  **/
2067 static void i40e_vsi_free_tx_resources(struct i40e_vsi *vsi)
2068 {
2069         int i;
2070
2071         for (i = 0; i < vsi->num_queue_pairs; i++)
2072                 if (vsi->tx_rings[i]->desc)
2073                         i40e_free_tx_resources(vsi->tx_rings[i]);
2074 }
2075
2076 /**
2077  * i40e_vsi_setup_rx_resources - Allocate VSI queues Rx resources
2078  * @vsi: ptr to the VSI
2079  *
2080  * If this function returns with an error, then it's possible one or
2081  * more of the rings is populated (while the rest are not).  It is the
2082  * callers duty to clean those orphaned rings.
2083  *
2084  * Return 0 on success, negative on failure
2085  **/
2086 static int i40e_vsi_setup_rx_resources(struct i40e_vsi *vsi)
2087 {
2088         int i, err = 0;
2089
2090         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
2091                 err = i40e_setup_rx_descriptors(vsi->rx_rings[i]);
2092         return err;
2093 }
2094
2095 /**
2096  * i40e_vsi_free_rx_resources - Free Rx Resources for VSI queues
2097  * @vsi: ptr to the VSI
2098  *
2099  * Free all receive software resources
2100  **/
2101 static void i40e_vsi_free_rx_resources(struct i40e_vsi *vsi)
2102 {
2103         int i;
2104
2105         for (i = 0; i < vsi->num_queue_pairs; i++)
2106                 if (vsi->rx_rings[i]->desc)
2107                         i40e_free_rx_resources(vsi->rx_rings[i]);
2108 }
2109
2110 /**
2111  * i40e_configure_tx_ring - Configure a transmit ring context and rest
2112  * @ring: The Tx ring to configure
2113  *
2114  * Configure the Tx descriptor ring in the HMC context.
2115  **/
2116 static int i40e_configure_tx_ring(struct i40e_ring *ring)
2117 {
2118         struct i40e_vsi *vsi = ring->vsi;
2119         u16 pf_q = vsi->base_queue + ring->queue_index;
2120         struct i40e_hw *hw = &vsi->back->hw;
2121         struct i40e_hmc_obj_txq tx_ctx;
2122         i40e_status err = 0;
2123         u32 qtx_ctl = 0;
2124
2125         /* some ATR related tx ring init */
2126         if (vsi->back->flags & I40E_FLAG_FDIR_ATR_ENABLED) {
2127                 ring->atr_sample_rate = vsi->back->atr_sample_rate;
2128                 ring->atr_count = 0;
2129         } else {
2130                 ring->atr_sample_rate = 0;
2131         }
2132
2133         /* initialize XPS */
2134         if (ring->q_vector && ring->netdev &&
2135             !test_and_set_bit(__I40E_TX_XPS_INIT_DONE, &ring->state))
2136                 netif_set_xps_queue(ring->netdev,
2137                                     &ring->q_vector->affinity_mask,
2138                                     ring->queue_index);
2139
2140         /* clear the context structure first */
2141         memset(&tx_ctx, 0, sizeof(tx_ctx));
2142
2143         tx_ctx.new_context = 1;
2144         tx_ctx.base = (ring->dma / 128);
2145         tx_ctx.qlen = ring->count;
2146         tx_ctx.fd_ena = !!(vsi->back->flags & (I40E_FLAG_FDIR_ENABLED |
2147                         I40E_FLAG_FDIR_ATR_ENABLED));
2148
2149         /* As part of VSI creation/update, FW allocates certain
2150          * Tx arbitration queue sets for each TC enabled for
2151          * the VSI. The FW returns the handles to these queue
2152          * sets as part of the response buffer to Add VSI,
2153          * Update VSI, etc. AQ commands. It is expected that
2154          * these queue set handles be associated with the Tx
2155          * queues by the driver as part of the TX queue context
2156          * initialization. This has to be done regardless of
2157          * DCB as by default everything is mapped to TC0.
2158          */
2159         tx_ctx.rdylist = le16_to_cpu(vsi->info.qs_handle[ring->dcb_tc]);
2160         tx_ctx.rdylist_act = 0;
2161
2162         /* clear the context in the HMC */
2163         err = i40e_clear_lan_tx_queue_context(hw, pf_q);
2164         if (err) {
2165                 dev_info(&vsi->back->pdev->dev,
2166                          "Failed to clear LAN Tx queue context on Tx ring %d (pf_q %d), error: %d\n",
2167                          ring->queue_index, pf_q, err);
2168                 return -ENOMEM;
2169         }
2170
2171         /* set the context in the HMC */
2172         err = i40e_set_lan_tx_queue_context(hw, pf_q, &tx_ctx);
2173         if (err) {
2174                 dev_info(&vsi->back->pdev->dev,
2175                          "Failed to set LAN Tx queue context on Tx ring %d (pf_q %d, error: %d\n",
2176                          ring->queue_index, pf_q, err);
2177                 return -ENOMEM;
2178         }
2179
2180         /* Now associate this queue with this PCI function */
2181         qtx_ctl = I40E_QTX_CTL_PF_QUEUE;
2182         qtx_ctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) &
2183                     I40E_QTX_CTL_PF_INDX_MASK);
2184         wr32(hw, I40E_QTX_CTL(pf_q), qtx_ctl);
2185         i40e_flush(hw);
2186
2187         clear_bit(__I40E_HANG_CHECK_ARMED, &ring->state);
2188
2189         /* cache tail off for easier writes later */
2190         ring->tail = hw->hw_addr + I40E_QTX_TAIL(pf_q);
2191
2192         return 0;
2193 }
2194
2195 /**
2196  * i40e_configure_rx_ring - Configure a receive ring context
2197  * @ring: The Rx ring to configure
2198  *
2199  * Configure the Rx descriptor ring in the HMC context.
2200  **/
2201 static int i40e_configure_rx_ring(struct i40e_ring *ring)
2202 {
2203         struct i40e_vsi *vsi = ring->vsi;
2204         u32 chain_len = vsi->back->hw.func_caps.rx_buf_chain_len;
2205         u16 pf_q = vsi->base_queue + ring->queue_index;
2206         struct i40e_hw *hw = &vsi->back->hw;
2207         struct i40e_hmc_obj_rxq rx_ctx;
2208         i40e_status err = 0;
2209
2210         ring->state = 0;
2211
2212         /* clear the context structure first */
2213         memset(&rx_ctx, 0, sizeof(rx_ctx));
2214
2215         ring->rx_buf_len = vsi->rx_buf_len;
2216         ring->rx_hdr_len = vsi->rx_hdr_len;
2217
2218         rx_ctx.dbuff = ring->rx_buf_len >> I40E_RXQ_CTX_DBUFF_SHIFT;
2219         rx_ctx.hbuff = ring->rx_hdr_len >> I40E_RXQ_CTX_HBUFF_SHIFT;
2220
2221         rx_ctx.base = (ring->dma / 128);
2222         rx_ctx.qlen = ring->count;
2223
2224         if (vsi->back->flags & I40E_FLAG_16BYTE_RX_DESC_ENABLED) {
2225                 set_ring_16byte_desc_enabled(ring);
2226                 rx_ctx.dsize = 0;
2227         } else {
2228                 rx_ctx.dsize = 1;
2229         }
2230
2231         rx_ctx.dtype = vsi->dtype;
2232         if (vsi->dtype) {
2233                 set_ring_ps_enabled(ring);
2234                 rx_ctx.hsplit_0 = I40E_RX_SPLIT_L2      |
2235                                   I40E_RX_SPLIT_IP      |
2236                                   I40E_RX_SPLIT_TCP_UDP |
2237                                   I40E_RX_SPLIT_SCTP;
2238         } else {
2239                 rx_ctx.hsplit_0 = 0;
2240         }
2241
2242         rx_ctx.rxmax = min_t(u16, vsi->max_frame,
2243                                   (chain_len * ring->rx_buf_len));
2244         rx_ctx.tphrdesc_ena = 1;
2245         rx_ctx.tphwdesc_ena = 1;
2246         rx_ctx.tphdata_ena = 1;
2247         rx_ctx.tphhead_ena = 1;
2248         if (hw->revision_id == 0)
2249                 rx_ctx.lrxqthresh = 0;
2250         else
2251                 rx_ctx.lrxqthresh = 2;
2252         rx_ctx.crcstrip = 1;
2253         rx_ctx.l2tsel = 1;
2254         rx_ctx.showiv = 1;
2255
2256         /* clear the context in the HMC */
2257         err = i40e_clear_lan_rx_queue_context(hw, pf_q);
2258         if (err) {
2259                 dev_info(&vsi->back->pdev->dev,
2260                          "Failed to clear LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
2261                          ring->queue_index, pf_q, err);
2262                 return -ENOMEM;
2263         }
2264
2265         /* set the context in the HMC */
2266         err = i40e_set_lan_rx_queue_context(hw, pf_q, &rx_ctx);
2267         if (err) {
2268                 dev_info(&vsi->back->pdev->dev,
2269                          "Failed to set LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
2270                          ring->queue_index, pf_q, err);
2271                 return -ENOMEM;
2272         }
2273
2274         /* cache tail for quicker writes, and clear the reg before use */
2275         ring->tail = hw->hw_addr + I40E_QRX_TAIL(pf_q);
2276         writel(0, ring->tail);
2277
2278         i40e_alloc_rx_buffers(ring, I40E_DESC_UNUSED(ring));
2279
2280         return 0;
2281 }
2282
2283 /**
2284  * i40e_vsi_configure_tx - Configure the VSI for Tx
2285  * @vsi: VSI structure describing this set of rings and resources
2286  *
2287  * Configure the Tx VSI for operation.
2288  **/
2289 static int i40e_vsi_configure_tx(struct i40e_vsi *vsi)
2290 {
2291         int err = 0;
2292         u16 i;
2293
2294         for (i = 0; (i < vsi->num_queue_pairs) && !err; i++)
2295                 err = i40e_configure_tx_ring(vsi->tx_rings[i]);
2296
2297         return err;
2298 }
2299
2300 /**
2301  * i40e_vsi_configure_rx - Configure the VSI for Rx
2302  * @vsi: the VSI being configured
2303  *
2304  * Configure the Rx VSI for operation.
2305  **/
2306 static int i40e_vsi_configure_rx(struct i40e_vsi *vsi)
2307 {
2308         int err = 0;
2309         u16 i;
2310
2311         if (vsi->netdev && (vsi->netdev->mtu > ETH_DATA_LEN))
2312                 vsi->max_frame = vsi->netdev->mtu + ETH_HLEN
2313                                + ETH_FCS_LEN + VLAN_HLEN;
2314         else
2315                 vsi->max_frame = I40E_RXBUFFER_2048;
2316
2317         /* figure out correct receive buffer length */
2318         switch (vsi->back->flags & (I40E_FLAG_RX_1BUF_ENABLED |
2319                                     I40E_FLAG_RX_PS_ENABLED)) {
2320         case I40E_FLAG_RX_1BUF_ENABLED:
2321                 vsi->rx_hdr_len = 0;
2322                 vsi->rx_buf_len = vsi->max_frame;
2323                 vsi->dtype = I40E_RX_DTYPE_NO_SPLIT;
2324                 break;
2325         case I40E_FLAG_RX_PS_ENABLED:
2326                 vsi->rx_hdr_len = I40E_RX_HDR_SIZE;
2327                 vsi->rx_buf_len = I40E_RXBUFFER_2048;
2328                 vsi->dtype = I40E_RX_DTYPE_HEADER_SPLIT;
2329                 break;
2330         default:
2331                 vsi->rx_hdr_len = I40E_RX_HDR_SIZE;
2332                 vsi->rx_buf_len = I40E_RXBUFFER_2048;
2333                 vsi->dtype = I40E_RX_DTYPE_SPLIT_ALWAYS;
2334                 break;
2335         }
2336
2337         /* round up for the chip's needs */
2338         vsi->rx_hdr_len = ALIGN(vsi->rx_hdr_len,
2339                                 (1 << I40E_RXQ_CTX_HBUFF_SHIFT));
2340         vsi->rx_buf_len = ALIGN(vsi->rx_buf_len,
2341                                 (1 << I40E_RXQ_CTX_DBUFF_SHIFT));
2342
2343         /* set up individual rings */
2344         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
2345                 err = i40e_configure_rx_ring(vsi->rx_rings[i]);
2346
2347         return err;
2348 }
2349
2350 /**
2351  * i40e_vsi_config_dcb_rings - Update rings to reflect DCB TC
2352  * @vsi: ptr to the VSI
2353  **/
2354 static void i40e_vsi_config_dcb_rings(struct i40e_vsi *vsi)
2355 {
2356         u16 qoffset, qcount;
2357         int i, n;
2358
2359         if (!(vsi->back->flags & I40E_FLAG_DCB_ENABLED))
2360                 return;
2361
2362         for (n = 0; n < I40E_MAX_TRAFFIC_CLASS; n++) {
2363                 if (!(vsi->tc_config.enabled_tc & (1 << n)))
2364                         continue;
2365
2366                 qoffset = vsi->tc_config.tc_info[n].qoffset;
2367                 qcount = vsi->tc_config.tc_info[n].qcount;
2368                 for (i = qoffset; i < (qoffset + qcount); i++) {
2369                         struct i40e_ring *rx_ring = vsi->rx_rings[i];
2370                         struct i40e_ring *tx_ring = vsi->tx_rings[i];
2371                         rx_ring->dcb_tc = n;
2372                         tx_ring->dcb_tc = n;
2373                 }
2374         }
2375 }
2376
2377 /**
2378  * i40e_set_vsi_rx_mode - Call set_rx_mode on a VSI
2379  * @vsi: ptr to the VSI
2380  **/
2381 static void i40e_set_vsi_rx_mode(struct i40e_vsi *vsi)
2382 {
2383         if (vsi->netdev)
2384                 i40e_set_rx_mode(vsi->netdev);
2385 }
2386
2387 /**
2388  * i40e_vsi_configure - Set up the VSI for action
2389  * @vsi: the VSI being configured
2390  **/
2391 static int i40e_vsi_configure(struct i40e_vsi *vsi)
2392 {
2393         int err;
2394
2395         i40e_set_vsi_rx_mode(vsi);
2396         i40e_restore_vlan(vsi);
2397         i40e_vsi_config_dcb_rings(vsi);
2398         err = i40e_vsi_configure_tx(vsi);
2399         if (!err)
2400                 err = i40e_vsi_configure_rx(vsi);
2401
2402         return err;
2403 }
2404
2405 /**
2406  * i40e_vsi_configure_msix - MSIX mode Interrupt Config in the HW
2407  * @vsi: the VSI being configured
2408  **/
2409 static void i40e_vsi_configure_msix(struct i40e_vsi *vsi)
2410 {
2411         struct i40e_pf *pf = vsi->back;
2412         struct i40e_q_vector *q_vector;
2413         struct i40e_hw *hw = &pf->hw;
2414         u16 vector;
2415         int i, q;
2416         u32 val;
2417         u32 qp;
2418
2419         /* The interrupt indexing is offset by 1 in the PFINT_ITRn
2420          * and PFINT_LNKLSTn registers, e.g.:
2421          *   PFINT_ITRn[0..n-1] gets msix-1..msix-n  (qpair interrupts)
2422          */
2423         qp = vsi->base_queue;
2424         vector = vsi->base_vector;
2425         for (i = 0; i < vsi->num_q_vectors; i++, vector++) {
2426                 q_vector = vsi->q_vectors[i];
2427                 q_vector->rx.itr = ITR_TO_REG(vsi->rx_itr_setting);
2428                 q_vector->rx.latency_range = I40E_LOW_LATENCY;
2429                 wr32(hw, I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1),
2430                      q_vector->rx.itr);
2431                 q_vector->tx.itr = ITR_TO_REG(vsi->tx_itr_setting);
2432                 q_vector->tx.latency_range = I40E_LOW_LATENCY;
2433                 wr32(hw, I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1),
2434                      q_vector->tx.itr);
2435
2436                 /* Linked list for the queuepairs assigned to this vector */
2437                 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), qp);
2438                 for (q = 0; q < q_vector->num_ringpairs; q++) {
2439                         val = I40E_QINT_RQCTL_CAUSE_ENA_MASK |
2440                               (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT)  |
2441                               (vector      << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
2442                               (qp          << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT)|
2443                               (I40E_QUEUE_TYPE_TX
2444                                       << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT);
2445
2446                         wr32(hw, I40E_QINT_RQCTL(qp), val);
2447
2448                         val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
2449                               (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT)  |
2450                               (vector      << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
2451                               ((qp+1)      << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT)|
2452                               (I40E_QUEUE_TYPE_RX
2453                                       << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
2454
2455                         /* Terminate the linked list */
2456                         if (q == (q_vector->num_ringpairs - 1))
2457                                 val |= (I40E_QUEUE_END_OF_LIST
2458                                            << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
2459
2460                         wr32(hw, I40E_QINT_TQCTL(qp), val);
2461                         qp++;
2462                 }
2463         }
2464
2465         i40e_flush(hw);
2466 }
2467
2468 /**
2469  * i40e_enable_misc_int_causes - enable the non-queue interrupts
2470  * @hw: ptr to the hardware info
2471  **/
2472 static void i40e_enable_misc_int_causes(struct i40e_hw *hw)
2473 {
2474         u32 val;
2475
2476         /* clear things first */
2477         wr32(hw, I40E_PFINT_ICR0_ENA, 0);  /* disable all */
2478         rd32(hw, I40E_PFINT_ICR0);         /* read to clear */
2479
2480         val = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK       |
2481               I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK    |
2482               I40E_PFINT_ICR0_ENA_GRST_MASK          |
2483               I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK |
2484               I40E_PFINT_ICR0_ENA_GPIO_MASK          |
2485               I40E_PFINT_ICR0_ENA_STORM_DETECT_MASK  |
2486               I40E_PFINT_ICR0_ENA_HMC_ERR_MASK       |
2487               I40E_PFINT_ICR0_ENA_VFLR_MASK          |
2488               I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
2489
2490         wr32(hw, I40E_PFINT_ICR0_ENA, val);
2491
2492         /* SW_ITR_IDX = 0, but don't change INTENA */
2493         wr32(hw, I40E_PFINT_DYN_CTL0, I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK |
2494                                         I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK);
2495
2496         /* OTHER_ITR_IDX = 0 */
2497         wr32(hw, I40E_PFINT_STAT_CTL0, 0);
2498 }
2499
2500 /**
2501  * i40e_configure_msi_and_legacy - Legacy mode interrupt config in the HW
2502  * @vsi: the VSI being configured
2503  **/
2504 static void i40e_configure_msi_and_legacy(struct i40e_vsi *vsi)
2505 {
2506         struct i40e_q_vector *q_vector = vsi->q_vectors[0];
2507         struct i40e_pf *pf = vsi->back;
2508         struct i40e_hw *hw = &pf->hw;
2509         u32 val;
2510
2511         /* set the ITR configuration */
2512         q_vector->rx.itr = ITR_TO_REG(vsi->rx_itr_setting);
2513         q_vector->rx.latency_range = I40E_LOW_LATENCY;
2514         wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), q_vector->rx.itr);
2515         q_vector->tx.itr = ITR_TO_REG(vsi->tx_itr_setting);
2516         q_vector->tx.latency_range = I40E_LOW_LATENCY;
2517         wr32(hw, I40E_PFINT_ITR0(I40E_TX_ITR), q_vector->tx.itr);
2518
2519         i40e_enable_misc_int_causes(hw);
2520
2521         /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */
2522         wr32(hw, I40E_PFINT_LNKLST0, 0);
2523
2524         /* Associate the queue pair to the vector and enable the q int */
2525         val = I40E_QINT_RQCTL_CAUSE_ENA_MASK                  |
2526               (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) |
2527               (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
2528
2529         wr32(hw, I40E_QINT_RQCTL(0), val);
2530
2531         val = I40E_QINT_TQCTL_CAUSE_ENA_MASK                  |
2532               (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
2533               (I40E_QUEUE_END_OF_LIST << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
2534
2535         wr32(hw, I40E_QINT_TQCTL(0), val);
2536         i40e_flush(hw);
2537 }
2538
2539 /**
2540  * i40e_irq_dynamic_disable_icr0 - Disable default interrupt generation for icr0
2541  * @pf: board private structure
2542  **/
2543 void i40e_irq_dynamic_disable_icr0(struct i40e_pf *pf)
2544 {
2545         struct i40e_hw *hw = &pf->hw;
2546
2547         wr32(hw, I40E_PFINT_DYN_CTL0,
2548              I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
2549         i40e_flush(hw);
2550 }
2551
2552 /**
2553  * i40e_irq_dynamic_enable_icr0 - Enable default interrupt generation for icr0
2554  * @pf: board private structure
2555  **/
2556 void i40e_irq_dynamic_enable_icr0(struct i40e_pf *pf)
2557 {
2558         struct i40e_hw *hw = &pf->hw;
2559         u32 val;
2560
2561         val = I40E_PFINT_DYN_CTL0_INTENA_MASK   |
2562               I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
2563               (I40E_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT);
2564
2565         wr32(hw, I40E_PFINT_DYN_CTL0, val);
2566         i40e_flush(hw);
2567 }
2568
2569 /**
2570  * i40e_irq_dynamic_enable - Enable default interrupt generation settings
2571  * @vsi: pointer to a vsi
2572  * @vector: enable a particular Hw Interrupt vector
2573  **/
2574 void i40e_irq_dynamic_enable(struct i40e_vsi *vsi, int vector)
2575 {
2576         struct i40e_pf *pf = vsi->back;
2577         struct i40e_hw *hw = &pf->hw;
2578         u32 val;
2579
2580         val = I40E_PFINT_DYN_CTLN_INTENA_MASK |
2581               I40E_PFINT_DYN_CTLN_CLEARPBA_MASK |
2582               (I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
2583         wr32(hw, I40E_PFINT_DYN_CTLN(vector - 1), val);
2584         /* skip the flush */
2585 }
2586
2587 /**
2588  * i40e_msix_clean_rings - MSIX mode Interrupt Handler
2589  * @irq: interrupt number
2590  * @data: pointer to a q_vector
2591  **/
2592 static irqreturn_t i40e_msix_clean_rings(int irq, void *data)
2593 {
2594         struct i40e_q_vector *q_vector = data;
2595
2596         if (!q_vector->tx.ring && !q_vector->rx.ring)
2597                 return IRQ_HANDLED;
2598
2599         napi_schedule(&q_vector->napi);
2600
2601         return IRQ_HANDLED;
2602 }
2603
2604 /**
2605  * i40e_fdir_clean_rings - Interrupt Handler for FDIR rings
2606  * @irq: interrupt number
2607  * @data: pointer to a q_vector
2608  **/
2609 static irqreturn_t i40e_fdir_clean_rings(int irq, void *data)
2610 {
2611         struct i40e_q_vector *q_vector = data;
2612
2613         if (!q_vector->tx.ring && !q_vector->rx.ring)
2614                 return IRQ_HANDLED;
2615
2616         pr_info("fdir ring cleaning needed\n");
2617
2618         return IRQ_HANDLED;
2619 }
2620
2621 /**
2622  * i40e_vsi_request_irq_msix - Initialize MSI-X interrupts
2623  * @vsi: the VSI being configured
2624  * @basename: name for the vector
2625  *
2626  * Allocates MSI-X vectors and requests interrupts from the kernel.
2627  **/
2628 static int i40e_vsi_request_irq_msix(struct i40e_vsi *vsi, char *basename)
2629 {
2630         int q_vectors = vsi->num_q_vectors;
2631         struct i40e_pf *pf = vsi->back;
2632         int base = vsi->base_vector;
2633         int rx_int_idx = 0;
2634         int tx_int_idx = 0;
2635         int vector, err;
2636
2637         for (vector = 0; vector < q_vectors; vector++) {
2638                 struct i40e_q_vector *q_vector = vsi->q_vectors[vector];
2639
2640                 if (q_vector->tx.ring && q_vector->rx.ring) {
2641                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
2642                                  "%s-%s-%d", basename, "TxRx", rx_int_idx++);
2643                         tx_int_idx++;
2644                 } else if (q_vector->rx.ring) {
2645                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
2646                                  "%s-%s-%d", basename, "rx", rx_int_idx++);
2647                 } else if (q_vector->tx.ring) {
2648                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
2649                                  "%s-%s-%d", basename, "tx", tx_int_idx++);
2650                 } else {
2651                         /* skip this unused q_vector */
2652                         continue;
2653                 }
2654                 err = request_irq(pf->msix_entries[base + vector].vector,
2655                                   vsi->irq_handler,
2656                                   0,
2657                                   q_vector->name,
2658                                   q_vector);
2659                 if (err) {
2660                         dev_info(&pf->pdev->dev,
2661                                  "%s: request_irq failed, error: %d\n",
2662                                  __func__, err);
2663                         goto free_queue_irqs;
2664                 }
2665                 /* assign the mask for this irq */
2666                 irq_set_affinity_hint(pf->msix_entries[base + vector].vector,
2667                                       &q_vector->affinity_mask);
2668         }
2669
2670         return 0;
2671
2672 free_queue_irqs:
2673         while (vector) {
2674                 vector--;
2675                 irq_set_affinity_hint(pf->msix_entries[base + vector].vector,
2676                                       NULL);
2677                 free_irq(pf->msix_entries[base + vector].vector,
2678                          &(vsi->q_vectors[vector]));
2679         }
2680         return err;
2681 }
2682
2683 /**
2684  * i40e_vsi_disable_irq - Mask off queue interrupt generation on the VSI
2685  * @vsi: the VSI being un-configured
2686  **/
2687 static void i40e_vsi_disable_irq(struct i40e_vsi *vsi)
2688 {
2689         struct i40e_pf *pf = vsi->back;
2690         struct i40e_hw *hw = &pf->hw;
2691         int base = vsi->base_vector;
2692         int i;
2693
2694         for (i = 0; i < vsi->num_queue_pairs; i++) {
2695                 wr32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx), 0);
2696                 wr32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx), 0);
2697         }
2698
2699         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
2700                 for (i = vsi->base_vector;
2701                      i < (vsi->num_q_vectors + vsi->base_vector); i++)
2702                         wr32(hw, I40E_PFINT_DYN_CTLN(i - 1), 0);
2703
2704                 i40e_flush(hw);
2705                 for (i = 0; i < vsi->num_q_vectors; i++)
2706                         synchronize_irq(pf->msix_entries[i + base].vector);
2707         } else {
2708                 /* Legacy and MSI mode - this stops all interrupt handling */
2709                 wr32(hw, I40E_PFINT_ICR0_ENA, 0);
2710                 wr32(hw, I40E_PFINT_DYN_CTL0, 0);
2711                 i40e_flush(hw);
2712                 synchronize_irq(pf->pdev->irq);
2713         }
2714 }
2715
2716 /**
2717  * i40e_vsi_enable_irq - Enable IRQ for the given VSI
2718  * @vsi: the VSI being configured
2719  **/
2720 static int i40e_vsi_enable_irq(struct i40e_vsi *vsi)
2721 {
2722         struct i40e_pf *pf = vsi->back;
2723         int i;
2724
2725         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
2726                 for (i = vsi->base_vector;
2727                      i < (vsi->num_q_vectors + vsi->base_vector); i++)
2728                         i40e_irq_dynamic_enable(vsi, i);
2729         } else {
2730                 i40e_irq_dynamic_enable_icr0(pf);
2731         }
2732
2733         i40e_flush(&pf->hw);
2734         return 0;
2735 }
2736
2737 /**
2738  * i40e_stop_misc_vector - Stop the vector that handles non-queue events
2739  * @pf: board private structure
2740  **/
2741 static void i40e_stop_misc_vector(struct i40e_pf *pf)
2742 {
2743         /* Disable ICR 0 */
2744         wr32(&pf->hw, I40E_PFINT_ICR0_ENA, 0);
2745         i40e_flush(&pf->hw);
2746 }
2747
2748 /**
2749  * i40e_intr - MSI/Legacy and non-queue interrupt handler
2750  * @irq: interrupt number
2751  * @data: pointer to a q_vector
2752  *
2753  * This is the handler used for all MSI/Legacy interrupts, and deals
2754  * with both queue and non-queue interrupts.  This is also used in
2755  * MSIX mode to handle the non-queue interrupts.
2756  **/
2757 static irqreturn_t i40e_intr(int irq, void *data)
2758 {
2759         struct i40e_pf *pf = (struct i40e_pf *)data;
2760         struct i40e_hw *hw = &pf->hw;
2761         u32 icr0, icr0_remaining;
2762         u32 val, ena_mask;
2763
2764         icr0 = rd32(hw, I40E_PFINT_ICR0);
2765
2766         /* if sharing a legacy IRQ, we might get called w/o an intr pending */
2767         if ((icr0 & I40E_PFINT_ICR0_INTEVENT_MASK) == 0)
2768                 return IRQ_NONE;
2769
2770         ena_mask = rd32(hw, I40E_PFINT_ICR0_ENA);
2771
2772         /* if interrupt but no bits showing, must be SWINT */
2773         if (((icr0 & ~I40E_PFINT_ICR0_INTEVENT_MASK) == 0) ||
2774             (icr0 & I40E_PFINT_ICR0_SWINT_MASK))
2775                 pf->sw_int_count++;
2776
2777         /* only q0 is used in MSI/Legacy mode, and none are used in MSIX */
2778         if (icr0 & I40E_PFINT_ICR0_QUEUE_0_MASK) {
2779
2780                 /* temporarily disable queue cause for NAPI processing */
2781                 u32 qval = rd32(hw, I40E_QINT_RQCTL(0));
2782                 qval &= ~I40E_QINT_RQCTL_CAUSE_ENA_MASK;
2783                 wr32(hw, I40E_QINT_RQCTL(0), qval);
2784
2785                 qval = rd32(hw, I40E_QINT_TQCTL(0));
2786                 qval &= ~I40E_QINT_TQCTL_CAUSE_ENA_MASK;
2787                 wr32(hw, I40E_QINT_TQCTL(0), qval);
2788
2789                 if (!test_bit(__I40E_DOWN, &pf->state))
2790                         napi_schedule(&pf->vsi[pf->lan_vsi]->q_vectors[0]->napi);
2791         }
2792
2793         if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) {
2794                 ena_mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
2795                 set_bit(__I40E_ADMINQ_EVENT_PENDING, &pf->state);
2796         }
2797
2798         if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK) {
2799                 ena_mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
2800                 set_bit(__I40E_MDD_EVENT_PENDING, &pf->state);
2801         }
2802
2803         if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) {
2804                 ena_mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK;
2805                 set_bit(__I40E_VFLR_EVENT_PENDING, &pf->state);
2806         }
2807
2808         if (icr0 & I40E_PFINT_ICR0_GRST_MASK) {
2809                 if (!test_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state))
2810                         set_bit(__I40E_RESET_INTR_RECEIVED, &pf->state);
2811                 ena_mask &= ~I40E_PFINT_ICR0_ENA_GRST_MASK;
2812                 val = rd32(hw, I40E_GLGEN_RSTAT);
2813                 val = (val & I40E_GLGEN_RSTAT_RESET_TYPE_MASK)
2814                        >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT;
2815                 if (val == I40E_RESET_CORER)
2816                         pf->corer_count++;
2817                 else if (val == I40E_RESET_GLOBR)
2818                         pf->globr_count++;
2819                 else if (val == I40E_RESET_EMPR)
2820                         pf->empr_count++;
2821         }
2822
2823         if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK) {
2824                 icr0 &= ~I40E_PFINT_ICR0_HMC_ERR_MASK;
2825                 dev_info(&pf->pdev->dev, "HMC error interrupt\n");
2826         }
2827
2828         /* If a critical error is pending we have no choice but to reset the
2829          * device.
2830          * Report and mask out any remaining unexpected interrupts.
2831          */
2832         icr0_remaining = icr0 & ena_mask;
2833         if (icr0_remaining) {
2834                 dev_info(&pf->pdev->dev, "unhandled interrupt icr0=0x%08x\n",
2835                          icr0_remaining);
2836                 if ((icr0_remaining & I40E_PFINT_ICR0_PE_CRITERR_MASK) ||
2837                     (icr0_remaining & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) ||
2838                     (icr0_remaining & I40E_PFINT_ICR0_ECC_ERR_MASK) ||
2839                     (icr0_remaining & I40E_PFINT_ICR0_MAL_DETECT_MASK)) {
2840                         dev_info(&pf->pdev->dev, "device will be reset\n");
2841                         set_bit(__I40E_PF_RESET_REQUESTED, &pf->state);
2842                         i40e_service_event_schedule(pf);
2843                 }
2844                 ena_mask &= ~icr0_remaining;
2845         }
2846
2847         /* re-enable interrupt causes */
2848         wr32(hw, I40E_PFINT_ICR0_ENA, ena_mask);
2849         if (!test_bit(__I40E_DOWN, &pf->state)) {
2850                 i40e_service_event_schedule(pf);
2851                 i40e_irq_dynamic_enable_icr0(pf);
2852         }
2853
2854         return IRQ_HANDLED;
2855 }
2856
2857 /**
2858  * i40e_map_vector_to_qp - Assigns the queue pair to the vector
2859  * @vsi: the VSI being configured
2860  * @v_idx: vector index
2861  * @qp_idx: queue pair index
2862  **/
2863 static void map_vector_to_qp(struct i40e_vsi *vsi, int v_idx, int qp_idx)
2864 {
2865         struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
2866         struct i40e_ring *tx_ring = vsi->tx_rings[qp_idx];
2867         struct i40e_ring *rx_ring = vsi->rx_rings[qp_idx];
2868
2869         tx_ring->q_vector = q_vector;
2870         tx_ring->next = q_vector->tx.ring;
2871         q_vector->tx.ring = tx_ring;
2872         q_vector->tx.count++;
2873
2874         rx_ring->q_vector = q_vector;
2875         rx_ring->next = q_vector->rx.ring;
2876         q_vector->rx.ring = rx_ring;
2877         q_vector->rx.count++;
2878 }
2879
2880 /**
2881  * i40e_vsi_map_rings_to_vectors - Maps descriptor rings to vectors
2882  * @vsi: the VSI being configured
2883  *
2884  * This function maps descriptor rings to the queue-specific vectors
2885  * we were allotted through the MSI-X enabling code.  Ideally, we'd have
2886  * one vector per queue pair, but on a constrained vector budget, we
2887  * group the queue pairs as "efficiently" as possible.
2888  **/
2889 static void i40e_vsi_map_rings_to_vectors(struct i40e_vsi *vsi)
2890 {
2891         int qp_remaining = vsi->num_queue_pairs;
2892         int q_vectors = vsi->num_q_vectors;
2893         int num_ringpairs;
2894         int v_start = 0;
2895         int qp_idx = 0;
2896
2897         /* If we don't have enough vectors for a 1-to-1 mapping, we'll have to
2898          * group them so there are multiple queues per vector.
2899          */
2900         for (; v_start < q_vectors && qp_remaining; v_start++) {
2901                 struct i40e_q_vector *q_vector = vsi->q_vectors[v_start];
2902
2903                 num_ringpairs = DIV_ROUND_UP(qp_remaining, q_vectors - v_start);
2904
2905                 q_vector->num_ringpairs = num_ringpairs;
2906
2907                 q_vector->rx.count = 0;
2908                 q_vector->tx.count = 0;
2909                 q_vector->rx.ring = NULL;
2910                 q_vector->tx.ring = NULL;
2911
2912                 while (num_ringpairs--) {
2913                         map_vector_to_qp(vsi, v_start, qp_idx);
2914                         qp_idx++;
2915                         qp_remaining--;
2916                 }
2917         }
2918 }
2919
2920 /**
2921  * i40e_vsi_request_irq - Request IRQ from the OS
2922  * @vsi: the VSI being configured
2923  * @basename: name for the vector
2924  **/
2925 static int i40e_vsi_request_irq(struct i40e_vsi *vsi, char *basename)
2926 {
2927         struct i40e_pf *pf = vsi->back;
2928         int err;
2929
2930         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
2931                 err = i40e_vsi_request_irq_msix(vsi, basename);
2932         else if (pf->flags & I40E_FLAG_MSI_ENABLED)
2933                 err = request_irq(pf->pdev->irq, i40e_intr, 0,
2934                                   pf->misc_int_name, pf);
2935         else
2936                 err = request_irq(pf->pdev->irq, i40e_intr, IRQF_SHARED,
2937                                   pf->misc_int_name, pf);
2938
2939         if (err)
2940                 dev_info(&pf->pdev->dev, "request_irq failed, Error %d\n", err);
2941
2942         return err;
2943 }
2944
2945 #ifdef CONFIG_NET_POLL_CONTROLLER
2946 /**
2947  * i40e_netpoll - A Polling 'interrupt'handler
2948  * @netdev: network interface device structure
2949  *
2950  * This is used by netconsole to send skbs without having to re-enable
2951  * interrupts.  It's not called while the normal interrupt routine is executing.
2952  **/
2953 static void i40e_netpoll(struct net_device *netdev)
2954 {
2955         struct i40e_netdev_priv *np = netdev_priv(netdev);
2956         struct i40e_vsi *vsi = np->vsi;
2957         struct i40e_pf *pf = vsi->back;
2958         int i;
2959
2960         /* if interface is down do nothing */
2961         if (test_bit(__I40E_DOWN, &vsi->state))
2962                 return;
2963
2964         pf->flags |= I40E_FLAG_IN_NETPOLL;
2965         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
2966                 for (i = 0; i < vsi->num_q_vectors; i++)
2967                         i40e_msix_clean_rings(0, vsi->q_vectors[i]);
2968         } else {
2969                 i40e_intr(pf->pdev->irq, netdev);
2970         }
2971         pf->flags &= ~I40E_FLAG_IN_NETPOLL;
2972 }
2973 #endif
2974
2975 /**
2976  * i40e_vsi_control_tx - Start or stop a VSI's rings
2977  * @vsi: the VSI being configured
2978  * @enable: start or stop the rings
2979  **/
2980 static int i40e_vsi_control_tx(struct i40e_vsi *vsi, bool enable)
2981 {
2982         struct i40e_pf *pf = vsi->back;
2983         struct i40e_hw *hw = &pf->hw;
2984         int i, j, pf_q;
2985         u32 tx_reg;
2986
2987         pf_q = vsi->base_queue;
2988         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
2989                 j = 1000;
2990                 do {
2991                         usleep_range(1000, 2000);
2992                         tx_reg = rd32(hw, I40E_QTX_ENA(pf_q));
2993                 } while (j-- && ((tx_reg >> I40E_QTX_ENA_QENA_REQ_SHIFT)
2994                                ^ (tx_reg >> I40E_QTX_ENA_QENA_STAT_SHIFT)) & 1);
2995
2996                 /* Skip if the queue is already in the requested state */
2997                 if (enable && (tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
2998                         continue;
2999                 if (!enable && !(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
3000                         continue;
3001
3002                 /* turn on/off the queue */
3003                 if (enable)
3004                         tx_reg |= I40E_QTX_ENA_QENA_REQ_MASK |
3005                                   I40E_QTX_ENA_QENA_STAT_MASK;
3006                 else
3007                         tx_reg &= ~I40E_QTX_ENA_QENA_REQ_MASK;
3008
3009                 wr32(hw, I40E_QTX_ENA(pf_q), tx_reg);
3010
3011                 /* wait for the change to finish */
3012                 for (j = 0; j < 10; j++) {
3013                         tx_reg = rd32(hw, I40E_QTX_ENA(pf_q));
3014                         if (enable) {
3015                                 if ((tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
3016                                         break;
3017                         } else {
3018                                 if (!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
3019                                         break;
3020                         }
3021
3022                         udelay(10);
3023                 }
3024                 if (j >= 10) {
3025                         dev_info(&pf->pdev->dev, "Tx ring %d %sable timeout\n",
3026                                  pf_q, (enable ? "en" : "dis"));
3027                         return -ETIMEDOUT;
3028                 }
3029         }
3030
3031         if (hw->revision_id == 0)
3032                 mdelay(50);
3033
3034         return 0;
3035 }
3036
3037 /**
3038  * i40e_vsi_control_rx - Start or stop a VSI's rings
3039  * @vsi: the VSI being configured
3040  * @enable: start or stop the rings
3041  **/
3042 static int i40e_vsi_control_rx(struct i40e_vsi *vsi, bool enable)
3043 {
3044         struct i40e_pf *pf = vsi->back;
3045         struct i40e_hw *hw = &pf->hw;
3046         int i, j, pf_q;
3047         u32 rx_reg;
3048
3049         pf_q = vsi->base_queue;
3050         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
3051                 j = 1000;
3052                 do {
3053                         usleep_range(1000, 2000);
3054                         rx_reg = rd32(hw, I40E_QRX_ENA(pf_q));
3055                 } while (j-- && ((rx_reg >> I40E_QRX_ENA_QENA_REQ_SHIFT)
3056                                ^ (rx_reg >> I40E_QRX_ENA_QENA_STAT_SHIFT)) & 1);
3057
3058                 if (enable) {
3059                         /* is STAT set ? */
3060                         if ((rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
3061                                 continue;
3062                 } else {
3063                         /* is !STAT set ? */
3064                         if (!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
3065                                 continue;
3066                 }
3067
3068                 /* turn on/off the queue */
3069                 if (enable)
3070                         rx_reg |= I40E_QRX_ENA_QENA_REQ_MASK |
3071                                   I40E_QRX_ENA_QENA_STAT_MASK;
3072                 else
3073                         rx_reg &= ~(I40E_QRX_ENA_QENA_REQ_MASK |
3074                                   I40E_QRX_ENA_QENA_STAT_MASK);
3075                 wr32(hw, I40E_QRX_ENA(pf_q), rx_reg);
3076
3077                 /* wait for the change to finish */
3078                 for (j = 0; j < 10; j++) {
3079                         rx_reg = rd32(hw, I40E_QRX_ENA(pf_q));
3080
3081                         if (enable) {
3082                                 if ((rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
3083                                         break;
3084                         } else {
3085                                 if (!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
3086                                         break;
3087                         }
3088
3089                         udelay(10);
3090                 }
3091                 if (j >= 10) {
3092                         dev_info(&pf->pdev->dev, "Rx ring %d %sable timeout\n",
3093                                  pf_q, (enable ? "en" : "dis"));
3094                         return -ETIMEDOUT;
3095                 }
3096         }
3097
3098         return 0;
3099 }
3100
3101 /**
3102  * i40e_vsi_control_rings - Start or stop a VSI's rings
3103  * @vsi: the VSI being configured
3104  * @enable: start or stop the rings
3105  **/
3106 int i40e_vsi_control_rings(struct i40e_vsi *vsi, bool request)
3107 {
3108         int ret;
3109
3110         /* do rx first for enable and last for disable */
3111         if (request) {
3112                 ret = i40e_vsi_control_rx(vsi, request);
3113                 if (ret)
3114                         return ret;
3115                 ret = i40e_vsi_control_tx(vsi, request);
3116         } else {
3117                 ret = i40e_vsi_control_tx(vsi, request);
3118                 if (ret)
3119                         return ret;
3120                 ret = i40e_vsi_control_rx(vsi, request);
3121         }
3122
3123         return ret;
3124 }
3125
3126 /**
3127  * i40e_vsi_free_irq - Free the irq association with the OS
3128  * @vsi: the VSI being configured
3129  **/
3130 static void i40e_vsi_free_irq(struct i40e_vsi *vsi)
3131 {
3132         struct i40e_pf *pf = vsi->back;
3133         struct i40e_hw *hw = &pf->hw;
3134         int base = vsi->base_vector;
3135         u32 val, qp;
3136         int i;
3137
3138         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
3139                 if (!vsi->q_vectors)
3140                         return;
3141
3142                 for (i = 0; i < vsi->num_q_vectors; i++) {
3143                         u16 vector = i + base;
3144
3145                         /* free only the irqs that were actually requested */
3146                         if (!vsi->q_vectors[i] ||
3147                             !vsi->q_vectors[i]->num_ringpairs)
3148                                 continue;
3149
3150                         /* clear the affinity_mask in the IRQ descriptor */
3151                         irq_set_affinity_hint(pf->msix_entries[vector].vector,
3152                                               NULL);
3153                         free_irq(pf->msix_entries[vector].vector,
3154                                  vsi->q_vectors[i]);
3155
3156                         /* Tear down the interrupt queue link list
3157                          *
3158                          * We know that they come in pairs and always
3159                          * the Rx first, then the Tx.  To clear the
3160                          * link list, stick the EOL value into the
3161                          * next_q field of the registers.
3162                          */
3163                         val = rd32(hw, I40E_PFINT_LNKLSTN(vector - 1));
3164                         qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK)
3165                                 >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
3166                         val |= I40E_QUEUE_END_OF_LIST
3167                                 << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
3168                         wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), val);
3169
3170                         while (qp != I40E_QUEUE_END_OF_LIST) {
3171                                 u32 next;
3172
3173                                 val = rd32(hw, I40E_QINT_RQCTL(qp));
3174
3175                                 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK  |
3176                                          I40E_QINT_RQCTL_MSIX0_INDX_MASK |
3177                                          I40E_QINT_RQCTL_CAUSE_ENA_MASK  |
3178                                          I40E_QINT_RQCTL_INTEVENT_MASK);
3179
3180                                 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
3181                                          I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
3182
3183                                 wr32(hw, I40E_QINT_RQCTL(qp), val);
3184
3185                                 val = rd32(hw, I40E_QINT_TQCTL(qp));
3186
3187                                 next = (val & I40E_QINT_TQCTL_NEXTQ_INDX_MASK)
3188                                         >> I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT;
3189
3190                                 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK  |
3191                                          I40E_QINT_TQCTL_MSIX0_INDX_MASK |
3192                                          I40E_QINT_TQCTL_CAUSE_ENA_MASK  |
3193                                          I40E_QINT_TQCTL_INTEVENT_MASK);
3194
3195                                 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
3196                                          I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
3197
3198                                 wr32(hw, I40E_QINT_TQCTL(qp), val);
3199                                 qp = next;
3200                         }
3201                 }
3202         } else {
3203                 free_irq(pf->pdev->irq, pf);
3204
3205                 val = rd32(hw, I40E_PFINT_LNKLST0);
3206                 qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK)
3207                         >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
3208                 val |= I40E_QUEUE_END_OF_LIST
3209                         << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT;
3210                 wr32(hw, I40E_PFINT_LNKLST0, val);
3211
3212                 val = rd32(hw, I40E_QINT_RQCTL(qp));
3213                 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK  |
3214                          I40E_QINT_RQCTL_MSIX0_INDX_MASK |
3215                          I40E_QINT_RQCTL_CAUSE_ENA_MASK  |
3216                          I40E_QINT_RQCTL_INTEVENT_MASK);
3217
3218                 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
3219                         I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
3220
3221                 wr32(hw, I40E_QINT_RQCTL(qp), val);
3222
3223                 val = rd32(hw, I40E_QINT_TQCTL(qp));
3224
3225                 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK  |
3226                          I40E_QINT_TQCTL_MSIX0_INDX_MASK |
3227                          I40E_QINT_TQCTL_CAUSE_ENA_MASK  |
3228                          I40E_QINT_TQCTL_INTEVENT_MASK);
3229
3230                 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
3231                         I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
3232
3233                 wr32(hw, I40E_QINT_TQCTL(qp), val);
3234         }
3235 }
3236
3237 /**
3238  * i40e_free_q_vector - Free memory allocated for specific interrupt vector
3239  * @vsi: the VSI being configured
3240  * @v_idx: Index of vector to be freed
3241  *
3242  * This function frees the memory allocated to the q_vector.  In addition if
3243  * NAPI is enabled it will delete any references to the NAPI struct prior
3244  * to freeing the q_vector.
3245  **/
3246 static void i40e_free_q_vector(struct i40e_vsi *vsi, int v_idx)
3247 {
3248         struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
3249         struct i40e_ring *ring;
3250
3251         if (!q_vector)
3252                 return;
3253
3254         /* disassociate q_vector from rings */
3255         i40e_for_each_ring(ring, q_vector->tx)
3256                 ring->q_vector = NULL;
3257
3258         i40e_for_each_ring(ring, q_vector->rx)
3259                 ring->q_vector = NULL;
3260
3261         /* only VSI w/ an associated netdev is set up w/ NAPI */
3262         if (vsi->netdev)
3263                 netif_napi_del(&q_vector->napi);
3264
3265         vsi->q_vectors[v_idx] = NULL;
3266
3267         kfree_rcu(q_vector, rcu);
3268 }
3269
3270 /**
3271  * i40e_vsi_free_q_vectors - Free memory allocated for interrupt vectors
3272  * @vsi: the VSI being un-configured
3273  *
3274  * This frees the memory allocated to the q_vectors and
3275  * deletes references to the NAPI struct.
3276  **/
3277 static void i40e_vsi_free_q_vectors(struct i40e_vsi *vsi)
3278 {
3279         int v_idx;
3280
3281         for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++)
3282                 i40e_free_q_vector(vsi, v_idx);
3283 }
3284
3285 /**
3286  * i40e_reset_interrupt_capability - Disable interrupt setup in OS
3287  * @pf: board private structure
3288  **/
3289 static void i40e_reset_interrupt_capability(struct i40e_pf *pf)
3290 {
3291         /* If we're in Legacy mode, the interrupt was cleaned in vsi_close */
3292         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
3293                 pci_disable_msix(pf->pdev);
3294                 kfree(pf->msix_entries);
3295                 pf->msix_entries = NULL;
3296         } else if (pf->flags & I40E_FLAG_MSI_ENABLED) {
3297                 pci_disable_msi(pf->pdev);
3298         }
3299         pf->flags &= ~(I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED);
3300 }
3301
3302 /**
3303  * i40e_clear_interrupt_scheme - Clear the current interrupt scheme settings
3304  * @pf: board private structure
3305  *
3306  * We go through and clear interrupt specific resources and reset the structure
3307  * to pre-load conditions
3308  **/
3309 static void i40e_clear_interrupt_scheme(struct i40e_pf *pf)
3310 {
3311         int i;
3312
3313         i40e_put_lump(pf->irq_pile, 0, I40E_PILE_VALID_BIT-1);
3314         for (i = 0; i < pf->hw.func_caps.num_vsis; i++)
3315                 if (pf->vsi[i])
3316                         i40e_vsi_free_q_vectors(pf->vsi[i]);
3317         i40e_reset_interrupt_capability(pf);
3318 }
3319
3320 /**
3321  * i40e_napi_enable_all - Enable NAPI for all q_vectors in the VSI
3322  * @vsi: the VSI being configured
3323  **/
3324 static void i40e_napi_enable_all(struct i40e_vsi *vsi)
3325 {
3326         int q_idx;
3327
3328         if (!vsi->netdev)
3329                 return;
3330
3331         for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++)
3332                 napi_enable(&vsi->q_vectors[q_idx]->napi);
3333 }
3334
3335 /**
3336  * i40e_napi_disable_all - Disable NAPI for all q_vectors in the VSI
3337  * @vsi: the VSI being configured
3338  **/
3339 static void i40e_napi_disable_all(struct i40e_vsi *vsi)
3340 {
3341         int q_idx;
3342
3343         if (!vsi->netdev)
3344                 return;
3345
3346         for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++)
3347                 napi_disable(&vsi->q_vectors[q_idx]->napi);
3348 }
3349
3350 /**
3351  * i40e_quiesce_vsi - Pause a given VSI
3352  * @vsi: the VSI being paused
3353  **/
3354 static void i40e_quiesce_vsi(struct i40e_vsi *vsi)
3355 {
3356         if (test_bit(__I40E_DOWN, &vsi->state))
3357                 return;
3358
3359         set_bit(__I40E_NEEDS_RESTART, &vsi->state);
3360         if (vsi->netdev && netif_running(vsi->netdev)) {
3361                 vsi->netdev->netdev_ops->ndo_stop(vsi->netdev);
3362         } else {
3363                 set_bit(__I40E_DOWN, &vsi->state);
3364                 i40e_down(vsi);
3365         }
3366 }
3367
3368 /**
3369  * i40e_unquiesce_vsi - Resume a given VSI
3370  * @vsi: the VSI being resumed
3371  **/
3372 static void i40e_unquiesce_vsi(struct i40e_vsi *vsi)
3373 {
3374         if (!test_bit(__I40E_NEEDS_RESTART, &vsi->state))
3375                 return;
3376
3377         clear_bit(__I40E_NEEDS_RESTART, &vsi->state);
3378         if (vsi->netdev && netif_running(vsi->netdev))
3379                 vsi->netdev->netdev_ops->ndo_open(vsi->netdev);
3380         else
3381                 i40e_up(vsi);   /* this clears the DOWN bit */
3382 }
3383
3384 /**
3385  * i40e_pf_quiesce_all_vsi - Pause all VSIs on a PF
3386  * @pf: the PF
3387  **/
3388 static void i40e_pf_quiesce_all_vsi(struct i40e_pf *pf)
3389 {
3390         int v;
3391
3392         for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
3393                 if (pf->vsi[v])
3394                         i40e_quiesce_vsi(pf->vsi[v]);
3395         }
3396 }
3397
3398 /**
3399  * i40e_pf_unquiesce_all_vsi - Resume all VSIs on a PF
3400  * @pf: the PF
3401  **/
3402 static void i40e_pf_unquiesce_all_vsi(struct i40e_pf *pf)
3403 {
3404         int v;
3405
3406         for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
3407                 if (pf->vsi[v])
3408                         i40e_unquiesce_vsi(pf->vsi[v]);
3409         }
3410 }
3411
3412 /**
3413  * i40e_dcb_get_num_tc -  Get the number of TCs from DCBx config
3414  * @dcbcfg: the corresponding DCBx configuration structure
3415  *
3416  * Return the number of TCs from given DCBx configuration
3417  **/
3418 static u8 i40e_dcb_get_num_tc(struct i40e_dcbx_config *dcbcfg)
3419 {
3420         u8 num_tc = 0;
3421         int i;
3422
3423         /* Scan the ETS Config Priority Table to find
3424          * traffic class enabled for a given priority
3425          * and use the traffic class index to get the
3426          * number of traffic classes enabled
3427          */
3428         for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) {
3429                 if (dcbcfg->etscfg.prioritytable[i] > num_tc)
3430                         num_tc = dcbcfg->etscfg.prioritytable[i];
3431         }
3432
3433         /* Traffic class index starts from zero so
3434          * increment to return the actual count
3435          */
3436         return num_tc + 1;
3437 }
3438
3439 /**
3440  * i40e_dcb_get_enabled_tc - Get enabled traffic classes
3441  * @dcbcfg: the corresponding DCBx configuration structure
3442  *
3443  * Query the current DCB configuration and return the number of
3444  * traffic classes enabled from the given DCBX config
3445  **/
3446 static u8 i40e_dcb_get_enabled_tc(struct i40e_dcbx_config *dcbcfg)
3447 {
3448         u8 num_tc = i40e_dcb_get_num_tc(dcbcfg);
3449         u8 enabled_tc = 1;
3450         u8 i;
3451
3452         for (i = 0; i < num_tc; i++)
3453                 enabled_tc |= 1 << i;
3454
3455         return enabled_tc;
3456 }
3457
3458 /**
3459  * i40e_pf_get_num_tc - Get enabled traffic classes for PF
3460  * @pf: PF being queried
3461  *
3462  * Return number of traffic classes enabled for the given PF
3463  **/
3464 static u8 i40e_pf_get_num_tc(struct i40e_pf *pf)
3465 {
3466         struct i40e_hw *hw = &pf->hw;
3467         u8 i, enabled_tc;
3468         u8 num_tc = 0;
3469         struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
3470
3471         /* If DCB is not enabled then always in single TC */
3472         if (!(pf->flags & I40E_FLAG_DCB_ENABLED))
3473                 return 1;
3474
3475         /* MFP mode return count of enabled TCs for this PF */
3476         if (pf->flags & I40E_FLAG_MFP_ENABLED) {
3477                 enabled_tc = pf->hw.func_caps.enabled_tcmap;
3478                 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
3479                         if (enabled_tc & (1 << i))
3480                                 num_tc++;
3481                 }
3482                 return num_tc;
3483         }
3484
3485         /* SFP mode will be enabled for all TCs on port */
3486         return i40e_dcb_get_num_tc(dcbcfg);
3487 }
3488
3489 /**
3490  * i40e_pf_get_default_tc - Get bitmap for first enabled TC
3491  * @pf: PF being queried
3492  *
3493  * Return a bitmap for first enabled traffic class for this PF.
3494  **/
3495 static u8 i40e_pf_get_default_tc(struct i40e_pf *pf)
3496 {
3497         u8 enabled_tc = pf->hw.func_caps.enabled_tcmap;
3498         u8 i = 0;
3499
3500         if (!enabled_tc)
3501                 return 0x1; /* TC0 */
3502
3503         /* Find the first enabled TC */
3504         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
3505                 if (enabled_tc & (1 << i))
3506                         break;
3507         }
3508
3509         return 1 << i;
3510 }
3511
3512 /**
3513  * i40e_pf_get_pf_tc_map - Get bitmap for enabled traffic classes
3514  * @pf: PF being queried
3515  *
3516  * Return a bitmap for enabled traffic classes for this PF.
3517  **/
3518 static u8 i40e_pf_get_tc_map(struct i40e_pf *pf)
3519 {
3520         /* If DCB is not enabled for this PF then just return default TC */
3521         if (!(pf->flags & I40E_FLAG_DCB_ENABLED))
3522                 return i40e_pf_get_default_tc(pf);
3523
3524         /* MFP mode will have enabled TCs set by FW */
3525         if (pf->flags & I40E_FLAG_MFP_ENABLED)
3526                 return pf->hw.func_caps.enabled_tcmap;
3527
3528         /* SFP mode we want PF to be enabled for all TCs */
3529         return i40e_dcb_get_enabled_tc(&pf->hw.local_dcbx_config);
3530 }
3531
3532 /**
3533  * i40e_vsi_get_bw_info - Query VSI BW Information
3534  * @vsi: the VSI being queried
3535  *
3536  * Returns 0 on success, negative value on failure
3537  **/
3538 static int i40e_vsi_get_bw_info(struct i40e_vsi *vsi)
3539 {
3540         struct i40e_aqc_query_vsi_ets_sla_config_resp bw_ets_config = {0};
3541         struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0};
3542         struct i40e_pf *pf = vsi->back;
3543         struct i40e_hw *hw = &pf->hw;
3544         i40e_status aq_ret;
3545         u32 tc_bw_max;
3546         int i;
3547
3548         /* Get the VSI level BW configuration */
3549         aq_ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL);
3550         if (aq_ret) {
3551                 dev_info(&pf->pdev->dev,
3552                          "couldn't get pf vsi bw config, err %d, aq_err %d\n",
3553                          aq_ret, pf->hw.aq.asq_last_status);
3554                 return -EINVAL;
3555         }
3556
3557         /* Get the VSI level BW configuration per TC */
3558         aq_ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid, &bw_ets_config,
3559                                                   NULL);
3560         if (aq_ret) {
3561                 dev_info(&pf->pdev->dev,
3562                          "couldn't get pf vsi ets bw config, err %d, aq_err %d\n",
3563                          aq_ret, pf->hw.aq.asq_last_status);
3564                 return -EINVAL;
3565         }
3566
3567         if (bw_config.tc_valid_bits != bw_ets_config.tc_valid_bits) {
3568                 dev_info(&pf->pdev->dev,
3569                          "Enabled TCs mismatch from querying VSI BW info 0x%08x 0x%08x\n",
3570                          bw_config.tc_valid_bits,
3571                          bw_ets_config.tc_valid_bits);
3572                 /* Still continuing */
3573         }
3574
3575         vsi->bw_limit = le16_to_cpu(bw_config.port_bw_limit);
3576         vsi->bw_max_quanta = bw_config.max_bw;
3577         tc_bw_max = le16_to_cpu(bw_ets_config.tc_bw_max[0]) |
3578                     (le16_to_cpu(bw_ets_config.tc_bw_max[1]) << 16);
3579         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
3580                 vsi->bw_ets_share_credits[i] = bw_ets_config.share_credits[i];
3581                 vsi->bw_ets_limit_credits[i] =
3582                                         le16_to_cpu(bw_ets_config.credits[i]);
3583                 /* 3 bits out of 4 for each TC */
3584                 vsi->bw_ets_max_quanta[i] = (u8)((tc_bw_max >> (i*4)) & 0x7);
3585         }
3586
3587         return 0;
3588 }
3589
3590 /**
3591  * i40e_vsi_configure_bw_alloc - Configure VSI BW allocation per TC
3592  * @vsi: the VSI being configured
3593  * @enabled_tc: TC bitmap
3594  * @bw_credits: BW shared credits per TC
3595  *
3596  * Returns 0 on success, negative value on failure
3597  **/
3598 static int i40e_vsi_configure_bw_alloc(struct i40e_vsi *vsi, u8 enabled_tc,
3599                                        u8 *bw_share)
3600 {
3601         struct i40e_aqc_configure_vsi_tc_bw_data bw_data;
3602         i40e_status aq_ret;
3603         int i;
3604
3605         bw_data.tc_valid_bits = enabled_tc;
3606         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
3607                 bw_data.tc_bw_credits[i] = bw_share[i];
3608
3609         aq_ret = i40e_aq_config_vsi_tc_bw(&vsi->back->hw, vsi->seid, &bw_data,
3610                                           NULL);
3611         if (aq_ret) {
3612                 dev_info(&vsi->back->pdev->dev,
3613                          "%s: AQ command Config VSI BW allocation per TC failed = %d\n",
3614                          __func__, vsi->back->hw.aq.asq_last_status);
3615                 return -EINVAL;
3616         }
3617
3618         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
3619                 vsi->info.qs_handle[i] = bw_data.qs_handles[i];
3620
3621         return 0;
3622 }
3623
3624 /**
3625  * i40e_vsi_config_netdev_tc - Setup the netdev TC configuration
3626  * @vsi: the VSI being configured
3627  * @enabled_tc: TC map to be enabled
3628  *
3629  **/
3630 static void i40e_vsi_config_netdev_tc(struct i40e_vsi *vsi, u8 enabled_tc)
3631 {
3632         struct net_device *netdev = vsi->netdev;
3633         struct i40e_pf *pf = vsi->back;
3634         struct i40e_hw *hw = &pf->hw;
3635         u8 netdev_tc = 0;
3636         int i;
3637         struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
3638
3639         if (!netdev)
3640                 return;
3641
3642         if (!enabled_tc) {
3643                 netdev_reset_tc(netdev);
3644                 return;
3645         }
3646
3647         /* Set up actual enabled TCs on the VSI */
3648         if (netdev_set_num_tc(netdev, vsi->tc_config.numtc))
3649                 return;
3650
3651         /* set per TC queues for the VSI */
3652         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
3653                 /* Only set TC queues for enabled tcs
3654                  *
3655                  * e.g. For a VSI that has TC0 and TC3 enabled the
3656                  * enabled_tc bitmap would be 0x00001001; the driver
3657                  * will set the numtc for netdev as 2 that will be
3658                  * referenced by the netdev layer as TC 0 and 1.
3659                  */
3660                 if (vsi->tc_config.enabled_tc & (1 << i))
3661                         netdev_set_tc_queue(netdev,
3662                                         vsi->tc_config.tc_info[i].netdev_tc,
3663                                         vsi->tc_config.tc_info[i].qcount,
3664                                         vsi->tc_config.tc_info[i].qoffset);
3665         }
3666
3667         /* Assign UP2TC map for the VSI */
3668         for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) {
3669                 /* Get the actual TC# for the UP */
3670                 u8 ets_tc = dcbcfg->etscfg.prioritytable[i];
3671                 /* Get the mapped netdev TC# for the UP */
3672                 netdev_tc =  vsi->tc_config.tc_info[ets_tc].netdev_tc;
3673                 netdev_set_prio_tc_map(netdev, i, netdev_tc);
3674         }
3675 }
3676
3677 /**
3678  * i40e_vsi_update_queue_map - Update our copy of VSi info with new queue map
3679  * @vsi: the VSI being configured
3680  * @ctxt: the ctxt buffer returned from AQ VSI update param command
3681  **/
3682 static void i40e_vsi_update_queue_map(struct i40e_vsi *vsi,
3683                                       struct i40e_vsi_context *ctxt)
3684 {
3685         /* copy just the sections touched not the entire info
3686          * since not all sections are valid as returned by
3687          * update vsi params
3688          */
3689         vsi->info.mapping_flags = ctxt->info.mapping_flags;
3690         memcpy(&vsi->info.queue_mapping,
3691                &ctxt->info.queue_mapping, sizeof(vsi->info.queue_mapping));
3692         memcpy(&vsi->info.tc_mapping, ctxt->info.tc_mapping,
3693                sizeof(vsi->info.tc_mapping));
3694 }
3695
3696 /**
3697  * i40e_vsi_config_tc - Configure VSI Tx Scheduler for given TC map
3698  * @vsi: VSI to be configured
3699  * @enabled_tc: TC bitmap
3700  *
3701  * This configures a particular VSI for TCs that are mapped to the
3702  * given TC bitmap. It uses default bandwidth share for TCs across
3703  * VSIs to configure TC for a particular VSI.
3704  *
3705  * NOTE:
3706  * It is expected that the VSI queues have been quisced before calling
3707  * this function.
3708  **/
3709 static int i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 enabled_tc)
3710 {
3711         u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0};
3712         struct i40e_vsi_context ctxt;
3713         int ret = 0;
3714         int i;
3715
3716         /* Check if enabled_tc is same as existing or new TCs */
3717         if (vsi->tc_config.enabled_tc == enabled_tc)
3718                 return ret;
3719
3720         /* Enable ETS TCs with equal BW Share for now across all VSIs */
3721         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
3722                 if (enabled_tc & (1 << i))
3723                         bw_share[i] = 1;
3724         }
3725
3726         ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share);
3727         if (ret) {
3728                 dev_info(&vsi->back->pdev->dev,
3729                          "Failed configuring TC map %d for VSI %d\n",
3730                          enabled_tc, vsi->seid);
3731                 goto out;
3732         }
3733
3734         /* Update Queue Pairs Mapping for currently enabled UPs */
3735         ctxt.seid = vsi->seid;
3736         ctxt.pf_num = vsi->back->hw.pf_id;
3737         ctxt.vf_num = 0;
3738         ctxt.uplink_seid = vsi->uplink_seid;
3739         memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info));
3740         i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);
3741
3742         /* Update the VSI after updating the VSI queue-mapping information */
3743         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
3744         if (ret) {
3745                 dev_info(&vsi->back->pdev->dev,
3746                          "update vsi failed, aq_err=%d\n",
3747                          vsi->back->hw.aq.asq_last_status);
3748                 goto out;
3749         }
3750         /* update the local VSI info with updated queue map */
3751         i40e_vsi_update_queue_map(vsi, &ctxt);
3752         vsi->info.valid_sections = 0;
3753
3754         /* Update current VSI BW information */
3755         ret = i40e_vsi_get_bw_info(vsi);
3756         if (ret) {
3757                 dev_info(&vsi->back->pdev->dev,
3758                          "Failed updating vsi bw info, aq_err=%d\n",
3759                          vsi->back->hw.aq.asq_last_status);
3760                 goto out;
3761         }
3762
3763         /* Update the netdev TC setup */
3764         i40e_vsi_config_netdev_tc(vsi, enabled_tc);
3765 out:
3766         return ret;
3767 }
3768
3769 /**
3770  * i40e_up_complete - Finish the last steps of bringing up a connection
3771  * @vsi: the VSI being configured
3772  **/
3773 static int i40e_up_complete(struct i40e_vsi *vsi)
3774 {
3775         struct i40e_pf *pf = vsi->back;
3776         int err;
3777
3778         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
3779                 i40e_vsi_configure_msix(vsi);
3780         else
3781                 i40e_configure_msi_and_legacy(vsi);
3782
3783         /* start rings */
3784         err = i40e_vsi_control_rings(vsi, true);
3785         if (err)
3786                 return err;
3787
3788         clear_bit(__I40E_DOWN, &vsi->state);
3789         i40e_napi_enable_all(vsi);
3790         i40e_vsi_enable_irq(vsi);
3791
3792         if ((pf->hw.phy.link_info.link_info & I40E_AQ_LINK_UP) &&
3793             (vsi->netdev)) {
3794                 netdev_info(vsi->netdev, "NIC Link is Up\n");
3795                 netif_tx_start_all_queues(vsi->netdev);
3796                 netif_carrier_on(vsi->netdev);
3797         } else if (vsi->netdev) {
3798                 netdev_info(vsi->netdev, "NIC Link is Down\n");
3799         }
3800         i40e_service_event_schedule(pf);
3801
3802         return 0;
3803 }
3804
3805 /**
3806  * i40e_vsi_reinit_locked - Reset the VSI
3807  * @vsi: the VSI being configured
3808  *
3809  * Rebuild the ring structs after some configuration
3810  * has changed, e.g. MTU size.
3811  **/
3812 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi)
3813 {
3814         struct i40e_pf *pf = vsi->back;
3815
3816         WARN_ON(in_interrupt());
3817         while (test_and_set_bit(__I40E_CONFIG_BUSY, &pf->state))
3818                 usleep_range(1000, 2000);
3819         i40e_down(vsi);
3820
3821         /* Give a VF some time to respond to the reset.  The
3822          * two second wait is based upon the watchdog cycle in
3823          * the VF driver.
3824          */
3825         if (vsi->type == I40E_VSI_SRIOV)
3826                 msleep(2000);
3827         i40e_up(vsi);
3828         clear_bit(__I40E_CONFIG_BUSY, &pf->state);
3829 }
3830
3831 /**
3832  * i40e_up - Bring the connection back up after being down
3833  * @vsi: the VSI being configured
3834  **/
3835 int i40e_up(struct i40e_vsi *vsi)
3836 {
3837         int err;
3838
3839         err = i40e_vsi_configure(vsi);
3840         if (!err)
3841                 err = i40e_up_complete(vsi);
3842
3843         return err;
3844 }
3845
3846 /**
3847  * i40e_down - Shutdown the connection processing
3848  * @vsi: the VSI being stopped
3849  **/
3850 void i40e_down(struct i40e_vsi *vsi)
3851 {
3852         int i;
3853
3854         /* It is assumed that the caller of this function
3855          * sets the vsi->state __I40E_DOWN bit.
3856          */
3857         if (vsi->netdev) {
3858                 netif_carrier_off(vsi->netdev);
3859                 netif_tx_disable(vsi->netdev);
3860         }
3861         i40e_vsi_disable_irq(vsi);
3862         i40e_vsi_control_rings(vsi, false);
3863         i40e_napi_disable_all(vsi);
3864
3865         for (i = 0; i < vsi->num_queue_pairs; i++) {
3866                 i40e_clean_tx_ring(vsi->tx_rings[i]);
3867                 i40e_clean_rx_ring(vsi->rx_rings[i]);
3868         }
3869 }
3870
3871 /**
3872  * i40e_setup_tc - configure multiple traffic classes
3873  * @netdev: net device to configure
3874  * @tc: number of traffic classes to enable
3875  **/
3876 static int i40e_setup_tc(struct net_device *netdev, u8 tc)
3877 {
3878         struct i40e_netdev_priv *np = netdev_priv(netdev);
3879         struct i40e_vsi *vsi = np->vsi;
3880         struct i40e_pf *pf = vsi->back;
3881         u8 enabled_tc = 0;
3882         int ret = -EINVAL;
3883         int i;
3884
3885         /* Check if DCB enabled to continue */
3886         if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) {
3887                 netdev_info(netdev, "DCB is not enabled for adapter\n");
3888                 goto exit;
3889         }
3890
3891         /* Check if MFP enabled */
3892         if (pf->flags & I40E_FLAG_MFP_ENABLED) {
3893                 netdev_info(netdev, "Configuring TC not supported in MFP mode\n");
3894                 goto exit;
3895         }
3896
3897         /* Check whether tc count is within enabled limit */
3898         if (tc > i40e_pf_get_num_tc(pf)) {
3899                 netdev_info(netdev, "TC count greater than enabled on link for adapter\n");
3900                 goto exit;
3901         }
3902
3903         /* Generate TC map for number of tc requested */
3904         for (i = 0; i < tc; i++)
3905                 enabled_tc |= (1 << i);
3906
3907         /* Requesting same TC configuration as already enabled */
3908         if (enabled_tc == vsi->tc_config.enabled_tc)
3909                 return 0;
3910
3911         /* Quiesce VSI queues */
3912         i40e_quiesce_vsi(vsi);
3913
3914         /* Configure VSI for enabled TCs */
3915         ret = i40e_vsi_config_tc(vsi, enabled_tc);
3916         if (ret) {
3917                 netdev_info(netdev, "Failed configuring TC for VSI seid=%d\n",
3918                             vsi->seid);
3919                 goto exit;
3920         }
3921
3922         /* Unquiesce VSI */
3923         i40e_unquiesce_vsi(vsi);
3924
3925 exit:
3926         return ret;
3927 }
3928
3929 /**
3930  * i40e_open - Called when a network interface is made active
3931  * @netdev: network interface device structure
3932  *
3933  * The open entry point is called when a network interface is made
3934  * active by the system (IFF_UP).  At this point all resources needed
3935  * for transmit and receive operations are allocated, the interrupt
3936  * handler is registered with the OS, the netdev watchdog subtask is
3937  * enabled, and the stack is notified that the interface is ready.
3938  *
3939  * Returns 0 on success, negative value on failure
3940  **/
3941 static int i40e_open(struct net_device *netdev)
3942 {
3943         struct i40e_netdev_priv *np = netdev_priv(netdev);
3944         struct i40e_vsi *vsi = np->vsi;
3945         struct i40e_pf *pf = vsi->back;
3946         char int_name[IFNAMSIZ];
3947         int err;
3948
3949         /* disallow open during test */
3950         if (test_bit(__I40E_TESTING, &pf->state))
3951                 return -EBUSY;
3952
3953         netif_carrier_off(netdev);
3954
3955         /* allocate descriptors */
3956         err = i40e_vsi_setup_tx_resources(vsi);
3957         if (err)
3958                 goto err_setup_tx;
3959         err = i40e_vsi_setup_rx_resources(vsi);
3960         if (err)
3961                 goto err_setup_rx;
3962
3963         err = i40e_vsi_configure(vsi);
3964         if (err)
3965                 goto err_setup_rx;
3966
3967         snprintf(int_name, sizeof(int_name) - 1, "%s-%s",
3968                  dev_driver_string(&pf->pdev->dev), netdev->name);
3969         err = i40e_vsi_request_irq(vsi, int_name);
3970         if (err)
3971                 goto err_setup_rx;
3972
3973         /* Notify the stack of the actual queue counts. */
3974         err = netif_set_real_num_tx_queues(netdev, vsi->num_queue_pairs);
3975         if (err)
3976                 goto err_set_queues;
3977
3978         err = netif_set_real_num_rx_queues(netdev, vsi->num_queue_pairs);
3979         if (err)
3980                 goto err_set_queues;
3981
3982         err = i40e_up_complete(vsi);
3983         if (err)
3984                 goto err_up_complete;
3985
3986 #ifdef CONFIG_I40E_VXLAN
3987         vxlan_get_rx_port(netdev);
3988 #endif
3989
3990         return 0;
3991
3992 err_up_complete:
3993         i40e_down(vsi);
3994 err_set_queues:
3995         i40e_vsi_free_irq(vsi);
3996 err_setup_rx:
3997         i40e_vsi_free_rx_resources(vsi);
3998 err_setup_tx:
3999         i40e_vsi_free_tx_resources(vsi);
4000         if (vsi == pf->vsi[pf->lan_vsi])
4001                 i40e_do_reset(pf, (1 << __I40E_PF_RESET_REQUESTED));
4002
4003         return err;
4004 }
4005
4006 /**
4007  * i40e_close - Disables a network interface
4008  * @netdev: network interface device structure
4009  *
4010  * The close entry point is called when an interface is de-activated
4011  * by the OS.  The hardware is still under the driver's control, but
4012  * this netdev interface is disabled.
4013  *
4014  * Returns 0, this is not allowed to fail
4015  **/
4016 static int i40e_close(struct net_device *netdev)
4017 {
4018         struct i40e_netdev_priv *np = netdev_priv(netdev);
4019         struct i40e_vsi *vsi = np->vsi;
4020
4021         if (test_and_set_bit(__I40E_DOWN, &vsi->state))
4022                 return 0;
4023
4024         i40e_down(vsi);
4025         i40e_vsi_free_irq(vsi);
4026
4027         i40e_vsi_free_tx_resources(vsi);
4028         i40e_vsi_free_rx_resources(vsi);
4029
4030         return 0;
4031 }
4032
4033 /**
4034  * i40e_do_reset - Start a PF or Core Reset sequence
4035  * @pf: board private structure
4036  * @reset_flags: which reset is requested
4037  *
4038  * The essential difference in resets is that the PF Reset
4039  * doesn't clear the packet buffers, doesn't reset the PE
4040  * firmware, and doesn't bother the other PFs on the chip.
4041  **/
4042 void i40e_do_reset(struct i40e_pf *pf, u32 reset_flags)
4043 {
4044         u32 val;
4045
4046         WARN_ON(in_interrupt());
4047
4048         /* do the biggest reset indicated */
4049         if (reset_flags & (1 << __I40E_GLOBAL_RESET_REQUESTED)) {
4050
4051                 /* Request a Global Reset
4052                  *
4053                  * This will start the chip's countdown to the actual full
4054                  * chip reset event, and a warning interrupt to be sent
4055                  * to all PFs, including the requestor.  Our handler
4056                  * for the warning interrupt will deal with the shutdown
4057                  * and recovery of the switch setup.
4058                  */
4059                 dev_info(&pf->pdev->dev, "GlobalR requested\n");
4060                 val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
4061                 val |= I40E_GLGEN_RTRIG_GLOBR_MASK;
4062                 wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
4063
4064         } else if (reset_flags & (1 << __I40E_CORE_RESET_REQUESTED)) {
4065
4066                 /* Request a Core Reset
4067                  *
4068                  * Same as Global Reset, except does *not* include the MAC/PHY
4069                  */
4070                 dev_info(&pf->pdev->dev, "CoreR requested\n");
4071                 val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
4072                 val |= I40E_GLGEN_RTRIG_CORER_MASK;
4073                 wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
4074                 i40e_flush(&pf->hw);
4075
4076         } else if (reset_flags & (1 << __I40E_EMP_RESET_REQUESTED)) {
4077
4078                 /* Request a Firmware Reset
4079                  *
4080                  * Same as Global reset, plus restarting the
4081                  * embedded firmware engine.
4082                  */
4083                 /* enable EMP Reset */
4084                 val = rd32(&pf->hw, I40E_GLGEN_RSTENA_EMP);
4085                 val |= I40E_GLGEN_RSTENA_EMP_EMP_RST_ENA_MASK;
4086                 wr32(&pf->hw, I40E_GLGEN_RSTENA_EMP, val);
4087
4088                 /* force the reset */
4089                 val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
4090                 val |= I40E_GLGEN_RTRIG_EMPFWR_MASK;
4091                 wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
4092                 i40e_flush(&pf->hw);
4093
4094         } else if (reset_flags & (1 << __I40E_PF_RESET_REQUESTED)) {
4095
4096                 /* Request a PF Reset
4097                  *
4098                  * Resets only the PF-specific registers
4099                  *
4100                  * This goes directly to the tear-down and rebuild of
4101                  * the switch, since we need to do all the recovery as
4102                  * for the Core Reset.
4103                  */
4104                 dev_info(&pf->pdev->dev, "PFR requested\n");
4105                 i40e_handle_reset_warning(pf);
4106
4107         } else if (reset_flags & (1 << __I40E_REINIT_REQUESTED)) {
4108                 int v;
4109
4110                 /* Find the VSI(s) that requested a re-init */
4111                 dev_info(&pf->pdev->dev,
4112                          "VSI reinit requested\n");
4113                 for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
4114                         struct i40e_vsi *vsi = pf->vsi[v];
4115                         if (vsi != NULL &&
4116                             test_bit(__I40E_REINIT_REQUESTED, &vsi->state)) {
4117                                 i40e_vsi_reinit_locked(pf->vsi[v]);
4118                                 clear_bit(__I40E_REINIT_REQUESTED, &vsi->state);
4119                         }
4120                 }
4121
4122                 /* no further action needed, so return now */
4123                 return;
4124         } else {
4125                 dev_info(&pf->pdev->dev,
4126                          "bad reset request 0x%08x\n", reset_flags);
4127                 return;
4128         }
4129 }
4130
4131 /**
4132  * i40e_do_reset_safe - Protected reset path for userland calls.
4133  * @pf: board private structure
4134  * @reset_flags: which reset is requested
4135  *
4136  **/
4137 void i40e_do_reset_safe(struct i40e_pf *pf, u32 reset_flags)
4138 {
4139         rtnl_lock();
4140         i40e_do_reset(pf, reset_flags);
4141         rtnl_unlock();
4142 }
4143
4144 /**
4145  * i40e_handle_lan_overflow_event - Handler for LAN queue overflow event
4146  * @pf: board private structure
4147  * @e: event info posted on ARQ
4148  *
4149  * Handler for LAN Queue Overflow Event generated by the firmware for PF
4150  * and VF queues
4151  **/
4152 static void i40e_handle_lan_overflow_event(struct i40e_pf *pf,
4153                                            struct i40e_arq_event_info *e)
4154 {
4155         struct i40e_aqc_lan_overflow *data =
4156                 (struct i40e_aqc_lan_overflow *)&e->desc.params.raw;
4157         u32 queue = le32_to_cpu(data->prtdcb_rupto);
4158         u32 qtx_ctl = le32_to_cpu(data->otx_ctl);
4159         struct i40e_hw *hw = &pf->hw;
4160         struct i40e_vf *vf;
4161         u16 vf_id;
4162
4163         dev_info(&pf->pdev->dev, "%s: Rx Queue Number = %d QTX_CTL=0x%08x\n",
4164                  __func__, queue, qtx_ctl);
4165
4166         /* Queue belongs to VF, find the VF and issue VF reset */
4167         if (((qtx_ctl & I40E_QTX_CTL_PFVF_Q_MASK)
4168             >> I40E_QTX_CTL_PFVF_Q_SHIFT) == I40E_QTX_CTL_VF_QUEUE) {
4169                 vf_id = (u16)((qtx_ctl & I40E_QTX_CTL_VFVM_INDX_MASK)
4170                          >> I40E_QTX_CTL_VFVM_INDX_SHIFT);
4171                 vf_id -= hw->func_caps.vf_base_id;
4172                 vf = &pf->vf[vf_id];
4173                 i40e_vc_notify_vf_reset(vf);
4174                 /* Allow VF to process pending reset notification */
4175                 msleep(20);
4176                 i40e_reset_vf(vf, false);
4177         }
4178 }
4179
4180 /**
4181  * i40e_service_event_complete - Finish up the service event
4182  * @pf: board private structure
4183  **/
4184 static void i40e_service_event_complete(struct i40e_pf *pf)
4185 {
4186         BUG_ON(!test_bit(__I40E_SERVICE_SCHED, &pf->state));
4187
4188         /* flush memory to make sure state is correct before next watchog */
4189         smp_mb__before_clear_bit();
4190         clear_bit(__I40E_SERVICE_SCHED, &pf->state);
4191 }
4192
4193 /**
4194  * i40e_fdir_reinit_subtask - Worker thread to reinit FDIR filter table
4195  * @pf: board private structure
4196  **/
4197 static void i40e_fdir_reinit_subtask(struct i40e_pf *pf)
4198 {
4199         if (!(pf->flags & I40E_FLAG_FDIR_REQUIRES_REINIT))
4200                 return;
4201
4202         pf->flags &= ~I40E_FLAG_FDIR_REQUIRES_REINIT;
4203
4204         /* if interface is down do nothing */
4205         if (test_bit(__I40E_DOWN, &pf->state))
4206                 return;
4207 }
4208
4209 /**
4210  * i40e_vsi_link_event - notify VSI of a link event
4211  * @vsi: vsi to be notified
4212  * @link_up: link up or down
4213  **/
4214 static void i40e_vsi_link_event(struct i40e_vsi *vsi, bool link_up)
4215 {
4216         if (!vsi)
4217                 return;
4218
4219         switch (vsi->type) {
4220         case I40E_VSI_MAIN:
4221                 if (!vsi->netdev || !vsi->netdev_registered)
4222                         break;
4223
4224                 if (link_up) {
4225                         netif_carrier_on(vsi->netdev);
4226                         netif_tx_wake_all_queues(vsi->netdev);
4227                 } else {
4228                         netif_carrier_off(vsi->netdev);
4229                         netif_tx_stop_all_queues(vsi->netdev);
4230                 }
4231                 break;
4232
4233         case I40E_VSI_SRIOV:
4234                 break;
4235
4236         case I40E_VSI_VMDQ2:
4237         case I40E_VSI_CTRL:
4238         case I40E_VSI_MIRROR:
4239         default:
4240                 /* there is no notification for other VSIs */
4241                 break;
4242         }
4243 }
4244
4245 /**
4246  * i40e_veb_link_event - notify elements on the veb of a link event
4247  * @veb: veb to be notified
4248  * @link_up: link up or down
4249  **/
4250 static void i40e_veb_link_event(struct i40e_veb *veb, bool link_up)
4251 {
4252         struct i40e_pf *pf;
4253         int i;
4254
4255         if (!veb || !veb->pf)
4256                 return;
4257         pf = veb->pf;
4258
4259         /* depth first... */
4260         for (i = 0; i < I40E_MAX_VEB; i++)
4261                 if (pf->veb[i] && (pf->veb[i]->uplink_seid == veb->seid))
4262                         i40e_veb_link_event(pf->veb[i], link_up);
4263
4264         /* ... now the local VSIs */
4265         for (i = 0; i < pf->hw.func_caps.num_vsis; i++)
4266                 if (pf->vsi[i] && (pf->vsi[i]->uplink_seid == veb->seid))
4267                         i40e_vsi_link_event(pf->vsi[i], link_up);
4268 }
4269
4270 /**
4271  * i40e_link_event - Update netif_carrier status
4272  * @pf: board private structure
4273  **/
4274 static void i40e_link_event(struct i40e_pf *pf)
4275 {
4276         bool new_link, old_link;
4277
4278         new_link = (pf->hw.phy.link_info.link_info & I40E_AQ_LINK_UP);
4279         old_link = (pf->hw.phy.link_info_old.link_info & I40E_AQ_LINK_UP);
4280
4281         if (new_link == old_link)
4282                 return;
4283
4284         if (!test_bit(__I40E_DOWN, &pf->vsi[pf->lan_vsi]->state))
4285                 netdev_info(pf->vsi[pf->lan_vsi]->netdev,
4286                             "NIC Link is %s\n", (new_link ? "Up" : "Down"));
4287
4288         /* Notify the base of the switch tree connected to
4289          * the link.  Floating VEBs are not notified.
4290          */
4291         if (pf->lan_veb != I40E_NO_VEB && pf->veb[pf->lan_veb])
4292                 i40e_veb_link_event(pf->veb[pf->lan_veb], new_link);
4293         else
4294                 i40e_vsi_link_event(pf->vsi[pf->lan_vsi], new_link);
4295
4296         if (pf->vf)
4297                 i40e_vc_notify_link_state(pf);
4298 }
4299
4300 /**
4301  * i40e_check_hang_subtask - Check for hung queues and dropped interrupts
4302  * @pf: board private structure
4303  *
4304  * Set the per-queue flags to request a check for stuck queues in the irq
4305  * clean functions, then force interrupts to be sure the irq clean is called.
4306  **/
4307 static void i40e_check_hang_subtask(struct i40e_pf *pf)
4308 {
4309         int i, v;
4310
4311         /* If we're down or resetting, just bail */
4312         if (test_bit(__I40E_CONFIG_BUSY, &pf->state))
4313                 return;
4314
4315         /* for each VSI/netdev
4316          *     for each Tx queue
4317          *         set the check flag
4318          *     for each q_vector
4319          *         force an interrupt
4320          */
4321         for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
4322                 struct i40e_vsi *vsi = pf->vsi[v];
4323                 int armed = 0;
4324
4325                 if (!pf->vsi[v] ||
4326                     test_bit(__I40E_DOWN, &vsi->state) ||
4327                     (vsi->netdev && !netif_carrier_ok(vsi->netdev)))
4328                         continue;
4329
4330                 for (i = 0; i < vsi->num_queue_pairs; i++) {
4331                         set_check_for_tx_hang(vsi->tx_rings[i]);
4332                         if (test_bit(__I40E_HANG_CHECK_ARMED,
4333                                      &vsi->tx_rings[i]->state))
4334                                 armed++;
4335                 }
4336
4337                 if (armed) {
4338                         if (!(pf->flags & I40E_FLAG_MSIX_ENABLED)) {
4339                                 wr32(&vsi->back->hw, I40E_PFINT_DYN_CTL0,
4340                                      (I40E_PFINT_DYN_CTL0_INTENA_MASK |
4341                                       I40E_PFINT_DYN_CTL0_SWINT_TRIG_MASK));
4342                         } else {
4343                                 u16 vec = vsi->base_vector - 1;
4344                                 u32 val = (I40E_PFINT_DYN_CTLN_INTENA_MASK |
4345                                            I40E_PFINT_DYN_CTLN_SWINT_TRIG_MASK);
4346                                 for (i = 0; i < vsi->num_q_vectors; i++, vec++)
4347                                         wr32(&vsi->back->hw,
4348                                              I40E_PFINT_DYN_CTLN(vec), val);
4349                         }
4350                         i40e_flush(&vsi->back->hw);
4351                 }
4352         }
4353 }
4354
4355 /**
4356  * i40e_watchdog_subtask - Check and bring link up
4357  * @pf: board private structure
4358  **/
4359 static void i40e_watchdog_subtask(struct i40e_pf *pf)
4360 {
4361         int i;
4362
4363         /* if interface is down do nothing */
4364         if (test_bit(__I40E_DOWN, &pf->state) ||
4365             test_bit(__I40E_CONFIG_BUSY, &pf->state))
4366                 return;
4367
4368         /* Update the stats for active netdevs so the network stack
4369          * can look at updated numbers whenever it cares to
4370          */
4371         for (i = 0; i < pf->hw.func_caps.num_vsis; i++)
4372                 if (pf->vsi[i] && pf->vsi[i]->netdev)
4373                         i40e_update_stats(pf->vsi[i]);
4374
4375         /* Update the stats for the active switching components */
4376         for (i = 0; i < I40E_MAX_VEB; i++)
4377                 if (pf->veb[i])
4378                         i40e_update_veb_stats(pf->veb[i]);
4379 }
4380
4381 /**
4382  * i40e_reset_subtask - Set up for resetting the device and driver
4383  * @pf: board private structure
4384  **/
4385 static void i40e_reset_subtask(struct i40e_pf *pf)
4386 {
4387         u32 reset_flags = 0;
4388
4389         rtnl_lock();
4390         if (test_bit(__I40E_REINIT_REQUESTED, &pf->state)) {
4391                 reset_flags |= (1 << __I40E_REINIT_REQUESTED);
4392                 clear_bit(__I40E_REINIT_REQUESTED, &pf->state);
4393         }
4394         if (test_bit(__I40E_PF_RESET_REQUESTED, &pf->state)) {
4395                 reset_flags |= (1 << __I40E_PF_RESET_REQUESTED);
4396                 clear_bit(__I40E_PF_RESET_REQUESTED, &pf->state);
4397         }
4398         if (test_bit(__I40E_CORE_RESET_REQUESTED, &pf->state)) {
4399                 reset_flags |= (1 << __I40E_CORE_RESET_REQUESTED);
4400                 clear_bit(__I40E_CORE_RESET_REQUESTED, &pf->state);
4401         }
4402         if (test_bit(__I40E_GLOBAL_RESET_REQUESTED, &pf->state)) {
4403                 reset_flags |= (1 << __I40E_GLOBAL_RESET_REQUESTED);
4404                 clear_bit(__I40E_GLOBAL_RESET_REQUESTED, &pf->state);
4405         }
4406
4407         /* If there's a recovery already waiting, it takes
4408          * precedence before starting a new reset sequence.
4409          */
4410         if (test_bit(__I40E_RESET_INTR_RECEIVED, &pf->state)) {
4411                 i40e_handle_reset_warning(pf);
4412                 goto unlock;
4413         }
4414
4415         /* If we're already down or resetting, just bail */
4416         if (reset_flags &&
4417             !test_bit(__I40E_DOWN, &pf->state) &&
4418             !test_bit(__I40E_CONFIG_BUSY, &pf->state))
4419                 i40e_do_reset(pf, reset_flags);
4420
4421 unlock:
4422         rtnl_unlock();
4423 }
4424
4425 /**
4426  * i40e_handle_link_event - Handle link event
4427  * @pf: board private structure
4428  * @e: event info posted on ARQ
4429  **/
4430 static void i40e_handle_link_event(struct i40e_pf *pf,
4431                                    struct i40e_arq_event_info *e)
4432 {
4433         struct i40e_hw *hw = &pf->hw;
4434         struct i40e_aqc_get_link_status *status =
4435                 (struct i40e_aqc_get_link_status *)&e->desc.params.raw;
4436         struct i40e_link_status *hw_link_info = &hw->phy.link_info;
4437
4438         /* save off old link status information */
4439         memcpy(&pf->hw.phy.link_info_old, hw_link_info,
4440                sizeof(pf->hw.phy.link_info_old));
4441
4442         /* update link status */
4443         hw_link_info->phy_type = (enum i40e_aq_phy_type)status->phy_type;
4444         hw_link_info->link_speed = (enum i40e_aq_link_speed)status->link_speed;
4445         hw_link_info->link_info = status->link_info;
4446         hw_link_info->an_info = status->an_info;
4447         hw_link_info->ext_info = status->ext_info;
4448         hw_link_info->lse_enable =
4449                 le16_to_cpu(status->command_flags) &
4450                             I40E_AQ_LSE_ENABLE;
4451
4452         /* process the event */
4453         i40e_link_event(pf);
4454
4455         /* Do a new status request to re-enable LSE reporting
4456          * and load new status information into the hw struct,
4457          * then see if the status changed while processing the
4458          * initial event.
4459          */
4460         i40e_aq_get_link_info(&pf->hw, true, NULL, NULL);
4461         i40e_link_event(pf);
4462 }
4463
4464 /**
4465  * i40e_clean_adminq_subtask - Clean the AdminQ rings
4466  * @pf: board private structure
4467  **/
4468 static void i40e_clean_adminq_subtask(struct i40e_pf *pf)
4469 {
4470         struct i40e_arq_event_info event;
4471         struct i40e_hw *hw = &pf->hw;
4472         u16 pending, i = 0;
4473         i40e_status ret;
4474         u16 opcode;
4475         u32 val;
4476
4477         if (!test_bit(__I40E_ADMINQ_EVENT_PENDING, &pf->state))
4478                 return;
4479
4480         event.msg_size = I40E_MAX_AQ_BUF_SIZE;
4481         event.msg_buf = kzalloc(event.msg_size, GFP_KERNEL);
4482         if (!event.msg_buf)
4483                 return;
4484
4485         do {
4486                 event.msg_size = I40E_MAX_AQ_BUF_SIZE; /* reinit each time */
4487                 ret = i40e_clean_arq_element(hw, &event, &pending);
4488                 if (ret == I40E_ERR_ADMIN_QUEUE_NO_WORK) {
4489                         dev_info(&pf->pdev->dev, "No ARQ event found\n");
4490                         break;
4491                 } else if (ret) {
4492                         dev_info(&pf->pdev->dev, "ARQ event error %d\n", ret);
4493                         break;
4494                 }
4495
4496                 opcode = le16_to_cpu(event.desc.opcode);
4497                 switch (opcode) {
4498
4499                 case i40e_aqc_opc_get_link_status:
4500                         i40e_handle_link_event(pf, &event);
4501                         break;
4502                 case i40e_aqc_opc_send_msg_to_pf:
4503                         ret = i40e_vc_process_vf_msg(pf,
4504                                         le16_to_cpu(event.desc.retval),
4505                                         le32_to_cpu(event.desc.cookie_high),
4506                                         le32_to_cpu(event.desc.cookie_low),
4507                                         event.msg_buf,
4508                                         event.msg_size);
4509                         break;
4510                 case i40e_aqc_opc_lldp_update_mib:
4511                         dev_info(&pf->pdev->dev, "ARQ: Update LLDP MIB event received\n");
4512                         break;
4513                 case i40e_aqc_opc_event_lan_overflow:
4514                         dev_info(&pf->pdev->dev, "ARQ LAN queue overflow event received\n");
4515                         i40e_handle_lan_overflow_event(pf, &event);
4516                         break;
4517                 default:
4518                         dev_info(&pf->pdev->dev,
4519                                  "ARQ Error: Unknown event %d received\n",
4520                                  event.desc.opcode);
4521                         break;
4522                 }
4523         } while (pending && (i++ < pf->adminq_work_limit));
4524
4525         clear_bit(__I40E_ADMINQ_EVENT_PENDING, &pf->state);
4526         /* re-enable Admin queue interrupt cause */
4527         val = rd32(hw, I40E_PFINT_ICR0_ENA);
4528         val |=  I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
4529         wr32(hw, I40E_PFINT_ICR0_ENA, val);
4530         i40e_flush(hw);
4531
4532         kfree(event.msg_buf);
4533 }
4534
4535 /**
4536  * i40e_reconstitute_veb - rebuild the VEB and anything connected to it
4537  * @veb: pointer to the VEB instance
4538  *
4539  * This is a recursive function that first builds the attached VSIs then
4540  * recurses in to build the next layer of VEB.  We track the connections
4541  * through our own index numbers because the seid's from the HW could
4542  * change across the reset.
4543  **/
4544 static int i40e_reconstitute_veb(struct i40e_veb *veb)
4545 {
4546         struct i40e_vsi *ctl_vsi = NULL;
4547         struct i40e_pf *pf = veb->pf;
4548         int v, veb_idx;
4549         int ret;
4550
4551         /* build VSI that owns this VEB, temporarily attached to base VEB */
4552         for (v = 0; v < pf->hw.func_caps.num_vsis && !ctl_vsi; v++) {
4553                 if (pf->vsi[v] &&
4554                     pf->vsi[v]->veb_idx == veb->idx &&
4555                     pf->vsi[v]->flags & I40E_VSI_FLAG_VEB_OWNER) {
4556                         ctl_vsi = pf->vsi[v];
4557                         break;
4558                 }
4559         }
4560         if (!ctl_vsi) {
4561                 dev_info(&pf->pdev->dev,
4562                          "missing owner VSI for veb_idx %d\n", veb->idx);
4563                 ret = -ENOENT;
4564                 goto end_reconstitute;
4565         }
4566         if (ctl_vsi != pf->vsi[pf->lan_vsi])
4567                 ctl_vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid;
4568         ret = i40e_add_vsi(ctl_vsi);
4569         if (ret) {
4570                 dev_info(&pf->pdev->dev,
4571                          "rebuild of owner VSI failed: %d\n", ret);
4572                 goto end_reconstitute;
4573         }
4574         i40e_vsi_reset_stats(ctl_vsi);
4575
4576         /* create the VEB in the switch and move the VSI onto the VEB */
4577         ret = i40e_add_veb(veb, ctl_vsi);
4578         if (ret)
4579                 goto end_reconstitute;
4580
4581         /* create the remaining VSIs attached to this VEB */
4582         for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
4583                 if (!pf->vsi[v] || pf->vsi[v] == ctl_vsi)
4584                         continue;
4585
4586                 if (pf->vsi[v]->veb_idx == veb->idx) {
4587                         struct i40e_vsi *vsi = pf->vsi[v];
4588                         vsi->uplink_seid = veb->seid;
4589                         ret = i40e_add_vsi(vsi);
4590                         if (ret) {
4591                                 dev_info(&pf->pdev->dev,
4592                                          "rebuild of vsi_idx %d failed: %d\n",
4593                                          v, ret);
4594                                 goto end_reconstitute;
4595                         }
4596                         i40e_vsi_reset_stats(vsi);
4597                 }
4598         }
4599
4600         /* create any VEBs attached to this VEB - RECURSION */
4601         for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) {
4602                 if (pf->veb[veb_idx] && pf->veb[veb_idx]->veb_idx == veb->idx) {
4603                         pf->veb[veb_idx]->uplink_seid = veb->seid;
4604                         ret = i40e_reconstitute_veb(pf->veb[veb_idx]);
4605                         if (ret)
4606                                 break;
4607                 }
4608         }
4609
4610 end_reconstitute:
4611         return ret;
4612 }
4613
4614 /**
4615  * i40e_get_capabilities - get info about the HW
4616  * @pf: the PF struct
4617  **/
4618 static int i40e_get_capabilities(struct i40e_pf *pf)
4619 {
4620         struct i40e_aqc_list_capabilities_element_resp *cap_buf;
4621         u16 data_size;
4622         int buf_len;
4623         int err;
4624
4625         buf_len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp);
4626         do {
4627                 cap_buf = kzalloc(buf_len, GFP_KERNEL);
4628                 if (!cap_buf)
4629                         return -ENOMEM;
4630
4631                 /* this loads the data into the hw struct for us */
4632                 err = i40e_aq_discover_capabilities(&pf->hw, cap_buf, buf_len,
4633                                             &data_size,
4634                                             i40e_aqc_opc_list_func_capabilities,
4635                                             NULL);
4636                 /* data loaded, buffer no longer needed */
4637                 kfree(cap_buf);
4638
4639                 if (pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) {
4640                         /* retry with a larger buffer */
4641                         buf_len = data_size;
4642                 } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK) {
4643                         dev_info(&pf->pdev->dev,
4644                                  "capability discovery failed: aq=%d\n",
4645                                  pf->hw.aq.asq_last_status);
4646                         return -ENODEV;
4647                 }
4648         } while (err);
4649
4650         if (pf->hw.revision_id == 0 && pf->hw.func_caps.npar_enable) {
4651                 pf->hw.func_caps.num_msix_vectors += 1;
4652                 pf->hw.func_caps.num_tx_qp =
4653                         min_t(int, pf->hw.func_caps.num_tx_qp,
4654                               I40E_MAX_NPAR_QPS);
4655         }
4656
4657         if (pf->hw.debug_mask & I40E_DEBUG_USER)
4658                 dev_info(&pf->pdev->dev,
4659                          "pf=%d, num_vfs=%d, msix_pf=%d, msix_vf=%d, fd_g=%d, fd_b=%d, pf_max_q=%d num_vsi=%d\n",
4660                          pf->hw.pf_id, pf->hw.func_caps.num_vfs,
4661                          pf->hw.func_caps.num_msix_vectors,
4662                          pf->hw.func_caps.num_msix_vectors_vf,
4663                          pf->hw.func_caps.fd_filters_guaranteed,
4664                          pf->hw.func_caps.fd_filters_best_effort,
4665                          pf->hw.func_caps.num_tx_qp,
4666                          pf->hw.func_caps.num_vsis);
4667
4668 #define DEF_NUM_VSI (1 + (pf->hw.func_caps.fcoe ? 1 : 0) \
4669                        + pf->hw.func_caps.num_vfs)
4670         if (pf->hw.revision_id == 0 && (DEF_NUM_VSI > pf->hw.func_caps.num_vsis)) {
4671                 dev_info(&pf->pdev->dev,
4672                          "got num_vsis %d, setting num_vsis to %d\n",
4673                          pf->hw.func_caps.num_vsis, DEF_NUM_VSI);
4674                 pf->hw.func_caps.num_vsis = DEF_NUM_VSI;
4675         }
4676
4677         return 0;
4678 }
4679
4680 /**
4681  * i40e_fdir_setup - initialize the Flow Director resources
4682  * @pf: board private structure
4683  **/
4684 static void i40e_fdir_setup(struct i40e_pf *pf)
4685 {
4686         struct i40e_vsi *vsi;
4687         bool new_vsi = false;
4688         int err, i;
4689
4690         if (!(pf->flags & (I40E_FLAG_FDIR_ENABLED |
4691                            I40E_FLAG_FDIR_ATR_ENABLED)))
4692                 return;
4693
4694         pf->atr_sample_rate = I40E_DEFAULT_ATR_SAMPLE_RATE;
4695
4696         /* find existing or make new FDIR VSI */
4697         vsi = NULL;
4698         for (i = 0; i < pf->hw.func_caps.num_vsis; i++)
4699                 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR)
4700                         vsi = pf->vsi[i];
4701         if (!vsi) {
4702                 vsi = i40e_vsi_setup(pf, I40E_VSI_FDIR, pf->mac_seid, 0);
4703                 if (!vsi) {
4704                         dev_info(&pf->pdev->dev, "Couldn't create FDir VSI\n");
4705                         pf->flags &= ~I40E_FLAG_FDIR_ENABLED;
4706                         return;
4707                 }
4708                 new_vsi = true;
4709         }
4710         WARN_ON(vsi->base_queue != I40E_FDIR_RING);
4711         i40e_vsi_setup_irqhandler(vsi, i40e_fdir_clean_rings);
4712
4713         err = i40e_vsi_setup_tx_resources(vsi);
4714         if (!err)
4715                 err = i40e_vsi_setup_rx_resources(vsi);
4716         if (!err)
4717                 err = i40e_vsi_configure(vsi);
4718         if (!err && new_vsi) {
4719                 char int_name[IFNAMSIZ + 9];
4720                 snprintf(int_name, sizeof(int_name) - 1, "%s-fdir",
4721                          dev_driver_string(&pf->pdev->dev));
4722                 err = i40e_vsi_request_irq(vsi, int_name);
4723         }
4724         if (!err)
4725                 err = i40e_up_complete(vsi);
4726
4727         clear_bit(__I40E_NEEDS_RESTART, &vsi->state);
4728 }
4729
4730 /**
4731  * i40e_fdir_teardown - release the Flow Director resources
4732  * @pf: board private structure
4733  **/
4734 static void i40e_fdir_teardown(struct i40e_pf *pf)
4735 {
4736         int i;
4737
4738         for (i = 0; i < pf->hw.func_caps.num_vsis; i++) {
4739                 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR) {
4740                         i40e_vsi_release(pf->vsi[i]);
4741                         break;
4742                 }
4743         }
4744 }
4745
4746 /**
4747  * i40e_prep_for_reset - prep for the core to reset
4748  * @pf: board private structure
4749  *
4750  * Close up the VFs and other things in prep for pf Reset.
4751   **/
4752 static int i40e_prep_for_reset(struct i40e_pf *pf)
4753 {
4754         struct i40e_hw *hw = &pf->hw;
4755         i40e_status ret;
4756         u32 v;
4757
4758         clear_bit(__I40E_RESET_INTR_RECEIVED, &pf->state);
4759         if (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state))
4760                 return 0;
4761
4762         dev_info(&pf->pdev->dev, "Tearing down internal switch for reset\n");
4763
4764         if (i40e_check_asq_alive(hw))
4765                 i40e_vc_notify_reset(pf);
4766
4767         /* quiesce the VSIs and their queues that are not already DOWN */
4768         i40e_pf_quiesce_all_vsi(pf);
4769
4770         for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
4771                 if (pf->vsi[v])
4772                         pf->vsi[v]->seid = 0;
4773         }
4774
4775         i40e_shutdown_adminq(&pf->hw);
4776
4777         /* call shutdown HMC */
4778         ret = i40e_shutdown_lan_hmc(hw);
4779         if (ret) {
4780                 dev_info(&pf->pdev->dev, "shutdown_lan_hmc failed: %d\n", ret);
4781                 clear_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state);
4782         }
4783         return ret;
4784 }
4785
4786 /**
4787  * i40e_reset_and_rebuild - reset and rebuid using a saved config
4788  * @pf: board private structure
4789  * @reinit: if the Main VSI needs to re-initialized.
4790  **/
4791 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit)
4792 {
4793         struct i40e_driver_version dv;
4794         struct i40e_hw *hw = &pf->hw;
4795         i40e_status ret;
4796         u32 v;
4797
4798         /* Now we wait for GRST to settle out.
4799          * We don't have to delete the VEBs or VSIs from the hw switch
4800          * because the reset will make them disappear.
4801          */
4802         ret = i40e_pf_reset(hw);
4803         if (ret)
4804                 dev_info(&pf->pdev->dev, "PF reset failed, %d\n", ret);
4805         pf->pfr_count++;
4806
4807         if (test_bit(__I40E_DOWN, &pf->state))
4808                 goto end_core_reset;
4809         dev_info(&pf->pdev->dev, "Rebuilding internal switch\n");
4810
4811         /* rebuild the basics for the AdminQ, HMC, and initial HW switch */
4812         ret = i40e_init_adminq(&pf->hw);
4813         if (ret) {
4814                 dev_info(&pf->pdev->dev, "Rebuild AdminQ failed, %d\n", ret);
4815                 goto end_core_reset;
4816         }
4817
4818         ret = i40e_get_capabilities(pf);
4819         if (ret) {
4820                 dev_info(&pf->pdev->dev, "i40e_get_capabilities failed, %d\n",
4821                          ret);
4822                 goto end_core_reset;
4823         }
4824
4825         ret = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
4826                                 hw->func_caps.num_rx_qp,
4827                                 pf->fcoe_hmc_cntx_num, pf->fcoe_hmc_filt_num);
4828         if (ret) {
4829                 dev_info(&pf->pdev->dev, "init_lan_hmc failed: %d\n", ret);
4830                 goto end_core_reset;
4831         }
4832         ret = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
4833         if (ret) {
4834                 dev_info(&pf->pdev->dev, "configure_lan_hmc failed: %d\n", ret);
4835                 goto end_core_reset;
4836         }
4837
4838         /* do basic switch setup */
4839         ret = i40e_setup_pf_switch(pf, reinit);
4840         if (ret)
4841                 goto end_core_reset;
4842
4843         /* Rebuild the VSIs and VEBs that existed before reset.
4844          * They are still in our local switch element arrays, so only
4845          * need to rebuild the switch model in the HW.
4846          *
4847          * If there were VEBs but the reconstitution failed, we'll try
4848          * try to recover minimal use by getting the basic PF VSI working.
4849          */
4850         if (pf->vsi[pf->lan_vsi]->uplink_seid != pf->mac_seid) {
4851                 dev_info(&pf->pdev->dev, "attempting to rebuild switch\n");
4852                 /* find the one VEB connected to the MAC, and find orphans */
4853                 for (v = 0; v < I40E_MAX_VEB; v++) {
4854                         if (!pf->veb[v])
4855                                 continue;
4856
4857                         if (pf->veb[v]->uplink_seid == pf->mac_seid ||
4858                             pf->veb[v]->uplink_seid == 0) {
4859                                 ret = i40e_reconstitute_veb(pf->veb[v]);
4860
4861                                 if (!ret)
4862                                         continue;
4863
4864                                 /* If Main VEB failed, we're in deep doodoo,
4865                                  * so give up rebuilding the switch and set up
4866                                  * for minimal rebuild of PF VSI.
4867                                  * If orphan failed, we'll report the error
4868                                  * but try to keep going.
4869                                  */
4870                                 if (pf->veb[v]->uplink_seid == pf->mac_seid) {
4871                                         dev_info(&pf->pdev->dev,
4872                                                  "rebuild of switch failed: %d, will try to set up simple PF connection\n",
4873                                                  ret);
4874                                         pf->vsi[pf->lan_vsi]->uplink_seid
4875                                                                 = pf->mac_seid;
4876                                         break;
4877                                 } else if (pf->veb[v]->uplink_seid == 0) {
4878                                         dev_info(&pf->pdev->dev,
4879                                                  "rebuild of orphan VEB failed: %d\n",
4880                                                  ret);
4881                                 }
4882                         }
4883                 }
4884         }
4885
4886         if (pf->vsi[pf->lan_vsi]->uplink_seid == pf->mac_seid) {
4887                 dev_info(&pf->pdev->dev, "attempting to rebuild PF VSI\n");
4888                 /* no VEB, so rebuild only the Main VSI */
4889                 ret = i40e_add_vsi(pf->vsi[pf->lan_vsi]);
4890                 if (ret) {
4891                         dev_info(&pf->pdev->dev,
4892                                  "rebuild of Main VSI failed: %d\n", ret);
4893                         goto end_core_reset;
4894                 }
4895         }
4896
4897         /* reinit the misc interrupt */
4898         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
4899                 ret = i40e_setup_misc_vector(pf);
4900
4901         /* restart the VSIs that were rebuilt and running before the reset */
4902         i40e_pf_unquiesce_all_vsi(pf);
4903
4904         /* tell the firmware that we're starting */
4905         dv.major_version = DRV_VERSION_MAJOR;
4906         dv.minor_version = DRV_VERSION_MINOR;
4907         dv.build_version = DRV_VERSION_BUILD;
4908         dv.subbuild_version = 0;
4909         i40e_aq_send_driver_version(&pf->hw, &dv, NULL);
4910
4911         dev_info(&pf->pdev->dev, "PF reset done\n");
4912
4913 end_core_reset:
4914         clear_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state);
4915 }
4916
4917 /**
4918  * i40e_handle_reset_warning - prep for the pf to reset, reset and rebuild
4919  * @pf: board private structure
4920  *
4921  * Close up the VFs and other things in prep for a Core Reset,
4922  * then get ready to rebuild the world.
4923  **/
4924 static void i40e_handle_reset_warning(struct i40e_pf *pf)
4925 {
4926         i40e_status ret;
4927
4928         ret = i40e_prep_for_reset(pf);
4929         if (!ret)
4930                 i40e_reset_and_rebuild(pf, false);
4931 }
4932
4933 /**
4934  * i40e_handle_mdd_event
4935  * @pf: pointer to the pf structure
4936  *
4937  * Called from the MDD irq handler to identify possibly malicious vfs
4938  **/
4939 static void i40e_handle_mdd_event(struct i40e_pf *pf)
4940 {
4941         struct i40e_hw *hw = &pf->hw;
4942         bool mdd_detected = false;
4943         struct i40e_vf *vf;
4944         u32 reg;
4945         int i;
4946
4947         if (!test_bit(__I40E_MDD_EVENT_PENDING, &pf->state))
4948                 return;
4949
4950         /* find what triggered the MDD event */
4951         reg = rd32(hw, I40E_GL_MDET_TX);
4952         if (reg & I40E_GL_MDET_TX_VALID_MASK) {
4953                 u8 func = (reg & I40E_GL_MDET_TX_FUNCTION_MASK)
4954                                 >> I40E_GL_MDET_TX_FUNCTION_SHIFT;
4955                 u8 event = (reg & I40E_GL_MDET_TX_EVENT_SHIFT)
4956                                 >> I40E_GL_MDET_TX_EVENT_SHIFT;
4957                 u8 queue = (reg & I40E_GL_MDET_TX_QUEUE_MASK)
4958                                 >> I40E_GL_MDET_TX_QUEUE_SHIFT;
4959                 dev_info(&pf->pdev->dev,
4960                          "Malicious Driver Detection TX event 0x%02x on q %d of function 0x%02x\n",
4961                          event, queue, func);
4962                 wr32(hw, I40E_GL_MDET_TX, 0xffffffff);
4963                 mdd_detected = true;
4964         }
4965         reg = rd32(hw, I40E_GL_MDET_RX);
4966         if (reg & I40E_GL_MDET_RX_VALID_MASK) {
4967                 u8 func = (reg & I40E_GL_MDET_RX_FUNCTION_MASK)
4968                                 >> I40E_GL_MDET_RX_FUNCTION_SHIFT;
4969                 u8 event = (reg & I40E_GL_MDET_RX_EVENT_SHIFT)
4970                                 >> I40E_GL_MDET_RX_EVENT_SHIFT;
4971                 u8 queue = (reg & I40E_GL_MDET_RX_QUEUE_MASK)
4972                                 >> I40E_GL_MDET_RX_QUEUE_SHIFT;
4973                 dev_info(&pf->pdev->dev,
4974                          "Malicious Driver Detection RX event 0x%02x on q %d of function 0x%02x\n",
4975                          event, queue, func);
4976                 wr32(hw, I40E_GL_MDET_RX, 0xffffffff);
4977                 mdd_detected = true;
4978         }
4979
4980         /* see if one of the VFs needs its hand slapped */
4981         for (i = 0; i < pf->num_alloc_vfs && mdd_detected; i++) {
4982                 vf = &(pf->vf[i]);
4983                 reg = rd32(hw, I40E_VP_MDET_TX(i));
4984                 if (reg & I40E_VP_MDET_TX_VALID_MASK) {
4985                         wr32(hw, I40E_VP_MDET_TX(i), 0xFFFF);
4986                         vf->num_mdd_events++;
4987                         dev_info(&pf->pdev->dev, "MDD TX event on VF %d\n", i);
4988                 }
4989
4990                 reg = rd32(hw, I40E_VP_MDET_RX(i));
4991                 if (reg & I40E_VP_MDET_RX_VALID_MASK) {
4992                         wr32(hw, I40E_VP_MDET_RX(i), 0xFFFF);
4993                         vf->num_mdd_events++;
4994                         dev_info(&pf->pdev->dev, "MDD RX event on VF %d\n", i);
4995                 }
4996
4997                 if (vf->num_mdd_events > I40E_DEFAULT_NUM_MDD_EVENTS_ALLOWED) {
4998                         dev_info(&pf->pdev->dev,
4999                                  "Too many MDD events on VF %d, disabled\n", i);
5000                         dev_info(&pf->pdev->dev,
5001                                  "Use PF Control I/F to re-enable the VF\n");
5002                         set_bit(I40E_VF_STAT_DISABLED, &vf->vf_states);
5003                 }
5004         }
5005
5006         /* re-enable mdd interrupt cause */
5007         clear_bit(__I40E_MDD_EVENT_PENDING, &pf->state);
5008         reg = rd32(hw, I40E_PFINT_ICR0_ENA);
5009         reg |=  I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
5010         wr32(hw, I40E_PFINT_ICR0_ENA, reg);
5011         i40e_flush(hw);
5012 }
5013
5014 #ifdef CONFIG_I40E_VXLAN
5015 /**
5016  * i40e_sync_vxlan_filters_subtask - Sync the VSI filter list with HW
5017  * @pf: board private structure
5018  **/
5019 static void i40e_sync_vxlan_filters_subtask(struct i40e_pf *pf)
5020 {
5021         const int vxlan_hdr_qwords = 4;
5022         struct i40e_hw *hw = &pf->hw;
5023         i40e_status ret;
5024         u8 filter_index;
5025         __be16 port;
5026         int i;
5027
5028         if (!(pf->flags & I40E_FLAG_VXLAN_FILTER_SYNC))
5029                 return;
5030
5031         pf->flags &= ~I40E_FLAG_VXLAN_FILTER_SYNC;
5032
5033         for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) {
5034                 if (pf->pending_vxlan_bitmap & (1 << i)) {
5035                         pf->pending_vxlan_bitmap &= ~(1 << i);
5036                         port = pf->vxlan_ports[i];
5037                         ret = port ?
5038                               i40e_aq_add_udp_tunnel(hw, ntohs(port),
5039                                                      vxlan_hdr_qwords,
5040                                                      I40E_AQC_TUNNEL_TYPE_VXLAN,
5041                                                      &filter_index, NULL)
5042                               : i40e_aq_del_udp_tunnel(hw, i, NULL);
5043
5044                         if (ret) {
5045                                 dev_info(&pf->pdev->dev, "Failed to execute AQ command for %s port %d with index %d\n",
5046                                          port ? "adding" : "deleting",
5047                                          ntohs(port), port ? i : i);
5048
5049                                 pf->vxlan_ports[i] = 0;
5050                         } else {
5051                                 dev_info(&pf->pdev->dev, "%s port %d with AQ command with index %d\n",
5052                                          port ? "Added" : "Deleted",
5053                                          ntohs(port), port ? i : filter_index);
5054                         }
5055                 }
5056         }
5057 }
5058
5059 #endif
5060 /**
5061  * i40e_service_task - Run the driver's async subtasks
5062  * @work: pointer to work_struct containing our data
5063  **/
5064 static void i40e_service_task(struct work_struct *work)
5065 {
5066         struct i40e_pf *pf = container_of(work,
5067                                           struct i40e_pf,
5068                                           service_task);
5069         unsigned long start_time = jiffies;
5070
5071         i40e_reset_subtask(pf);
5072         i40e_handle_mdd_event(pf);
5073         i40e_vc_process_vflr_event(pf);
5074         i40e_watchdog_subtask(pf);
5075         i40e_fdir_reinit_subtask(pf);
5076         i40e_check_hang_subtask(pf);
5077         i40e_sync_filters_subtask(pf);
5078 #ifdef CONFIG_I40E_VXLAN
5079         i40e_sync_vxlan_filters_subtask(pf);
5080 #endif
5081         i40e_clean_adminq_subtask(pf);
5082
5083         i40e_service_event_complete(pf);
5084
5085         /* If the tasks have taken longer than one timer cycle or there
5086          * is more work to be done, reschedule the service task now
5087          * rather than wait for the timer to tick again.
5088          */
5089         if (time_after(jiffies, (start_time + pf->service_timer_period)) ||
5090             test_bit(__I40E_ADMINQ_EVENT_PENDING, &pf->state)            ||
5091             test_bit(__I40E_MDD_EVENT_PENDING, &pf->state)               ||
5092             test_bit(__I40E_VFLR_EVENT_PENDING, &pf->state))
5093                 i40e_service_event_schedule(pf);
5094 }
5095
5096 /**
5097  * i40e_service_timer - timer callback
5098  * @data: pointer to PF struct
5099  **/
5100 static void i40e_service_timer(unsigned long data)
5101 {
5102         struct i40e_pf *pf = (struct i40e_pf *)data;
5103
5104         mod_timer(&pf->service_timer,
5105                   round_jiffies(jiffies + pf->service_timer_period));
5106         i40e_service_event_schedule(pf);
5107 }
5108
5109 /**
5110  * i40e_set_num_rings_in_vsi - Determine number of rings in the VSI
5111  * @vsi: the VSI being configured
5112  **/
5113 static int i40e_set_num_rings_in_vsi(struct i40e_vsi *vsi)
5114 {
5115         struct i40e_pf *pf = vsi->back;
5116
5117         switch (vsi->type) {
5118         case I40E_VSI_MAIN:
5119                 vsi->alloc_queue_pairs = pf->num_lan_qps;
5120                 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
5121                                       I40E_REQ_DESCRIPTOR_MULTIPLE);
5122                 if (pf->flags & I40E_FLAG_MSIX_ENABLED)
5123                         vsi->num_q_vectors = pf->num_lan_msix;
5124                 else
5125                         vsi->num_q_vectors = 1;
5126
5127                 break;
5128
5129         case I40E_VSI_FDIR:
5130                 vsi->alloc_queue_pairs = 1;
5131                 vsi->num_desc = ALIGN(I40E_FDIR_RING_COUNT,
5132                                       I40E_REQ_DESCRIPTOR_MULTIPLE);
5133                 vsi->num_q_vectors = 1;
5134                 break;
5135
5136         case I40E_VSI_VMDQ2:
5137                 vsi->alloc_queue_pairs = pf->num_vmdq_qps;
5138                 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
5139                                       I40E_REQ_DESCRIPTOR_MULTIPLE);
5140                 vsi->num_q_vectors = pf->num_vmdq_msix;
5141                 break;
5142
5143         case I40E_VSI_SRIOV:
5144                 vsi->alloc_queue_pairs = pf->num_vf_qps;
5145                 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
5146                                       I40E_REQ_DESCRIPTOR_MULTIPLE);
5147                 break;
5148
5149         default:
5150                 WARN_ON(1);
5151                 return -ENODATA;
5152         }
5153
5154         return 0;
5155 }
5156
5157 /**
5158  * i40e_vsi_alloc_arrays - Allocate queue and vector pointer arrays for the vsi
5159  * @type: VSI pointer
5160  * @alloc_qvectors: a bool to specify if q_vectors need to be allocated.
5161  *
5162  * On error: returns error code (negative)
5163  * On success: returns 0
5164  **/
5165 static int i40e_vsi_alloc_arrays(struct i40e_vsi *vsi, bool alloc_qvectors)
5166 {
5167         int size;
5168         int ret = 0;
5169
5170         /* allocate memory for both Tx and Rx ring pointers */
5171         size = sizeof(struct i40e_ring *) * vsi->alloc_queue_pairs * 2;
5172         vsi->tx_rings = kzalloc(size, GFP_KERNEL);
5173         if (!vsi->tx_rings)
5174                 return -ENOMEM;
5175         vsi->rx_rings = &vsi->tx_rings[vsi->alloc_queue_pairs];
5176
5177         if (alloc_qvectors) {
5178                 /* allocate memory for q_vector pointers */
5179                 size = sizeof(struct i40e_q_vectors *) * vsi->num_q_vectors;
5180                 vsi->q_vectors = kzalloc(size, GFP_KERNEL);
5181                 if (!vsi->q_vectors) {
5182                         ret = -ENOMEM;
5183                         goto err_vectors;
5184                 }
5185         }
5186         return ret;
5187
5188 err_vectors:
5189         kfree(vsi->tx_rings);
5190         return ret;
5191 }
5192
5193 /**
5194  * i40e_vsi_mem_alloc - Allocates the next available struct vsi in the PF
5195  * @pf: board private structure
5196  * @type: type of VSI
5197  *
5198  * On error: returns error code (negative)
5199  * On success: returns vsi index in PF (positive)
5200  **/
5201 static int i40e_vsi_mem_alloc(struct i40e_pf *pf, enum i40e_vsi_type type)
5202 {
5203         int ret = -ENODEV;
5204         struct i40e_vsi *vsi;
5205         int vsi_idx;
5206         int i;
5207
5208         /* Need to protect the allocation of the VSIs at the PF level */
5209         mutex_lock(&pf->switch_mutex);
5210
5211         /* VSI list may be fragmented if VSI creation/destruction has
5212          * been happening.  We can afford to do a quick scan to look
5213          * for any free VSIs in the list.
5214          *
5215          * find next empty vsi slot, looping back around if necessary
5216          */
5217         i = pf->next_vsi;
5218         while (i < pf->hw.func_caps.num_vsis && pf->vsi[i])
5219                 i++;
5220         if (i >= pf->hw.func_caps.num_vsis) {
5221                 i = 0;
5222                 while (i < pf->next_vsi && pf->vsi[i])
5223                         i++;
5224         }
5225
5226         if (i < pf->hw.func_caps.num_vsis && !pf->vsi[i]) {
5227                 vsi_idx = i;             /* Found one! */
5228         } else {
5229                 ret = -ENODEV;
5230                 goto unlock_pf;  /* out of VSI slots! */
5231         }
5232         pf->next_vsi = ++i;
5233
5234         vsi = kzalloc(sizeof(*vsi), GFP_KERNEL);
5235         if (!vsi) {
5236                 ret = -ENOMEM;
5237                 goto unlock_pf;
5238         }
5239         vsi->type = type;
5240         vsi->back = pf;
5241         set_bit(__I40E_DOWN, &vsi->state);
5242         vsi->flags = 0;
5243         vsi->idx = vsi_idx;
5244         vsi->rx_itr_setting = pf->rx_itr_default;
5245         vsi->tx_itr_setting = pf->tx_itr_default;
5246         vsi->netdev_registered = false;
5247         vsi->work_limit = I40E_DEFAULT_IRQ_WORK;
5248         INIT_LIST_HEAD(&vsi->mac_filter_list);
5249
5250         ret = i40e_set_num_rings_in_vsi(vsi);
5251         if (ret)
5252                 goto err_rings;
5253
5254         ret = i40e_vsi_alloc_arrays(vsi, true);
5255         if (ret)
5256                 goto err_rings;
5257
5258         /* Setup default MSIX irq handler for VSI */
5259         i40e_vsi_setup_irqhandler(vsi, i40e_msix_clean_rings);
5260
5261         pf->vsi[vsi_idx] = vsi;
5262         ret = vsi_idx;
5263         goto unlock_pf;
5264
5265 err_rings:
5266         pf->next_vsi = i - 1;
5267         kfree(vsi);
5268 unlock_pf:
5269         mutex_unlock(&pf->switch_mutex);
5270         return ret;
5271 }
5272
5273 /**
5274  * i40e_vsi_free_arrays - Free queue and vector pointer arrays for the VSI
5275  * @type: VSI pointer
5276  * @free_qvectors: a bool to specify if q_vectors need to be freed.
5277  *
5278  * On error: returns error code (negative)
5279  * On success: returns 0
5280  **/
5281 static void i40e_vsi_free_arrays(struct i40e_vsi *vsi, bool free_qvectors)
5282 {
5283         /* free the ring and vector containers */
5284         if (free_qvectors) {
5285                 kfree(vsi->q_vectors);
5286                 vsi->q_vectors = NULL;
5287         }
5288         kfree(vsi->tx_rings);
5289         vsi->tx_rings = NULL;
5290         vsi->rx_rings = NULL;
5291 }
5292
5293 /**
5294  * i40e_vsi_clear - Deallocate the VSI provided
5295  * @vsi: the VSI being un-configured
5296  **/
5297 static int i40e_vsi_clear(struct i40e_vsi *vsi)
5298 {
5299         struct i40e_pf *pf;
5300
5301         if (!vsi)
5302                 return 0;
5303
5304         if (!vsi->back)
5305                 goto free_vsi;
5306         pf = vsi->back;
5307
5308         mutex_lock(&pf->switch_mutex);
5309         if (!pf->vsi[vsi->idx]) {
5310                 dev_err(&pf->pdev->dev, "pf->vsi[%d] is NULL, just free vsi[%d](%p,type %d)\n",
5311                         vsi->idx, vsi->idx, vsi, vsi->type);
5312                 goto unlock_vsi;
5313         }
5314
5315         if (pf->vsi[vsi->idx] != vsi) {
5316                 dev_err(&pf->pdev->dev,
5317                         "pf->vsi[%d](%p, type %d) != vsi[%d](%p,type %d): no free!\n",
5318                         pf->vsi[vsi->idx]->idx,
5319                         pf->vsi[vsi->idx],
5320                         pf->vsi[vsi->idx]->type,
5321                         vsi->idx, vsi, vsi->type);
5322                 goto unlock_vsi;
5323         }
5324
5325         /* updates the pf for this cleared vsi */
5326         i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);
5327         i40e_put_lump(pf->irq_pile, vsi->base_vector, vsi->idx);
5328
5329         i40e_vsi_free_arrays(vsi, true);
5330
5331         pf->vsi[vsi->idx] = NULL;
5332         if (vsi->idx < pf->next_vsi)
5333                 pf->next_vsi = vsi->idx;
5334
5335 unlock_vsi:
5336         mutex_unlock(&pf->switch_mutex);
5337 free_vsi:
5338         kfree(vsi);
5339
5340         return 0;
5341 }
5342
5343 /**
5344  * i40e_vsi_clear_rings - Deallocates the Rx and Tx rings for the provided VSI
5345  * @vsi: the VSI being cleaned
5346  **/
5347 static void i40e_vsi_clear_rings(struct i40e_vsi *vsi)
5348 {
5349         int i;
5350
5351         if (vsi->tx_rings[0]) {
5352                 for (i = 0; i < vsi->alloc_queue_pairs; i++) {
5353                         kfree_rcu(vsi->tx_rings[i], rcu);
5354                         vsi->tx_rings[i] = NULL;
5355                         vsi->rx_rings[i] = NULL;
5356                 }
5357         }
5358 }
5359
5360 /**
5361  * i40e_alloc_rings - Allocates the Rx and Tx rings for the provided VSI
5362  * @vsi: the VSI being configured
5363  **/
5364 static int i40e_alloc_rings(struct i40e_vsi *vsi)
5365 {
5366         struct i40e_pf *pf = vsi->back;
5367         int i;
5368
5369         /* Set basic values in the rings to be used later during open() */
5370         for (i = 0; i < vsi->alloc_queue_pairs; i++) {
5371                 struct i40e_ring *tx_ring;
5372                 struct i40e_ring *rx_ring;
5373
5374                 /* allocate space for both Tx and Rx in one shot */
5375                 tx_ring = kzalloc(sizeof(struct i40e_ring) * 2, GFP_KERNEL);
5376                 if (!tx_ring)
5377                         goto err_out;
5378
5379                 tx_ring->queue_index = i;
5380                 tx_ring->reg_idx = vsi->base_queue + i;
5381                 tx_ring->ring_active = false;
5382                 tx_ring->vsi = vsi;
5383                 tx_ring->netdev = vsi->netdev;
5384                 tx_ring->dev = &pf->pdev->dev;
5385                 tx_ring->count = vsi->num_desc;
5386                 tx_ring->size = 0;
5387                 tx_ring->dcb_tc = 0;
5388                 vsi->tx_rings[i] = tx_ring;
5389
5390                 rx_ring = &tx_ring[1];
5391                 rx_ring->queue_index = i;
5392                 rx_ring->reg_idx = vsi->base_queue + i;
5393                 rx_ring->ring_active = false;
5394                 rx_ring->vsi = vsi;
5395                 rx_ring->netdev = vsi->netdev;
5396                 rx_ring->dev = &pf->pdev->dev;
5397                 rx_ring->count = vsi->num_desc;
5398                 rx_ring->size = 0;
5399                 rx_ring->dcb_tc = 0;
5400                 if (pf->flags & I40E_FLAG_16BYTE_RX_DESC_ENABLED)
5401                         set_ring_16byte_desc_enabled(rx_ring);
5402                 else
5403                         clear_ring_16byte_desc_enabled(rx_ring);
5404                 vsi->rx_rings[i] = rx_ring;
5405         }
5406
5407         return 0;
5408
5409 err_out:
5410         i40e_vsi_clear_rings(vsi);
5411         return -ENOMEM;
5412 }
5413
5414 /**
5415  * i40e_reserve_msix_vectors - Reserve MSI-X vectors in the kernel
5416  * @pf: board private structure
5417  * @vectors: the number of MSI-X vectors to request
5418  *
5419  * Returns the number of vectors reserved, or error
5420  **/
5421 static int i40e_reserve_msix_vectors(struct i40e_pf *pf, int vectors)
5422 {
5423         int err = 0;
5424
5425         pf->num_msix_entries = 0;
5426         while (vectors >= I40E_MIN_MSIX) {
5427                 err = pci_enable_msix(pf->pdev, pf->msix_entries, vectors);
5428                 if (err == 0) {
5429                         /* good to go */
5430                         pf->num_msix_entries = vectors;
5431                         break;
5432                 } else if (err < 0) {
5433                         /* total failure */
5434                         dev_info(&pf->pdev->dev,
5435                                  "MSI-X vector reservation failed: %d\n", err);
5436                         vectors = 0;
5437                         break;
5438                 } else {
5439                         /* err > 0 is the hint for retry */
5440                         dev_info(&pf->pdev->dev,
5441                                  "MSI-X vectors wanted %d, retrying with %d\n",
5442                                  vectors, err);
5443                         vectors = err;
5444                 }
5445         }
5446
5447         if (vectors > 0 && vectors < I40E_MIN_MSIX) {
5448                 dev_info(&pf->pdev->dev,
5449                          "Couldn't get enough vectors, only %d available\n",
5450                          vectors);
5451                 vectors = 0;
5452         }
5453
5454         return vectors;
5455 }
5456
5457 /**
5458  * i40e_init_msix - Setup the MSIX capability
5459  * @pf: board private structure
5460  *
5461  * Work with the OS to set up the MSIX vectors needed.
5462  *
5463  * Returns 0 on success, negative on failure
5464  **/
5465 static int i40e_init_msix(struct i40e_pf *pf)
5466 {
5467         i40e_status err = 0;
5468         struct i40e_hw *hw = &pf->hw;
5469         int v_budget, i;
5470         int vec;
5471
5472         if (!(pf->flags & I40E_FLAG_MSIX_ENABLED))
5473                 return -ENODEV;
5474
5475         /* The number of vectors we'll request will be comprised of:
5476          *   - Add 1 for "other" cause for Admin Queue events, etc.
5477          *   - The number of LAN queue pairs
5478          *      - Queues being used for RSS.
5479          *              We don't need as many as max_rss_size vectors.
5480          *              use rss_size instead in the calculation since that
5481          *              is governed by number of cpus in the system.
5482          *      - assumes symmetric Tx/Rx pairing
5483          *   - The number of VMDq pairs
5484          * Once we count this up, try the request.
5485          *
5486          * If we can't get what we want, we'll simplify to nearly nothing
5487          * and try again.  If that still fails, we punt.
5488          */
5489         pf->num_lan_msix = pf->num_lan_qps - (pf->rss_size_max - pf->rss_size);
5490         pf->num_vmdq_msix = pf->num_vmdq_qps;
5491         v_budget = 1 + pf->num_lan_msix;
5492         v_budget += (pf->num_vmdq_vsis * pf->num_vmdq_msix);
5493         if (pf->flags & I40E_FLAG_FDIR_ENABLED)
5494                 v_budget++;
5495
5496         /* Scale down if necessary, and the rings will share vectors */
5497         v_budget = min_t(int, v_budget, hw->func_caps.num_msix_vectors);
5498
5499         pf->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry),
5500                                    GFP_KERNEL);
5501         if (!pf->msix_entries)
5502                 return -ENOMEM;
5503
5504         for (i = 0; i < v_budget; i++)
5505                 pf->msix_entries[i].entry = i;
5506         vec = i40e_reserve_msix_vectors(pf, v_budget);
5507         if (vec < I40E_MIN_MSIX) {
5508                 pf->flags &= ~I40E_FLAG_MSIX_ENABLED;
5509                 kfree(pf->msix_entries);
5510                 pf->msix_entries = NULL;
5511                 return -ENODEV;
5512
5513         } else if (vec == I40E_MIN_MSIX) {
5514                 /* Adjust for minimal MSIX use */
5515                 dev_info(&pf->pdev->dev, "Features disabled, not enough MSIX vectors\n");
5516                 pf->flags &= ~I40E_FLAG_VMDQ_ENABLED;
5517                 pf->num_vmdq_vsis = 0;
5518                 pf->num_vmdq_qps = 0;
5519                 pf->num_vmdq_msix = 0;
5520                 pf->num_lan_qps = 1;
5521                 pf->num_lan_msix = 1;
5522
5523         } else if (vec != v_budget) {
5524                 /* Scale vector usage down */
5525                 pf->num_vmdq_msix = 1;    /* force VMDqs to only one vector */
5526                 vec--;                    /* reserve the misc vector */
5527
5528                 /* partition out the remaining vectors */
5529                 switch (vec) {
5530                 case 2:
5531                         pf->num_vmdq_vsis = 1;
5532                         pf->num_lan_msix = 1;
5533                         break;
5534                 case 3:
5535                         pf->num_vmdq_vsis = 1;
5536                         pf->num_lan_msix = 2;
5537                         break;
5538                 default:
5539                         pf->num_lan_msix = min_t(int, (vec / 2),
5540                                                  pf->num_lan_qps);
5541                         pf->num_vmdq_vsis = min_t(int, (vec - pf->num_lan_msix),
5542                                                   I40E_DEFAULT_NUM_VMDQ_VSI);
5543                         break;
5544                 }
5545         }
5546
5547         return err;
5548 }
5549
5550 /**
5551  * i40e_alloc_q_vector - Allocate memory for a single interrupt vector
5552  * @vsi: the VSI being configured
5553  * @v_idx: index of the vector in the vsi struct
5554  *
5555  * We allocate one q_vector.  If allocation fails we return -ENOMEM.
5556  **/
5557 static int i40e_alloc_q_vector(struct i40e_vsi *vsi, int v_idx)
5558 {
5559         struct i40e_q_vector *q_vector;
5560
5561         /* allocate q_vector */
5562         q_vector = kzalloc(sizeof(struct i40e_q_vector), GFP_KERNEL);
5563         if (!q_vector)
5564                 return -ENOMEM;
5565
5566         q_vector->vsi = vsi;
5567         q_vector->v_idx = v_idx;
5568         cpumask_set_cpu(v_idx, &q_vector->affinity_mask);
5569         if (vsi->netdev)
5570                 netif_napi_add(vsi->netdev, &q_vector->napi,
5571                                i40e_napi_poll, vsi->work_limit);
5572
5573         q_vector->rx.latency_range = I40E_LOW_LATENCY;
5574         q_vector->tx.latency_range = I40E_LOW_LATENCY;
5575
5576         /* tie q_vector and vsi together */
5577         vsi->q_vectors[v_idx] = q_vector;
5578
5579         return 0;
5580 }
5581
5582 /**
5583  * i40e_alloc_q_vectors - Allocate memory for interrupt vectors
5584  * @vsi: the VSI being configured
5585  *
5586  * We allocate one q_vector per queue interrupt.  If allocation fails we
5587  * return -ENOMEM.
5588  **/
5589 static int i40e_alloc_q_vectors(struct i40e_vsi *vsi)
5590 {
5591         struct i40e_pf *pf = vsi->back;
5592         int v_idx, num_q_vectors;
5593         int err;
5594
5595         /* if not MSIX, give the one vector only to the LAN VSI */
5596         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
5597                 num_q_vectors = vsi->num_q_vectors;
5598         else if (vsi == pf->vsi[pf->lan_vsi])
5599                 num_q_vectors = 1;
5600         else
5601                 return -EINVAL;
5602
5603         for (v_idx = 0; v_idx < num_q_vectors; v_idx++) {
5604                 err = i40e_alloc_q_vector(vsi, v_idx);
5605                 if (err)
5606                         goto err_out;
5607         }
5608
5609         return 0;
5610
5611 err_out:
5612         while (v_idx--)
5613                 i40e_free_q_vector(vsi, v_idx);
5614
5615         return err;
5616 }
5617
5618 /**
5619  * i40e_init_interrupt_scheme - Determine proper interrupt scheme
5620  * @pf: board private structure to initialize
5621  **/
5622 static void i40e_init_interrupt_scheme(struct i40e_pf *pf)
5623 {
5624         int err = 0;
5625
5626         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
5627                 err = i40e_init_msix(pf);
5628                 if (err) {
5629                         pf->flags &= ~(I40E_FLAG_MSIX_ENABLED      |
5630                                         I40E_FLAG_RSS_ENABLED      |
5631                                         I40E_FLAG_DCB_ENABLED      |
5632                                         I40E_FLAG_SRIOV_ENABLED    |
5633                                         I40E_FLAG_FDIR_ENABLED     |
5634                                         I40E_FLAG_FDIR_ATR_ENABLED |
5635                                         I40E_FLAG_VMDQ_ENABLED);
5636
5637                         /* rework the queue expectations without MSIX */
5638                         i40e_determine_queue_usage(pf);
5639                 }
5640         }
5641
5642         if (!(pf->flags & I40E_FLAG_MSIX_ENABLED) &&
5643             (pf->flags & I40E_FLAG_MSI_ENABLED)) {
5644                 dev_info(&pf->pdev->dev, "MSIX not available, trying MSI\n");
5645                 err = pci_enable_msi(pf->pdev);
5646                 if (err) {
5647                         dev_info(&pf->pdev->dev, "MSI init failed - %d\n", err);
5648                         pf->flags &= ~I40E_FLAG_MSI_ENABLED;
5649                 }
5650         }
5651
5652         if (!(pf->flags & (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED)))
5653                 dev_info(&pf->pdev->dev, "MSIX and MSI not available, falling back to Legacy IRQ\n");
5654
5655         /* track first vector for misc interrupts */
5656         err = i40e_get_lump(pf, pf->irq_pile, 1, I40E_PILE_VALID_BIT-1);
5657 }
5658
5659 /**
5660  * i40e_setup_misc_vector - Setup the misc vector to handle non queue events
5661  * @pf: board private structure
5662  *
5663  * This sets up the handler for MSIX 0, which is used to manage the
5664  * non-queue interrupts, e.g. AdminQ and errors.  This is not used
5665  * when in MSI or Legacy interrupt mode.
5666  **/
5667 static int i40e_setup_misc_vector(struct i40e_pf *pf)
5668 {
5669         struct i40e_hw *hw = &pf->hw;
5670         int err = 0;
5671
5672         /* Only request the irq if this is the first time through, and
5673          * not when we're rebuilding after a Reset
5674          */
5675         if (!test_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state)) {
5676                 err = request_irq(pf->msix_entries[0].vector,
5677                                   i40e_intr, 0, pf->misc_int_name, pf);
5678                 if (err) {
5679                         dev_info(&pf->pdev->dev,
5680                                  "request_irq for msix_misc failed: %d\n", err);
5681                         return -EFAULT;
5682                 }
5683         }
5684
5685         i40e_enable_misc_int_causes(hw);
5686
5687         /* associate no queues to the misc vector */
5688         wr32(hw, I40E_PFINT_LNKLST0, I40E_QUEUE_END_OF_LIST);
5689         wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), I40E_ITR_8K);
5690
5691         i40e_flush(hw);
5692
5693         i40e_irq_dynamic_enable_icr0(pf);
5694
5695         return err;
5696 }
5697
5698 /**
5699  * i40e_config_rss - Prepare for RSS if used
5700  * @pf: board private structure
5701  **/
5702 static int i40e_config_rss(struct i40e_pf *pf)
5703 {
5704         /* Set of random keys generated using kernel random number generator */
5705         static const u32 seed[I40E_PFQF_HKEY_MAX_INDEX + 1] = {0x41b01687,
5706                                 0x183cfd8c, 0xce880440, 0x580cbc3c, 0x35897377,
5707                                 0x328b25e1, 0x4fa98922, 0xb7d90c14, 0xd5bad70d,
5708                                 0xcd15a2c1, 0xe8580225, 0x4a1e9d11, 0xfe5731be};
5709         struct i40e_hw *hw = &pf->hw;
5710         u32 lut = 0;
5711         int i, j;
5712         u64 hena;
5713
5714         /* Fill out hash function seed */
5715         for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
5716                 wr32(hw, I40E_PFQF_HKEY(i), seed[i]);
5717
5718         /* By default we enable TCP/UDP with IPv4/IPv6 ptypes */
5719         hena = (u64)rd32(hw, I40E_PFQF_HENA(0)) |
5720                 ((u64)rd32(hw, I40E_PFQF_HENA(1)) << 32);
5721         hena |= I40E_DEFAULT_RSS_HENA;
5722         wr32(hw, I40E_PFQF_HENA(0), (u32)hena);
5723         wr32(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32));
5724
5725         /* Populate the LUT with max no. of queues in round robin fashion */
5726         for (i = 0, j = 0; i < pf->hw.func_caps.rss_table_size; i++, j++) {
5727
5728                 /* The assumption is that lan qp count will be the highest
5729                  * qp count for any PF VSI that needs RSS.
5730                  * If multiple VSIs need RSS support, all the qp counts
5731                  * for those VSIs should be a power of 2 for RSS to work.
5732                  * If LAN VSI is the only consumer for RSS then this requirement
5733                  * is not necessary.
5734                  */
5735                 if (j == pf->rss_size)
5736                         j = 0;
5737                 /* lut = 4-byte sliding window of 4 lut entries */
5738                 lut = (lut << 8) | (j &
5739                          ((0x1 << pf->hw.func_caps.rss_table_entry_width) - 1));
5740                 /* On i = 3, we have 4 entries in lut; write to the register */
5741                 if ((i & 3) == 3)
5742                         wr32(hw, I40E_PFQF_HLUT(i >> 2), lut);
5743         }
5744         i40e_flush(hw);
5745
5746         return 0;
5747 }
5748
5749 /**
5750  * i40e_reconfig_rss_queues - change number of queues for rss and rebuild
5751  * @pf: board private structure
5752  * @queue_count: the requested queue count for rss.
5753  *
5754  * returns 0 if rss is not enabled, if enabled returns the final rss queue
5755  * count which may be different from the requested queue count.
5756  **/
5757 int i40e_reconfig_rss_queues(struct i40e_pf *pf, int queue_count)
5758 {
5759         if (!(pf->flags & I40E_FLAG_RSS_ENABLED))
5760                 return 0;
5761
5762         queue_count = min_t(int, queue_count, pf->rss_size_max);
5763         queue_count = rounddown_pow_of_two(queue_count);
5764
5765         if (queue_count != pf->rss_size) {
5766                 if (pf->queues_left < (queue_count - pf->rss_size)) {
5767                         dev_info(&pf->pdev->dev,
5768                                 "Not enough queues to do RSS on %d queues: remaining queues %d\n",
5769                                 queue_count, pf->queues_left);
5770                         return pf->rss_size;
5771                 }
5772                 i40e_prep_for_reset(pf);
5773
5774                 pf->num_lan_qps += (queue_count - pf->rss_size);
5775                 pf->queues_left -= (queue_count - pf->rss_size);
5776                 pf->rss_size = queue_count;
5777
5778                 i40e_reset_and_rebuild(pf, true);
5779                 i40e_config_rss(pf);
5780         }
5781         dev_info(&pf->pdev->dev, "RSS count:  %d\n", pf->rss_size);
5782         return pf->rss_size;
5783 }
5784
5785 /**
5786  * i40e_sw_init - Initialize general software structures (struct i40e_pf)
5787  * @pf: board private structure to initialize
5788  *
5789  * i40e_sw_init initializes the Adapter private data structure.
5790  * Fields are initialized based on PCI device information and
5791  * OS network device settings (MTU size).
5792  **/
5793 static int i40e_sw_init(struct i40e_pf *pf)
5794 {
5795         int err = 0;
5796         int size;
5797
5798         pf->msg_enable = netif_msg_init(I40E_DEFAULT_MSG_ENABLE,
5799                                 (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK));
5800         pf->hw.debug_mask = pf->msg_enable | I40E_DEBUG_DIAG;
5801         if (debug != -1 && debug != I40E_DEFAULT_MSG_ENABLE) {
5802                 if (I40E_DEBUG_USER & debug)
5803                         pf->hw.debug_mask = debug;
5804                 pf->msg_enable = netif_msg_init((debug & ~I40E_DEBUG_USER),
5805                                                 I40E_DEFAULT_MSG_ENABLE);
5806         }
5807
5808         /* Set default capability flags */
5809         pf->flags = I40E_FLAG_RX_CSUM_ENABLED |
5810                     I40E_FLAG_MSI_ENABLED     |
5811                     I40E_FLAG_MSIX_ENABLED    |
5812                     I40E_FLAG_RX_PS_ENABLED   |
5813                     I40E_FLAG_RX_1BUF_ENABLED;
5814
5815         /* Depending on PF configurations, it is possible that the RSS
5816          * maximum might end up larger than the available queues
5817          */
5818         pf->rss_size_max = 0x1 << pf->hw.func_caps.rss_table_entry_width;
5819         pf->rss_size_max = min_t(int, pf->rss_size_max,
5820                                  pf->hw.func_caps.num_tx_qp);
5821         if (pf->hw.func_caps.rss) {
5822                 pf->flags |= I40E_FLAG_RSS_ENABLED;
5823                 pf->rss_size = min_t(int, pf->rss_size_max, num_online_cpus());
5824         } else {
5825                 pf->rss_size = 1;
5826         }
5827
5828         if (pf->hw.func_caps.dcb)
5829                 pf->num_tc_qps = I40E_DEFAULT_QUEUES_PER_TC;
5830         else
5831                 pf->num_tc_qps = 0;
5832
5833         if (pf->hw.func_caps.fd) {
5834                 /* FW/NVM is not yet fixed in this regard */
5835                 if ((pf->hw.func_caps.fd_filters_guaranteed > 0) ||
5836                     (pf->hw.func_caps.fd_filters_best_effort > 0)) {
5837                         pf->flags |= I40E_FLAG_FDIR_ATR_ENABLED;
5838                         dev_info(&pf->pdev->dev,
5839                                  "Flow Director ATR mode Enabled\n");
5840                         pf->flags |= I40E_FLAG_FDIR_ENABLED;
5841                         dev_info(&pf->pdev->dev,
5842                                  "Flow Director Side Band mode Enabled\n");
5843                         pf->fdir_pf_filter_count =
5844                                          pf->hw.func_caps.fd_filters_guaranteed;
5845                 }
5846         } else {
5847                 pf->fdir_pf_filter_count = 0;
5848         }
5849
5850         if (pf->hw.func_caps.vmdq) {
5851                 pf->flags |= I40E_FLAG_VMDQ_ENABLED;
5852                 pf->num_vmdq_vsis = I40E_DEFAULT_NUM_VMDQ_VSI;
5853                 pf->num_vmdq_qps = I40E_DEFAULT_QUEUES_PER_VMDQ;
5854         }
5855
5856         /* MFP mode enabled */
5857         if (pf->hw.func_caps.npar_enable || pf->hw.func_caps.mfp_mode_1) {
5858                 pf->flags |= I40E_FLAG_MFP_ENABLED;
5859                 dev_info(&pf->pdev->dev, "MFP mode Enabled\n");
5860         }
5861
5862 #ifdef CONFIG_PCI_IOV
5863         if (pf->hw.func_caps.num_vfs) {
5864                 pf->num_vf_qps = I40E_DEFAULT_QUEUES_PER_VF;
5865                 pf->flags |= I40E_FLAG_SRIOV_ENABLED;
5866                 pf->num_req_vfs = min_t(int,
5867                                         pf->hw.func_caps.num_vfs,
5868                                         I40E_MAX_VF_COUNT);
5869                 dev_info(&pf->pdev->dev,
5870                          "Number of VFs being requested for PF[%d] = %d\n",
5871                          pf->hw.pf_id, pf->num_req_vfs);
5872         }
5873 #endif /* CONFIG_PCI_IOV */
5874         pf->eeprom_version = 0xDEAD;
5875         pf->lan_veb = I40E_NO_VEB;
5876         pf->lan_vsi = I40E_NO_VSI;
5877
5878         /* set up queue assignment tracking */
5879         size = sizeof(struct i40e_lump_tracking)
5880                 + (sizeof(u16) * pf->hw.func_caps.num_tx_qp);
5881         pf->qp_pile = kzalloc(size, GFP_KERNEL);
5882         if (!pf->qp_pile) {
5883                 err = -ENOMEM;
5884                 goto sw_init_done;
5885         }
5886         pf->qp_pile->num_entries = pf->hw.func_caps.num_tx_qp;
5887         pf->qp_pile->search_hint = 0;
5888
5889         /* set up vector assignment tracking */
5890         size = sizeof(struct i40e_lump_tracking)
5891                 + (sizeof(u16) * pf->hw.func_caps.num_msix_vectors);
5892         pf->irq_pile = kzalloc(size, GFP_KERNEL);
5893         if (!pf->irq_pile) {
5894                 kfree(pf->qp_pile);
5895                 err = -ENOMEM;
5896                 goto sw_init_done;
5897         }
5898         pf->irq_pile->num_entries = pf->hw.func_caps.num_msix_vectors;
5899         pf->irq_pile->search_hint = 0;
5900
5901         mutex_init(&pf->switch_mutex);
5902
5903 sw_init_done:
5904         return err;
5905 }
5906
5907 /**
5908  * i40e_set_features - set the netdev feature flags
5909  * @netdev: ptr to the netdev being adjusted
5910  * @features: the feature set that the stack is suggesting
5911  **/
5912 static int i40e_set_features(struct net_device *netdev,
5913                              netdev_features_t features)
5914 {
5915         struct i40e_netdev_priv *np = netdev_priv(netdev);
5916         struct i40e_vsi *vsi = np->vsi;
5917
5918         if (features & NETIF_F_HW_VLAN_CTAG_RX)
5919                 i40e_vlan_stripping_enable(vsi);
5920         else
5921                 i40e_vlan_stripping_disable(vsi);
5922
5923         return 0;
5924 }
5925
5926 #ifdef CONFIG_I40E_VXLAN
5927 /**
5928  * i40e_get_vxlan_port_idx - Lookup a possibly offloaded for Rx UDP port
5929  * @pf: board private structure
5930  * @port: The UDP port to look up
5931  *
5932  * Returns the index number or I40E_MAX_PF_UDP_OFFLOAD_PORTS if port not found
5933  **/
5934 static u8 i40e_get_vxlan_port_idx(struct i40e_pf *pf, __be16 port)
5935 {
5936         u8 i;
5937
5938         for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) {
5939                 if (pf->vxlan_ports[i] == port)
5940                         return i;
5941         }
5942
5943         return i;
5944 }
5945
5946 /**
5947  * i40e_add_vxlan_port - Get notifications about VXLAN ports that come up
5948  * @netdev: This physical port's netdev
5949  * @sa_family: Socket Family that VXLAN is notifying us about
5950  * @port: New UDP port number that VXLAN started listening to
5951  **/
5952 static void i40e_add_vxlan_port(struct net_device *netdev,
5953                                 sa_family_t sa_family, __be16 port)
5954 {
5955         struct i40e_netdev_priv *np = netdev_priv(netdev);
5956         struct i40e_vsi *vsi = np->vsi;
5957         struct i40e_pf *pf = vsi->back;
5958         u8 next_idx;
5959         u8 idx;
5960
5961         if (sa_family == AF_INET6)
5962                 return;
5963
5964         idx = i40e_get_vxlan_port_idx(pf, port);
5965
5966         /* Check if port already exists */
5967         if (idx < I40E_MAX_PF_UDP_OFFLOAD_PORTS) {
5968                 netdev_info(netdev, "Port %d already offloaded\n", ntohs(port));
5969                 return;
5970         }
5971
5972         /* Now check if there is space to add the new port */
5973         next_idx = i40e_get_vxlan_port_idx(pf, 0);
5974
5975         if (next_idx == I40E_MAX_PF_UDP_OFFLOAD_PORTS) {
5976                 netdev_info(netdev, "Maximum number of UDP ports reached, not adding port %d\n",
5977                             ntohs(port));
5978                 return;
5979         }
5980
5981         /* New port: add it and mark its index in the bitmap */
5982         pf->vxlan_ports[next_idx] = port;
5983         pf->pending_vxlan_bitmap |= (1 << next_idx);
5984
5985         pf->flags |= I40E_FLAG_VXLAN_FILTER_SYNC;
5986 }
5987
5988 /**
5989  * i40e_del_vxlan_port - Get notifications about VXLAN ports that go away
5990  * @netdev: This physical port's netdev
5991  * @sa_family: Socket Family that VXLAN is notifying us about
5992  * @port: UDP port number that VXLAN stopped listening to
5993  **/
5994 static void i40e_del_vxlan_port(struct net_device *netdev,
5995                                 sa_family_t sa_family, __be16 port)
5996 {
5997         struct i40e_netdev_priv *np = netdev_priv(netdev);
5998         struct i40e_vsi *vsi = np->vsi;
5999         struct i40e_pf *pf = vsi->back;
6000         u8 idx;
6001
6002         if (sa_family == AF_INET6)
6003                 return;
6004
6005         idx = i40e_get_vxlan_port_idx(pf, port);
6006
6007         /* Check if port already exists */
6008         if (idx < I40E_MAX_PF_UDP_OFFLOAD_PORTS) {
6009                 /* if port exists, set it to 0 (mark for deletion)
6010                  * and make it pending
6011                  */
6012                 pf->vxlan_ports[idx] = 0;
6013
6014                 pf->pending_vxlan_bitmap |= (1 << idx);
6015
6016                 pf->flags |= I40E_FLAG_VXLAN_FILTER_SYNC;
6017         } else {
6018                 netdev_warn(netdev, "Port %d was not found, not deleting\n",
6019                             ntohs(port));
6020         }
6021 }
6022
6023 #endif
6024 static const struct net_device_ops i40e_netdev_ops = {
6025         .ndo_open               = i40e_open,
6026         .ndo_stop               = i40e_close,
6027         .ndo_start_xmit         = i40e_lan_xmit_frame,
6028         .ndo_get_stats64        = i40e_get_netdev_stats_struct,
6029         .ndo_set_rx_mode        = i40e_set_rx_mode,
6030         .ndo_validate_addr      = eth_validate_addr,
6031         .ndo_set_mac_address    = i40e_set_mac,
6032         .ndo_change_mtu         = i40e_change_mtu,
6033         .ndo_tx_timeout         = i40e_tx_timeout,
6034         .ndo_vlan_rx_add_vid    = i40e_vlan_rx_add_vid,
6035         .ndo_vlan_rx_kill_vid   = i40e_vlan_rx_kill_vid,
6036 #ifdef CONFIG_NET_POLL_CONTROLLER
6037         .ndo_poll_controller    = i40e_netpoll,
6038 #endif
6039         .ndo_setup_tc           = i40e_setup_tc,
6040         .ndo_set_features       = i40e_set_features,
6041         .ndo_set_vf_mac         = i40e_ndo_set_vf_mac,
6042         .ndo_set_vf_vlan        = i40e_ndo_set_vf_port_vlan,
6043         .ndo_set_vf_tx_rate     = i40e_ndo_set_vf_bw,
6044         .ndo_get_vf_config      = i40e_ndo_get_vf_config,
6045 #ifdef CONFIG_I40E_VXLAN
6046         .ndo_add_vxlan_port     = i40e_add_vxlan_port,
6047         .ndo_del_vxlan_port     = i40e_del_vxlan_port,
6048 #endif
6049 };
6050
6051 /**
6052  * i40e_config_netdev - Setup the netdev flags
6053  * @vsi: the VSI being configured
6054  *
6055  * Returns 0 on success, negative value on failure
6056  **/
6057 static int i40e_config_netdev(struct i40e_vsi *vsi)
6058 {
6059         u8 brdcast[ETH_ALEN] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
6060         struct i40e_pf *pf = vsi->back;
6061         struct i40e_hw *hw = &pf->hw;
6062         struct i40e_netdev_priv *np;
6063         struct net_device *netdev;
6064         u8 mac_addr[ETH_ALEN];
6065         int etherdev_size;
6066
6067         etherdev_size = sizeof(struct i40e_netdev_priv);
6068         netdev = alloc_etherdev_mq(etherdev_size, vsi->alloc_queue_pairs);
6069         if (!netdev)
6070                 return -ENOMEM;
6071
6072         vsi->netdev = netdev;
6073         np = netdev_priv(netdev);
6074         np->vsi = vsi;
6075
6076         netdev->hw_enc_features = NETIF_F_IP_CSUM        |
6077                                   NETIF_F_GSO_UDP_TUNNEL |
6078                                   NETIF_F_TSO            |
6079                                   NETIF_F_SG;
6080
6081         netdev->features = NETIF_F_SG                  |
6082                            NETIF_F_IP_CSUM             |
6083                            NETIF_F_SCTP_CSUM           |
6084                            NETIF_F_HIGHDMA             |
6085                            NETIF_F_GSO_UDP_TUNNEL      |
6086                            NETIF_F_HW_VLAN_CTAG_TX     |
6087                            NETIF_F_HW_VLAN_CTAG_RX     |
6088                            NETIF_F_HW_VLAN_CTAG_FILTER |
6089                            NETIF_F_IPV6_CSUM           |
6090                            NETIF_F_TSO                 |
6091                            NETIF_F_TSO6                |
6092                            NETIF_F_RXCSUM              |
6093                            NETIF_F_RXHASH              |
6094                            0;
6095
6096         /* copy netdev features into list of user selectable features */
6097         netdev->hw_features |= netdev->features;
6098
6099         if (vsi->type == I40E_VSI_MAIN) {
6100                 SET_NETDEV_DEV(netdev, &pf->pdev->dev);
6101                 memcpy(mac_addr, hw->mac.perm_addr, ETH_ALEN);
6102         } else {
6103                 /* relate the VSI_VMDQ name to the VSI_MAIN name */
6104                 snprintf(netdev->name, IFNAMSIZ, "%sv%%d",
6105                          pf->vsi[pf->lan_vsi]->netdev->name);
6106                 random_ether_addr(mac_addr);
6107                 i40e_add_filter(vsi, mac_addr, I40E_VLAN_ANY, false, false);
6108         }
6109         i40e_add_filter(vsi, brdcast, I40E_VLAN_ANY, false, false);
6110
6111         memcpy(netdev->dev_addr, mac_addr, ETH_ALEN);
6112         memcpy(netdev->perm_addr, mac_addr, ETH_ALEN);
6113         /* vlan gets same features (except vlan offload)
6114          * after any tweaks for specific VSI types
6115          */
6116         netdev->vlan_features = netdev->features & ~(NETIF_F_HW_VLAN_CTAG_TX |
6117                                                      NETIF_F_HW_VLAN_CTAG_RX |
6118                                                    NETIF_F_HW_VLAN_CTAG_FILTER);
6119         netdev->priv_flags |= IFF_UNICAST_FLT;
6120         netdev->priv_flags |= IFF_SUPP_NOFCS;
6121         /* Setup netdev TC information */
6122         i40e_vsi_config_netdev_tc(vsi, vsi->tc_config.enabled_tc);
6123
6124         netdev->netdev_ops = &i40e_netdev_ops;
6125         netdev->watchdog_timeo = 5 * HZ;
6126         i40e_set_ethtool_ops(netdev);
6127
6128         return 0;
6129 }
6130
6131 /**
6132  * i40e_vsi_delete - Delete a VSI from the switch
6133  * @vsi: the VSI being removed
6134  *
6135  * Returns 0 on success, negative value on failure
6136  **/
6137 static void i40e_vsi_delete(struct i40e_vsi *vsi)
6138 {
6139         /* remove default VSI is not allowed */
6140         if (vsi == vsi->back->vsi[vsi->back->lan_vsi])
6141                 return;
6142
6143         /* there is no HW VSI for FDIR */
6144         if (vsi->type == I40E_VSI_FDIR)
6145                 return;
6146
6147         i40e_aq_delete_element(&vsi->back->hw, vsi->seid, NULL);
6148         return;
6149 }
6150
6151 /**
6152  * i40e_add_vsi - Add a VSI to the switch
6153  * @vsi: the VSI being configured
6154  *
6155  * This initializes a VSI context depending on the VSI type to be added and
6156  * passes it down to the add_vsi aq command.
6157  **/
6158 static int i40e_add_vsi(struct i40e_vsi *vsi)
6159 {
6160         int ret = -ENODEV;
6161         struct i40e_mac_filter *f, *ftmp;
6162         struct i40e_pf *pf = vsi->back;
6163         struct i40e_hw *hw = &pf->hw;
6164         struct i40e_vsi_context ctxt;
6165         u8 enabled_tc = 0x1; /* TC0 enabled */
6166         int f_count = 0;
6167
6168         memset(&ctxt, 0, sizeof(ctxt));
6169         switch (vsi->type) {
6170         case I40E_VSI_MAIN:
6171                 /* The PF's main VSI is already setup as part of the
6172                  * device initialization, so we'll not bother with
6173                  * the add_vsi call, but we will retrieve the current
6174                  * VSI context.
6175                  */
6176                 ctxt.seid = pf->main_vsi_seid;
6177                 ctxt.pf_num = pf->hw.pf_id;
6178                 ctxt.vf_num = 0;
6179                 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
6180                 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
6181                 if (ret) {
6182                         dev_info(&pf->pdev->dev,
6183                                  "couldn't get pf vsi config, err %d, aq_err %d\n",
6184                                  ret, pf->hw.aq.asq_last_status);
6185                         return -ENOENT;
6186                 }
6187                 memcpy(&vsi->info, &ctxt.info, sizeof(ctxt.info));
6188                 vsi->info.valid_sections = 0;
6189
6190                 vsi->seid = ctxt.seid;
6191                 vsi->id = ctxt.vsi_number;
6192
6193                 enabled_tc = i40e_pf_get_tc_map(pf);
6194
6195                 /* MFP mode setup queue map and update VSI */
6196                 if (pf->flags & I40E_FLAG_MFP_ENABLED) {
6197                         memset(&ctxt, 0, sizeof(ctxt));
6198                         ctxt.seid = pf->main_vsi_seid;
6199                         ctxt.pf_num = pf->hw.pf_id;
6200                         ctxt.vf_num = 0;
6201                         i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);
6202                         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
6203                         if (ret) {
6204                                 dev_info(&pf->pdev->dev,
6205                                          "update vsi failed, aq_err=%d\n",
6206                                          pf->hw.aq.asq_last_status);
6207                                 ret = -ENOENT;
6208                                 goto err;
6209                         }
6210                         /* update the local VSI info queue map */
6211                         i40e_vsi_update_queue_map(vsi, &ctxt);
6212                         vsi->info.valid_sections = 0;
6213                 } else {
6214                         /* Default/Main VSI is only enabled for TC0
6215                          * reconfigure it to enable all TCs that are
6216                          * available on the port in SFP mode.
6217                          */
6218                         ret = i40e_vsi_config_tc(vsi, enabled_tc);
6219                         if (ret) {
6220                                 dev_info(&pf->pdev->dev,
6221                                          "failed to configure TCs for main VSI tc_map 0x%08x, err %d, aq_err %d\n",
6222                                          enabled_tc, ret,
6223                                          pf->hw.aq.asq_last_status);
6224                                 ret = -ENOENT;
6225                         }
6226                 }
6227                 break;
6228
6229         case I40E_VSI_FDIR:
6230                 /* no queue mapping or actual HW VSI needed */
6231                 vsi->info.valid_sections = 0;
6232                 vsi->seid = 0;
6233                 vsi->id = 0;
6234                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
6235                 return 0;
6236                 break;
6237
6238         case I40E_VSI_VMDQ2:
6239                 ctxt.pf_num = hw->pf_id;
6240                 ctxt.vf_num = 0;
6241                 ctxt.uplink_seid = vsi->uplink_seid;
6242                 ctxt.connection_type = 0x1;     /* regular data port */
6243                 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2;
6244
6245                 ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
6246
6247                 /* This VSI is connected to VEB so the switch_id
6248                  * should be set to zero by default.
6249                  */
6250                 ctxt.info.switch_id = 0;
6251                 ctxt.info.switch_id |= cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_LOCAL_LB);
6252                 ctxt.info.switch_id |= cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
6253
6254                 /* Setup the VSI tx/rx queue map for TC0 only for now */
6255                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
6256                 break;
6257
6258         case I40E_VSI_SRIOV:
6259                 ctxt.pf_num = hw->pf_id;
6260                 ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id;
6261                 ctxt.uplink_seid = vsi->uplink_seid;
6262                 ctxt.connection_type = 0x1;     /* regular data port */
6263                 ctxt.flags = I40E_AQ_VSI_TYPE_VF;
6264
6265                 ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
6266
6267                 /* This VSI is connected to VEB so the switch_id
6268                  * should be set to zero by default.
6269                  */
6270                 ctxt.info.switch_id = cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
6271
6272                 ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
6273                 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL;
6274                 /* Setup the VSI tx/rx queue map for TC0 only for now */
6275                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
6276                 break;
6277
6278         default:
6279                 return -ENODEV;
6280         }
6281
6282         if (vsi->type != I40E_VSI_MAIN) {
6283                 ret = i40e_aq_add_vsi(hw, &ctxt, NULL);
6284                 if (ret) {
6285                         dev_info(&vsi->back->pdev->dev,
6286                                  "add vsi failed, aq_err=%d\n",
6287                                  vsi->back->hw.aq.asq_last_status);
6288                         ret = -ENOENT;
6289                         goto err;
6290                 }
6291                 memcpy(&vsi->info, &ctxt.info, sizeof(ctxt.info));
6292                 vsi->info.valid_sections = 0;
6293                 vsi->seid = ctxt.seid;
6294                 vsi->id = ctxt.vsi_number;
6295         }
6296
6297         /* If macvlan filters already exist, force them to get loaded */
6298         list_for_each_entry_safe(f, ftmp, &vsi->mac_filter_list, list) {
6299                 f->changed = true;
6300                 f_count++;
6301         }
6302         if (f_count) {
6303                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
6304                 pf->flags |= I40E_FLAG_FILTER_SYNC;
6305         }
6306
6307         /* Update VSI BW information */
6308         ret = i40e_vsi_get_bw_info(vsi);
6309         if (ret) {
6310                 dev_info(&pf->pdev->dev,
6311                          "couldn't get vsi bw info, err %d, aq_err %d\n",
6312                          ret, pf->hw.aq.asq_last_status);
6313                 /* VSI is already added so not tearing that up */
6314                 ret = 0;
6315         }
6316
6317 err:
6318         return ret;
6319 }
6320
6321 /**
6322  * i40e_vsi_release - Delete a VSI and free its resources
6323  * @vsi: the VSI being removed
6324  *
6325  * Returns 0 on success or < 0 on error
6326  **/
6327 int i40e_vsi_release(struct i40e_vsi *vsi)
6328 {
6329         struct i40e_mac_filter *f, *ftmp;
6330         struct i40e_veb *veb = NULL;
6331         struct i40e_pf *pf;
6332         u16 uplink_seid;
6333         int i, n;
6334
6335         pf = vsi->back;
6336
6337         /* release of a VEB-owner or last VSI is not allowed */
6338         if (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) {
6339                 dev_info(&pf->pdev->dev, "VSI %d has existing VEB %d\n",
6340                          vsi->seid, vsi->uplink_seid);
6341                 return -ENODEV;
6342         }
6343         if (vsi == pf->vsi[pf->lan_vsi] &&
6344             !test_bit(__I40E_DOWN, &pf->state)) {
6345                 dev_info(&pf->pdev->dev, "Can't remove PF VSI\n");
6346                 return -ENODEV;
6347         }
6348
6349         uplink_seid = vsi->uplink_seid;
6350         if (vsi->type != I40E_VSI_SRIOV) {
6351                 if (vsi->netdev_registered) {
6352                         vsi->netdev_registered = false;
6353                         if (vsi->netdev) {
6354                                 /* results in a call to i40e_close() */
6355                                 unregister_netdev(vsi->netdev);
6356                                 free_netdev(vsi->netdev);
6357                                 vsi->netdev = NULL;
6358                         }
6359                 } else {
6360                         if (!test_and_set_bit(__I40E_DOWN, &vsi->state))
6361                                 i40e_down(vsi);
6362                         i40e_vsi_free_irq(vsi);
6363                         i40e_vsi_free_tx_resources(vsi);
6364                         i40e_vsi_free_rx_resources(vsi);
6365                 }
6366                 i40e_vsi_disable_irq(vsi);
6367         }
6368
6369         list_for_each_entry_safe(f, ftmp, &vsi->mac_filter_list, list)
6370                 i40e_del_filter(vsi, f->macaddr, f->vlan,
6371                                 f->is_vf, f->is_netdev);
6372         i40e_sync_vsi_filters(vsi);
6373
6374         i40e_vsi_delete(vsi);
6375         i40e_vsi_free_q_vectors(vsi);
6376         i40e_vsi_clear_rings(vsi);
6377         i40e_vsi_clear(vsi);
6378
6379         /* If this was the last thing on the VEB, except for the
6380          * controlling VSI, remove the VEB, which puts the controlling
6381          * VSI onto the next level down in the switch.
6382          *
6383          * Well, okay, there's one more exception here: don't remove
6384          * the orphan VEBs yet.  We'll wait for an explicit remove request
6385          * from up the network stack.
6386          */
6387         for (n = 0, i = 0; i < pf->hw.func_caps.num_vsis; i++) {
6388                 if (pf->vsi[i] &&
6389                     pf->vsi[i]->uplink_seid == uplink_seid &&
6390                     (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) {
6391                         n++;      /* count the VSIs */
6392                 }
6393         }
6394         for (i = 0; i < I40E_MAX_VEB; i++) {
6395                 if (!pf->veb[i])
6396                         continue;
6397                 if (pf->veb[i]->uplink_seid == uplink_seid)
6398                         n++;     /* count the VEBs */
6399                 if (pf->veb[i]->seid == uplink_seid)
6400                         veb = pf->veb[i];
6401         }
6402         if (n == 0 && veb && veb->uplink_seid != 0)
6403                 i40e_veb_release(veb);
6404
6405         return 0;
6406 }
6407
6408 /**
6409  * i40e_vsi_setup_vectors - Set up the q_vectors for the given VSI
6410  * @vsi: ptr to the VSI
6411  *
6412  * This should only be called after i40e_vsi_mem_alloc() which allocates the
6413  * corresponding SW VSI structure and initializes num_queue_pairs for the
6414  * newly allocated VSI.
6415  *
6416  * Returns 0 on success or negative on failure
6417  **/
6418 static int i40e_vsi_setup_vectors(struct i40e_vsi *vsi)
6419 {
6420         int ret = -ENOENT;
6421         struct i40e_pf *pf = vsi->back;
6422
6423         if (vsi->q_vectors[0]) {
6424                 dev_info(&pf->pdev->dev, "VSI %d has existing q_vectors\n",
6425                          vsi->seid);
6426                 return -EEXIST;
6427         }
6428
6429         if (vsi->base_vector) {
6430                 dev_info(&pf->pdev->dev,
6431                          "VSI %d has non-zero base vector %d\n",
6432                          vsi->seid, vsi->base_vector);
6433                 return -EEXIST;
6434         }
6435
6436         ret = i40e_alloc_q_vectors(vsi);
6437         if (ret) {
6438                 dev_info(&pf->pdev->dev,
6439                          "failed to allocate %d q_vector for VSI %d, ret=%d\n",
6440                          vsi->num_q_vectors, vsi->seid, ret);
6441                 vsi->num_q_vectors = 0;
6442                 goto vector_setup_out;
6443         }
6444
6445         if (vsi->num_q_vectors)
6446                 vsi->base_vector = i40e_get_lump(pf, pf->irq_pile,
6447                                                  vsi->num_q_vectors, vsi->idx);
6448         if (vsi->base_vector < 0) {
6449                 dev_info(&pf->pdev->dev,
6450                          "failed to get q tracking for VSI %d, err=%d\n",
6451                          vsi->seid, vsi->base_vector);
6452                 i40e_vsi_free_q_vectors(vsi);
6453                 ret = -ENOENT;
6454                 goto vector_setup_out;
6455         }
6456
6457 vector_setup_out:
6458         return ret;
6459 }
6460
6461 /**
6462  * i40e_vsi_reinit_setup - return and reallocate resources for a VSI
6463  * @vsi: pointer to the vsi.
6464  *
6465  * This re-allocates a vsi's queue resources.
6466  *
6467  * Returns pointer to the successfully allocated and configured VSI sw struct
6468  * on success, otherwise returns NULL on failure.
6469  **/
6470 static struct i40e_vsi *i40e_vsi_reinit_setup(struct i40e_vsi *vsi)
6471 {
6472         struct i40e_pf *pf = vsi->back;
6473         u8 enabled_tc;
6474         int ret;
6475
6476         i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);
6477         i40e_vsi_clear_rings(vsi);
6478
6479         i40e_vsi_free_arrays(vsi, false);
6480         i40e_set_num_rings_in_vsi(vsi);
6481         ret = i40e_vsi_alloc_arrays(vsi, false);
6482         if (ret)
6483                 goto err_vsi;
6484
6485         ret = i40e_get_lump(pf, pf->qp_pile, vsi->alloc_queue_pairs, vsi->idx);
6486         if (ret < 0) {
6487                 dev_info(&pf->pdev->dev, "VSI %d get_lump failed %d\n",
6488                          vsi->seid, ret);
6489                 goto err_vsi;
6490         }
6491         vsi->base_queue = ret;
6492
6493         /* Update the FW view of the VSI. Force a reset of TC and queue
6494          * layout configurations.
6495          */
6496         enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc;
6497         pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0;
6498         pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid;
6499         i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc);
6500
6501         /* assign it some queues */
6502         ret = i40e_alloc_rings(vsi);
6503         if (ret)
6504                 goto err_rings;
6505
6506         /* map all of the rings to the q_vectors */
6507         i40e_vsi_map_rings_to_vectors(vsi);
6508         return vsi;
6509
6510 err_rings:
6511         i40e_vsi_free_q_vectors(vsi);
6512         if (vsi->netdev_registered) {
6513                 vsi->netdev_registered = false;
6514                 unregister_netdev(vsi->netdev);
6515                 free_netdev(vsi->netdev);
6516                 vsi->netdev = NULL;
6517         }
6518         i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);
6519 err_vsi:
6520         i40e_vsi_clear(vsi);
6521         return NULL;
6522 }
6523
6524 /**
6525  * i40e_vsi_setup - Set up a VSI by a given type
6526  * @pf: board private structure
6527  * @type: VSI type
6528  * @uplink_seid: the switch element to link to
6529  * @param1: usage depends upon VSI type. For VF types, indicates VF id
6530  *
6531  * This allocates the sw VSI structure and its queue resources, then add a VSI
6532  * to the identified VEB.
6533  *
6534  * Returns pointer to the successfully allocated and configure VSI sw struct on
6535  * success, otherwise returns NULL on failure.
6536  **/
6537 struct i40e_vsi *i40e_vsi_setup(struct i40e_pf *pf, u8 type,
6538                                 u16 uplink_seid, u32 param1)
6539 {
6540         struct i40e_vsi *vsi = NULL;
6541         struct i40e_veb *veb = NULL;
6542         int ret, i;
6543         int v_idx;
6544
6545         /* The requested uplink_seid must be either
6546          *     - the PF's port seid
6547          *              no VEB is needed because this is the PF
6548          *              or this is a Flow Director special case VSI
6549          *     - seid of an existing VEB
6550          *     - seid of a VSI that owns an existing VEB
6551          *     - seid of a VSI that doesn't own a VEB
6552          *              a new VEB is created and the VSI becomes the owner
6553          *     - seid of the PF VSI, which is what creates the first VEB
6554          *              this is a special case of the previous
6555          *
6556          * Find which uplink_seid we were given and create a new VEB if needed
6557          */
6558         for (i = 0; i < I40E_MAX_VEB; i++) {
6559                 if (pf->veb[i] && pf->veb[i]->seid == uplink_seid) {
6560                         veb = pf->veb[i];
6561                         break;
6562                 }
6563         }
6564
6565         if (!veb && uplink_seid != pf->mac_seid) {
6566
6567                 for (i = 0; i < pf->hw.func_caps.num_vsis; i++) {
6568                         if (pf->vsi[i] && pf->vsi[i]->seid == uplink_seid) {
6569                                 vsi = pf->vsi[i];
6570                                 break;
6571                         }
6572                 }
6573                 if (!vsi) {
6574                         dev_info(&pf->pdev->dev, "no such uplink_seid %d\n",
6575                                  uplink_seid);
6576                         return NULL;
6577                 }
6578
6579                 if (vsi->uplink_seid == pf->mac_seid)
6580                         veb = i40e_veb_setup(pf, 0, pf->mac_seid, vsi->seid,
6581                                              vsi->tc_config.enabled_tc);
6582                 else if ((vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0)
6583                         veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid,
6584                                              vsi->tc_config.enabled_tc);
6585
6586                 for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
6587                         if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
6588                                 veb = pf->veb[i];
6589                 }
6590                 if (!veb) {
6591                         dev_info(&pf->pdev->dev, "couldn't add VEB\n");
6592                         return NULL;
6593                 }
6594
6595                 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;
6596                 uplink_seid = veb->seid;
6597         }
6598
6599         /* get vsi sw struct */
6600         v_idx = i40e_vsi_mem_alloc(pf, type);
6601         if (v_idx < 0)
6602                 goto err_alloc;
6603         vsi = pf->vsi[v_idx];
6604         vsi->type = type;
6605         vsi->veb_idx = (veb ? veb->idx : I40E_NO_VEB);
6606
6607         if (type == I40E_VSI_MAIN)
6608                 pf->lan_vsi = v_idx;
6609         else if (type == I40E_VSI_SRIOV)
6610                 vsi->vf_id = param1;
6611         /* assign it some queues */
6612         ret = i40e_get_lump(pf, pf->qp_pile, vsi->alloc_queue_pairs, vsi->idx);
6613         if (ret < 0) {
6614                 dev_info(&pf->pdev->dev, "VSI %d get_lump failed %d\n",
6615                          vsi->seid, ret);
6616                 goto err_vsi;
6617         }
6618         vsi->base_queue = ret;
6619
6620         /* get a VSI from the hardware */
6621         vsi->uplink_seid = uplink_seid;
6622         ret = i40e_add_vsi(vsi);
6623         if (ret)
6624                 goto err_vsi;
6625
6626         switch (vsi->type) {
6627         /* setup the netdev if needed */
6628         case I40E_VSI_MAIN:
6629         case I40E_VSI_VMDQ2:
6630                 ret = i40e_config_netdev(vsi);
6631                 if (ret)
6632                         goto err_netdev;
6633                 ret = register_netdev(vsi->netdev);
6634                 if (ret)
6635                         goto err_netdev;
6636                 vsi->netdev_registered = true;
6637                 netif_carrier_off(vsi->netdev);
6638                 /* fall through */
6639
6640         case I40E_VSI_FDIR:
6641                 /* set up vectors and rings if needed */
6642                 ret = i40e_vsi_setup_vectors(vsi);
6643                 if (ret)
6644                         goto err_msix;
6645
6646                 ret = i40e_alloc_rings(vsi);
6647                 if (ret)
6648                         goto err_rings;
6649
6650                 /* map all of the rings to the q_vectors */
6651                 i40e_vsi_map_rings_to_vectors(vsi);
6652
6653                 i40e_vsi_reset_stats(vsi);
6654                 break;
6655
6656         default:
6657                 /* no netdev or rings for the other VSI types */
6658                 break;
6659         }
6660
6661         return vsi;
6662
6663 err_rings:
6664         i40e_vsi_free_q_vectors(vsi);
6665 err_msix:
6666         if (vsi->netdev_registered) {
6667                 vsi->netdev_registered = false;
6668                 unregister_netdev(vsi->netdev);
6669                 free_netdev(vsi->netdev);
6670                 vsi->netdev = NULL;
6671         }
6672 err_netdev:
6673         i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);
6674 err_vsi:
6675         i40e_vsi_clear(vsi);
6676 err_alloc:
6677         return NULL;
6678 }
6679
6680 /**
6681  * i40e_veb_get_bw_info - Query VEB BW information
6682  * @veb: the veb to query
6683  *
6684  * Query the Tx scheduler BW configuration data for given VEB
6685  **/
6686 static int i40e_veb_get_bw_info(struct i40e_veb *veb)
6687 {
6688         struct i40e_aqc_query_switching_comp_ets_config_resp ets_data;
6689         struct i40e_aqc_query_switching_comp_bw_config_resp bw_data;
6690         struct i40e_pf *pf = veb->pf;
6691         struct i40e_hw *hw = &pf->hw;
6692         u32 tc_bw_max;
6693         int ret = 0;
6694         int i;
6695
6696         ret = i40e_aq_query_switch_comp_bw_config(hw, veb->seid,
6697                                                   &bw_data, NULL);
6698         if (ret) {
6699                 dev_info(&pf->pdev->dev,
6700                          "query veb bw config failed, aq_err=%d\n",
6701                          hw->aq.asq_last_status);
6702                 goto out;
6703         }
6704
6705         ret = i40e_aq_query_switch_comp_ets_config(hw, veb->seid,
6706                                                    &ets_data, NULL);
6707         if (ret) {
6708                 dev_info(&pf->pdev->dev,
6709                          "query veb bw ets config failed, aq_err=%d\n",
6710                          hw->aq.asq_last_status);
6711                 goto out;
6712         }
6713
6714         veb->bw_limit = le16_to_cpu(ets_data.port_bw_limit);
6715         veb->bw_max_quanta = ets_data.tc_bw_max;
6716         veb->is_abs_credits = bw_data.absolute_credits_enable;
6717         tc_bw_max = le16_to_cpu(bw_data.tc_bw_max[0]) |
6718                     (le16_to_cpu(bw_data.tc_bw_max[1]) << 16);
6719         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
6720                 veb->bw_tc_share_credits[i] = bw_data.tc_bw_share_credits[i];
6721                 veb->bw_tc_limit_credits[i] =
6722                                         le16_to_cpu(bw_data.tc_bw_limits[i]);
6723                 veb->bw_tc_max_quanta[i] = ((tc_bw_max >> (i*4)) & 0x7);
6724         }
6725
6726 out:
6727         return ret;
6728 }
6729
6730 /**
6731  * i40e_veb_mem_alloc - Allocates the next available struct veb in the PF
6732  * @pf: board private structure
6733  *
6734  * On error: returns error code (negative)
6735  * On success: returns vsi index in PF (positive)
6736  **/
6737 static int i40e_veb_mem_alloc(struct i40e_pf *pf)
6738 {
6739         int ret = -ENOENT;
6740         struct i40e_veb *veb;
6741         int i;
6742
6743         /* Need to protect the allocation of switch elements at the PF level */
6744         mutex_lock(&pf->switch_mutex);
6745
6746         /* VEB list may be fragmented if VEB creation/destruction has
6747          * been happening.  We can afford to do a quick scan to look
6748          * for any free slots in the list.
6749          *
6750          * find next empty veb slot, looping back around if necessary
6751          */
6752         i = 0;
6753         while ((i < I40E_MAX_VEB) && (pf->veb[i] != NULL))
6754                 i++;
6755         if (i >= I40E_MAX_VEB) {
6756                 ret = -ENOMEM;
6757                 goto err_alloc_veb;  /* out of VEB slots! */
6758         }
6759
6760         veb = kzalloc(sizeof(*veb), GFP_KERNEL);
6761         if (!veb) {
6762                 ret = -ENOMEM;
6763                 goto err_alloc_veb;
6764         }
6765         veb->pf = pf;
6766         veb->idx = i;
6767         veb->enabled_tc = 1;
6768
6769         pf->veb[i] = veb;
6770         ret = i;
6771 err_alloc_veb:
6772         mutex_unlock(&pf->switch_mutex);
6773         return ret;
6774 }
6775
6776 /**
6777  * i40e_switch_branch_release - Delete a branch of the switch tree
6778  * @branch: where to start deleting
6779  *
6780  * This uses recursion to find the tips of the branch to be
6781  * removed, deleting until we get back to and can delete this VEB.
6782  **/
6783 static void i40e_switch_branch_release(struct i40e_veb *branch)
6784 {
6785         struct i40e_pf *pf = branch->pf;
6786         u16 branch_seid = branch->seid;
6787         u16 veb_idx = branch->idx;
6788         int i;
6789
6790         /* release any VEBs on this VEB - RECURSION */
6791         for (i = 0; i < I40E_MAX_VEB; i++) {
6792                 if (!pf->veb[i])
6793                         continue;
6794                 if (pf->veb[i]->uplink_seid == branch->seid)
6795                         i40e_switch_branch_release(pf->veb[i]);
6796         }
6797
6798         /* Release the VSIs on this VEB, but not the owner VSI.
6799          *
6800          * NOTE: Removing the last VSI on a VEB has the SIDE EFFECT of removing
6801          *       the VEB itself, so don't use (*branch) after this loop.
6802          */
6803         for (i = 0; i < pf->hw.func_caps.num_vsis; i++) {
6804                 if (!pf->vsi[i])
6805                         continue;
6806                 if (pf->vsi[i]->uplink_seid == branch_seid &&
6807                    (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) {
6808                         i40e_vsi_release(pf->vsi[i]);
6809                 }
6810         }
6811
6812         /* There's one corner case where the VEB might not have been
6813          * removed, so double check it here and remove it if needed.
6814          * This case happens if the veb was created from the debugfs
6815          * commands and no VSIs were added to it.
6816          */
6817         if (pf->veb[veb_idx])
6818                 i40e_veb_release(pf->veb[veb_idx]);
6819 }
6820
6821 /**
6822  * i40e_veb_clear - remove veb struct
6823  * @veb: the veb to remove
6824  **/
6825 static void i40e_veb_clear(struct i40e_veb *veb)
6826 {
6827         if (!veb)
6828                 return;
6829
6830         if (veb->pf) {
6831                 struct i40e_pf *pf = veb->pf;
6832
6833                 mutex_lock(&pf->switch_mutex);
6834                 if (pf->veb[veb->idx] == veb)
6835                         pf->veb[veb->idx] = NULL;
6836                 mutex_unlock(&pf->switch_mutex);
6837         }
6838
6839         kfree(veb);
6840 }
6841
6842 /**
6843  * i40e_veb_release - Delete a VEB and free its resources
6844  * @veb: the VEB being removed
6845  **/
6846 void i40e_veb_release(struct i40e_veb *veb)
6847 {
6848         struct i40e_vsi *vsi = NULL;
6849         struct i40e_pf *pf;
6850         int i, n = 0;
6851
6852         pf = veb->pf;
6853
6854         /* find the remaining VSI and check for extras */
6855         for (i = 0; i < pf->hw.func_caps.num_vsis; i++) {
6856                 if (pf->vsi[i] && pf->vsi[i]->uplink_seid == veb->seid) {
6857                         n++;
6858                         vsi = pf->vsi[i];
6859                 }
6860         }
6861         if (n != 1) {
6862                 dev_info(&pf->pdev->dev,
6863                          "can't remove VEB %d with %d VSIs left\n",
6864                          veb->seid, n);
6865                 return;
6866         }
6867
6868         /* move the remaining VSI to uplink veb */
6869         vsi->flags &= ~I40E_VSI_FLAG_VEB_OWNER;
6870         if (veb->uplink_seid) {
6871                 vsi->uplink_seid = veb->uplink_seid;
6872                 if (veb->uplink_seid == pf->mac_seid)
6873                         vsi->veb_idx = I40E_NO_VEB;
6874                 else
6875                         vsi->veb_idx = veb->veb_idx;
6876         } else {
6877                 /* floating VEB */
6878                 vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid;
6879                 vsi->veb_idx = pf->vsi[pf->lan_vsi]->veb_idx;
6880         }
6881
6882         i40e_aq_delete_element(&pf->hw, veb->seid, NULL);
6883         i40e_veb_clear(veb);
6884
6885         return;
6886 }
6887
6888 /**
6889  * i40e_add_veb - create the VEB in the switch
6890  * @veb: the VEB to be instantiated
6891  * @vsi: the controlling VSI
6892  **/
6893 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi)
6894 {
6895         bool is_default = false;
6896         bool is_cloud = false;
6897         int ret;
6898
6899         /* get a VEB from the hardware */
6900         ret = i40e_aq_add_veb(&veb->pf->hw, veb->uplink_seid, vsi->seid,
6901                               veb->enabled_tc, is_default,
6902                               is_cloud, &veb->seid, NULL);
6903         if (ret) {
6904                 dev_info(&veb->pf->pdev->dev,
6905                          "couldn't add VEB, err %d, aq_err %d\n",
6906                          ret, veb->pf->hw.aq.asq_last_status);
6907                 return -EPERM;
6908         }
6909
6910         /* get statistics counter */
6911         ret = i40e_aq_get_veb_parameters(&veb->pf->hw, veb->seid, NULL, NULL,
6912                                          &veb->stats_idx, NULL, NULL, NULL);
6913         if (ret) {
6914                 dev_info(&veb->pf->pdev->dev,
6915                          "couldn't get VEB statistics idx, err %d, aq_err %d\n",
6916                          ret, veb->pf->hw.aq.asq_last_status);
6917                 return -EPERM;
6918         }
6919         ret = i40e_veb_get_bw_info(veb);
6920         if (ret) {
6921                 dev_info(&veb->pf->pdev->dev,
6922                          "couldn't get VEB bw info, err %d, aq_err %d\n",
6923                          ret, veb->pf->hw.aq.asq_last_status);
6924                 i40e_aq_delete_element(&veb->pf->hw, veb->seid, NULL);
6925                 return -ENOENT;
6926         }
6927
6928         vsi->uplink_seid = veb->seid;
6929         vsi->veb_idx = veb->idx;
6930         vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;
6931
6932         return 0;
6933 }
6934
6935 /**
6936  * i40e_veb_setup - Set up a VEB
6937  * @pf: board private structure
6938  * @flags: VEB setup flags
6939  * @uplink_seid: the switch element to link to
6940  * @vsi_seid: the initial VSI seid
6941  * @enabled_tc: Enabled TC bit-map
6942  *
6943  * This allocates the sw VEB structure and links it into the switch
6944  * It is possible and legal for this to be a duplicate of an already
6945  * existing VEB.  It is also possible for both uplink and vsi seids
6946  * to be zero, in order to create a floating VEB.
6947  *
6948  * Returns pointer to the successfully allocated VEB sw struct on
6949  * success, otherwise returns NULL on failure.
6950  **/
6951 struct i40e_veb *i40e_veb_setup(struct i40e_pf *pf, u16 flags,
6952                                 u16 uplink_seid, u16 vsi_seid,
6953                                 u8 enabled_tc)
6954 {
6955         struct i40e_veb *veb, *uplink_veb = NULL;
6956         int vsi_idx, veb_idx;
6957         int ret;
6958
6959         /* if one seid is 0, the other must be 0 to create a floating relay */
6960         if ((uplink_seid == 0 || vsi_seid == 0) &&
6961             (uplink_seid + vsi_seid != 0)) {
6962                 dev_info(&pf->pdev->dev,
6963                          "one, not both seid's are 0: uplink=%d vsi=%d\n",
6964                          uplink_seid, vsi_seid);
6965                 return NULL;
6966         }
6967
6968         /* make sure there is such a vsi and uplink */
6969         for (vsi_idx = 0; vsi_idx < pf->hw.func_caps.num_vsis; vsi_idx++)
6970                 if (pf->vsi[vsi_idx] && pf->vsi[vsi_idx]->seid == vsi_seid)
6971                         break;
6972         if (vsi_idx >= pf->hw.func_caps.num_vsis && vsi_seid != 0) {
6973                 dev_info(&pf->pdev->dev, "vsi seid %d not found\n",
6974                          vsi_seid);
6975                 return NULL;
6976         }
6977
6978         if (uplink_seid && uplink_seid != pf->mac_seid) {
6979                 for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) {
6980                         if (pf->veb[veb_idx] &&
6981                             pf->veb[veb_idx]->seid == uplink_seid) {
6982                                 uplink_veb = pf->veb[veb_idx];
6983                                 break;
6984                         }
6985                 }
6986                 if (!uplink_veb) {
6987                         dev_info(&pf->pdev->dev,
6988                                  "uplink seid %d not found\n", uplink_seid);
6989                         return NULL;
6990                 }
6991         }
6992
6993         /* get veb sw struct */
6994         veb_idx = i40e_veb_mem_alloc(pf);
6995         if (veb_idx < 0)
6996                 goto err_alloc;
6997         veb = pf->veb[veb_idx];
6998         veb->flags = flags;
6999         veb->uplink_seid = uplink_seid;
7000         veb->veb_idx = (uplink_veb ? uplink_veb->idx : I40E_NO_VEB);
7001         veb->enabled_tc = (enabled_tc ? enabled_tc : 0x1);
7002
7003         /* create the VEB in the switch */
7004         ret = i40e_add_veb(veb, pf->vsi[vsi_idx]);
7005         if (ret)
7006                 goto err_veb;
7007
7008         return veb;
7009
7010 err_veb:
7011         i40e_veb_clear(veb);
7012 err_alloc:
7013         return NULL;
7014 }
7015
7016 /**
7017  * i40e_setup_pf_switch_element - set pf vars based on switch type
7018  * @pf: board private structure
7019  * @ele: element we are building info from
7020  * @num_reported: total number of elements
7021  * @printconfig: should we print the contents
7022  *
7023  * helper function to assist in extracting a few useful SEID values.
7024  **/
7025 static void i40e_setup_pf_switch_element(struct i40e_pf *pf,
7026                                 struct i40e_aqc_switch_config_element_resp *ele,
7027                                 u16 num_reported, bool printconfig)
7028 {
7029         u16 downlink_seid = le16_to_cpu(ele->downlink_seid);
7030         u16 uplink_seid = le16_to_cpu(ele->uplink_seid);
7031         u8 element_type = ele->element_type;
7032         u16 seid = le16_to_cpu(ele->seid);
7033
7034         if (printconfig)
7035                 dev_info(&pf->pdev->dev,
7036                          "type=%d seid=%d uplink=%d downlink=%d\n",
7037                          element_type, seid, uplink_seid, downlink_seid);
7038
7039         switch (element_type) {
7040         case I40E_SWITCH_ELEMENT_TYPE_MAC:
7041                 pf->mac_seid = seid;
7042                 break;
7043         case I40E_SWITCH_ELEMENT_TYPE_VEB:
7044                 /* Main VEB? */
7045                 if (uplink_seid != pf->mac_seid)
7046                         break;
7047                 if (pf->lan_veb == I40E_NO_VEB) {
7048                         int v;
7049
7050                         /* find existing or else empty VEB */
7051                         for (v = 0; v < I40E_MAX_VEB; v++) {
7052                                 if (pf->veb[v] && (pf->veb[v]->seid == seid)) {
7053                                         pf->lan_veb = v;
7054                                         break;
7055                                 }
7056                         }
7057                         if (pf->lan_veb == I40E_NO_VEB) {
7058                                 v = i40e_veb_mem_alloc(pf);
7059                                 if (v < 0)
7060                                         break;
7061                                 pf->lan_veb = v;
7062                         }
7063                 }
7064
7065                 pf->veb[pf->lan_veb]->seid = seid;
7066                 pf->veb[pf->lan_veb]->uplink_seid = pf->mac_seid;
7067                 pf->veb[pf->lan_veb]->pf = pf;
7068                 pf->veb[pf->lan_veb]->veb_idx = I40E_NO_VEB;
7069                 break;
7070         case I40E_SWITCH_ELEMENT_TYPE_VSI:
7071                 if (num_reported != 1)
7072                         break;
7073                 /* This is immediately after a reset so we can assume this is
7074                  * the PF's VSI
7075                  */
7076                 pf->mac_seid = uplink_seid;
7077                 pf->pf_seid = downlink_seid;
7078                 pf->main_vsi_seid = seid;
7079                 if (printconfig)
7080                         dev_info(&pf->pdev->dev,
7081                                  "pf_seid=%d main_vsi_seid=%d\n",
7082                                  pf->pf_seid, pf->main_vsi_seid);
7083                 break;
7084         case I40E_SWITCH_ELEMENT_TYPE_PF:
7085         case I40E_SWITCH_ELEMENT_TYPE_VF:
7086         case I40E_SWITCH_ELEMENT_TYPE_EMP:
7087         case I40E_SWITCH_ELEMENT_TYPE_BMC:
7088         case I40E_SWITCH_ELEMENT_TYPE_PE:
7089         case I40E_SWITCH_ELEMENT_TYPE_PA:
7090                 /* ignore these for now */
7091                 break;
7092         default:
7093                 dev_info(&pf->pdev->dev, "unknown element type=%d seid=%d\n",
7094                          element_type, seid);
7095                 break;
7096         }
7097 }
7098
7099 /**
7100  * i40e_fetch_switch_configuration - Get switch config from firmware
7101  * @pf: board private structure
7102  * @printconfig: should we print the contents
7103  *
7104  * Get the current switch configuration from the device and
7105  * extract a few useful SEID values.
7106  **/
7107 int i40e_fetch_switch_configuration(struct i40e_pf *pf, bool printconfig)
7108 {
7109         struct i40e_aqc_get_switch_config_resp *sw_config;
7110         u16 next_seid = 0;
7111         int ret = 0;
7112         u8 *aq_buf;
7113         int i;
7114
7115         aq_buf = kzalloc(I40E_AQ_LARGE_BUF, GFP_KERNEL);
7116         if (!aq_buf)
7117                 return -ENOMEM;
7118
7119         sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf;
7120         do {
7121                 u16 num_reported, num_total;
7122
7123                 ret = i40e_aq_get_switch_config(&pf->hw, sw_config,
7124                                                 I40E_AQ_LARGE_BUF,
7125                                                 &next_seid, NULL);
7126                 if (ret) {
7127                         dev_info(&pf->pdev->dev,
7128                                  "get switch config failed %d aq_err=%x\n",
7129                                  ret, pf->hw.aq.asq_last_status);
7130                         kfree(aq_buf);
7131                         return -ENOENT;
7132                 }
7133
7134                 num_reported = le16_to_cpu(sw_config->header.num_reported);
7135                 num_total = le16_to_cpu(sw_config->header.num_total);
7136
7137                 if (printconfig)
7138                         dev_info(&pf->pdev->dev,
7139                                  "header: %d reported %d total\n",
7140                                  num_reported, num_total);
7141
7142                 if (num_reported) {
7143                         int sz = sizeof(*sw_config) * num_reported;
7144
7145                         kfree(pf->sw_config);
7146                         pf->sw_config = kzalloc(sz, GFP_KERNEL);
7147                         if (pf->sw_config)
7148                                 memcpy(pf->sw_config, sw_config, sz);
7149                 }
7150
7151                 for (i = 0; i < num_reported; i++) {
7152                         struct i40e_aqc_switch_config_element_resp *ele =
7153                                 &sw_config->element[i];
7154
7155                         i40e_setup_pf_switch_element(pf, ele, num_reported,
7156                                                      printconfig);
7157                 }
7158         } while (next_seid != 0);
7159
7160         kfree(aq_buf);
7161         return ret;
7162 }
7163
7164 /**
7165  * i40e_setup_pf_switch - Setup the HW switch on startup or after reset
7166  * @pf: board private structure
7167  * @reinit: if the Main VSI needs to re-initialized.
7168  *
7169  * Returns 0 on success, negative value on failure
7170  **/
7171 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit)
7172 {
7173         u32 rxfc = 0, txfc = 0, rxfc_reg;
7174         int ret;
7175
7176         /* find out what's out there already */
7177         ret = i40e_fetch_switch_configuration(pf, false);
7178         if (ret) {
7179                 dev_info(&pf->pdev->dev,
7180                          "couldn't fetch switch config, err %d, aq_err %d\n",
7181                          ret, pf->hw.aq.asq_last_status);
7182                 return ret;
7183         }
7184         i40e_pf_reset_stats(pf);
7185
7186         /* fdir VSI must happen first to be sure it gets queue 0, but only
7187          * if there is enough room for the fdir VSI
7188          */
7189         if (pf->num_lan_qps > 1)
7190                 i40e_fdir_setup(pf);
7191
7192         /* first time setup */
7193         if (pf->lan_vsi == I40E_NO_VSI || reinit) {
7194                 struct i40e_vsi *vsi = NULL;
7195                 u16 uplink_seid;
7196
7197                 /* Set up the PF VSI associated with the PF's main VSI
7198                  * that is already in the HW switch
7199                  */
7200                 if (pf->lan_veb != I40E_NO_VEB && pf->veb[pf->lan_veb])
7201                         uplink_seid = pf->veb[pf->lan_veb]->seid;
7202                 else
7203                         uplink_seid = pf->mac_seid;
7204                 if (pf->lan_vsi == I40E_NO_VSI)
7205                         vsi = i40e_vsi_setup(pf, I40E_VSI_MAIN, uplink_seid, 0);
7206                 else if (reinit)
7207                         vsi = i40e_vsi_reinit_setup(pf->vsi[pf->lan_vsi]);
7208                 if (!vsi) {
7209                         dev_info(&pf->pdev->dev, "setup of MAIN VSI failed\n");
7210                         i40e_fdir_teardown(pf);
7211                         return -EAGAIN;
7212                 }
7213         } else {
7214                 /* force a reset of TC and queue layout configurations */
7215                 u8 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc;
7216                 pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0;
7217                 pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid;
7218                 i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc);
7219         }
7220         i40e_vlan_stripping_disable(pf->vsi[pf->lan_vsi]);
7221
7222         /* Setup static PF queue filter control settings */
7223         ret = i40e_setup_pf_filter_control(pf);
7224         if (ret) {
7225                 dev_info(&pf->pdev->dev, "setup_pf_filter_control failed: %d\n",
7226                          ret);
7227                 /* Failure here should not stop continuing other steps */
7228         }
7229
7230         /* enable RSS in the HW, even for only one queue, as the stack can use
7231          * the hash
7232          */
7233         if ((pf->flags & I40E_FLAG_RSS_ENABLED))
7234                 i40e_config_rss(pf);
7235
7236         /* fill in link information and enable LSE reporting */
7237         i40e_aq_get_link_info(&pf->hw, true, NULL, NULL);
7238         i40e_link_event(pf);
7239
7240         /* Initialize user-specific link properties */
7241         pf->fc_autoneg_status = ((pf->hw.phy.link_info.an_info &
7242                                   I40E_AQ_AN_COMPLETED) ? true : false);
7243         /* requested_mode is set in probe or by ethtool */
7244         if (!pf->fc_autoneg_status)
7245                 goto no_autoneg;
7246
7247         if ((pf->hw.phy.link_info.an_info & I40E_AQ_LINK_PAUSE_TX) &&
7248             (pf->hw.phy.link_info.an_info & I40E_AQ_LINK_PAUSE_RX))
7249                 pf->hw.fc.current_mode = I40E_FC_FULL;
7250         else if (pf->hw.phy.link_info.an_info & I40E_AQ_LINK_PAUSE_TX)
7251                 pf->hw.fc.current_mode = I40E_FC_TX_PAUSE;
7252         else if (pf->hw.phy.link_info.an_info & I40E_AQ_LINK_PAUSE_RX)
7253                 pf->hw.fc.current_mode = I40E_FC_RX_PAUSE;
7254         else
7255                 pf->hw.fc.current_mode = I40E_FC_NONE;
7256
7257         /* sync the flow control settings with the auto-neg values */
7258         switch (pf->hw.fc.current_mode) {
7259         case I40E_FC_FULL:
7260                 txfc = 1;
7261                 rxfc = 1;
7262                 break;
7263         case I40E_FC_TX_PAUSE:
7264                 txfc = 1;
7265                 rxfc = 0;
7266                 break;
7267         case I40E_FC_RX_PAUSE:
7268                 txfc = 0;
7269                 rxfc = 1;
7270                 break;
7271         case I40E_FC_NONE:
7272         case I40E_FC_DEFAULT:
7273                 txfc = 0;
7274                 rxfc = 0;
7275                 break;
7276         case I40E_FC_PFC:
7277                 /* TBD */
7278                 break;
7279         /* no default case, we have to handle all possibilities here */
7280         }
7281
7282         wr32(&pf->hw, I40E_PRTDCB_FCCFG, txfc << I40E_PRTDCB_FCCFG_TFCE_SHIFT);
7283
7284         rxfc_reg = rd32(&pf->hw, I40E_PRTDCB_MFLCN) &
7285                    ~I40E_PRTDCB_MFLCN_RFCE_MASK;
7286         rxfc_reg |= (rxfc << I40E_PRTDCB_MFLCN_RFCE_SHIFT);
7287
7288         wr32(&pf->hw, I40E_PRTDCB_MFLCN, rxfc_reg);
7289
7290         goto fc_complete;
7291
7292 no_autoneg:
7293         /* disable L2 flow control, user can turn it on if they wish */
7294         wr32(&pf->hw, I40E_PRTDCB_FCCFG, 0);
7295         wr32(&pf->hw, I40E_PRTDCB_MFLCN, rd32(&pf->hw, I40E_PRTDCB_MFLCN) &
7296                                          ~I40E_PRTDCB_MFLCN_RFCE_MASK);
7297
7298 fc_complete:
7299         return ret;
7300 }
7301
7302 /**
7303  * i40e_set_rss_size - helper to set rss_size
7304  * @pf: board private structure
7305  * @queues_left: how many queues
7306  */
7307 static u16 i40e_set_rss_size(struct i40e_pf *pf, int queues_left)
7308 {
7309         int num_tc0;
7310
7311         num_tc0 = min_t(int, queues_left, pf->rss_size_max);
7312         num_tc0 = min_t(int, num_tc0, num_online_cpus());
7313         num_tc0 = rounddown_pow_of_two(num_tc0);
7314
7315         return num_tc0;
7316 }
7317
7318 /**
7319  * i40e_determine_queue_usage - Work out queue distribution
7320  * @pf: board private structure
7321  **/
7322 static void i40e_determine_queue_usage(struct i40e_pf *pf)
7323 {
7324         int accum_tc_size;
7325         int queues_left;
7326
7327         pf->num_lan_qps = 0;
7328         pf->num_tc_qps = rounddown_pow_of_two(pf->num_tc_qps);
7329         accum_tc_size = (I40E_MAX_TRAFFIC_CLASS - 1) * pf->num_tc_qps;
7330
7331         /* Find the max queues to be put into basic use.  We'll always be
7332          * using TC0, whether or not DCB is running, and TC0 will get the
7333          * big RSS set.
7334          */
7335         queues_left = pf->hw.func_caps.num_tx_qp;
7336
7337         if   (!(pf->flags & I40E_FLAG_MSIX_ENABLED) ||
7338                 !(pf->flags & (I40E_FLAG_RSS_ENABLED |
7339                 I40E_FLAG_FDIR_ENABLED | I40E_FLAG_DCB_ENABLED)) ||
7340                 (queues_left == 1)) {
7341
7342                 /* one qp for PF, no queues for anything else */
7343                 queues_left = 0;
7344                 pf->rss_size = pf->num_lan_qps = 1;
7345
7346                 /* make sure all the fancies are disabled */
7347                 pf->flags &= ~(I40E_FLAG_RSS_ENABLED       |
7348                                 I40E_FLAG_FDIR_ENABLED     |
7349                                 I40E_FLAG_FDIR_ATR_ENABLED |
7350                                 I40E_FLAG_DCB_ENABLED      |
7351                                 I40E_FLAG_SRIOV_ENABLED    |
7352                                 I40E_FLAG_VMDQ_ENABLED);
7353
7354         } else if (pf->flags & I40E_FLAG_RSS_ENABLED      &&
7355                    !(pf->flags & I40E_FLAG_FDIR_ENABLED)  &&
7356                    !(pf->flags & I40E_FLAG_DCB_ENABLED)) {
7357
7358                 pf->rss_size = i40e_set_rss_size(pf, queues_left);
7359
7360                 queues_left -= pf->rss_size;
7361                 pf->num_lan_qps = pf->rss_size_max;
7362
7363         } else if (pf->flags & I40E_FLAG_RSS_ENABLED      &&
7364                    !(pf->flags & I40E_FLAG_FDIR_ENABLED)  &&
7365                    (pf->flags & I40E_FLAG_DCB_ENABLED)) {
7366
7367                 /* save num_tc_qps queues for TCs 1 thru 7 and the rest
7368                  * are set up for RSS in TC0
7369                  */
7370                 queues_left -= accum_tc_size;
7371
7372                 pf->rss_size = i40e_set_rss_size(pf, queues_left);
7373
7374                 queues_left -= pf->rss_size;
7375                 if (queues_left < 0) {
7376                         dev_info(&pf->pdev->dev, "not enough queues for DCB\n");
7377                         return;
7378                 }
7379
7380                 pf->num_lan_qps = pf->rss_size_max + accum_tc_size;
7381
7382         } else if (pf->flags & I40E_FLAG_RSS_ENABLED   &&
7383                   (pf->flags & I40E_FLAG_FDIR_ENABLED) &&
7384                   !(pf->flags & I40E_FLAG_DCB_ENABLED)) {
7385
7386                 queues_left -= 1; /* save 1 queue for FD */
7387
7388                 pf->rss_size = i40e_set_rss_size(pf, queues_left);
7389
7390                 queues_left -= pf->rss_size;
7391                 if (queues_left < 0) {
7392                         dev_info(&pf->pdev->dev, "not enough queues for Flow Director\n");
7393                         return;
7394                 }
7395
7396                 pf->num_lan_qps = pf->rss_size_max;
7397
7398         } else if (pf->flags & I40E_FLAG_RSS_ENABLED   &&
7399                   (pf->flags & I40E_FLAG_FDIR_ENABLED) &&
7400                   (pf->flags & I40E_FLAG_DCB_ENABLED)) {
7401
7402                 /* save 1 queue for TCs 1 thru 7,
7403                  * 1 queue for flow director,
7404                  * and the rest are set up for RSS in TC0
7405                  */
7406                 queues_left -= 1;
7407                 queues_left -= accum_tc_size;
7408
7409                 pf->rss_size = i40e_set_rss_size(pf, queues_left);
7410                 queues_left -= pf->rss_size;
7411                 if (queues_left < 0) {
7412                         dev_info(&pf->pdev->dev, "not enough queues for DCB and Flow Director\n");
7413                         return;
7414                 }
7415
7416                 pf->num_lan_qps = pf->rss_size_max + accum_tc_size;
7417
7418         } else {
7419                 dev_info(&pf->pdev->dev,
7420                          "Invalid configuration, flags=0x%08llx\n", pf->flags);
7421                 return;
7422         }
7423
7424         if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
7425             pf->num_vf_qps && pf->num_req_vfs && queues_left) {
7426                 pf->num_req_vfs = min_t(int, pf->num_req_vfs, (queues_left /
7427                                                                pf->num_vf_qps));
7428                 queues_left -= (pf->num_req_vfs * pf->num_vf_qps);
7429         }
7430
7431         if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) &&
7432             pf->num_vmdq_vsis && pf->num_vmdq_qps && queues_left) {
7433                 pf->num_vmdq_vsis = min_t(int, pf->num_vmdq_vsis,
7434                                           (queues_left / pf->num_vmdq_qps));
7435                 queues_left -= (pf->num_vmdq_vsis * pf->num_vmdq_qps);
7436         }
7437
7438         pf->queues_left = queues_left;
7439         return;
7440 }
7441
7442 /**
7443  * i40e_setup_pf_filter_control - Setup PF static filter control
7444  * @pf: PF to be setup
7445  *
7446  * i40e_setup_pf_filter_control sets up a pf's initial filter control
7447  * settings. If PE/FCoE are enabled then it will also set the per PF
7448  * based filter sizes required for them. It also enables Flow director,
7449  * ethertype and macvlan type filter settings for the pf.
7450  *
7451  * Returns 0 on success, negative on failure
7452  **/
7453 static int i40e_setup_pf_filter_control(struct i40e_pf *pf)
7454 {
7455         struct i40e_filter_control_settings *settings = &pf->filter_settings;
7456
7457         settings->hash_lut_size = I40E_HASH_LUT_SIZE_128;
7458
7459         /* Flow Director is enabled */
7460         if (pf->flags & (I40E_FLAG_FDIR_ENABLED | I40E_FLAG_FDIR_ATR_ENABLED))
7461                 settings->enable_fdir = true;
7462
7463         /* Ethtype and MACVLAN filters enabled for PF */
7464         settings->enable_ethtype = true;
7465         settings->enable_macvlan = true;
7466
7467         if (i40e_set_filter_control(&pf->hw, settings))
7468                 return -ENOENT;
7469
7470         return 0;
7471 }
7472
7473 /**
7474  * i40e_probe - Device initialization routine
7475  * @pdev: PCI device information struct
7476  * @ent: entry in i40e_pci_tbl
7477  *
7478  * i40e_probe initializes a pf identified by a pci_dev structure.
7479  * The OS initialization, configuring of the pf private structure,
7480  * and a hardware reset occur.
7481  *
7482  * Returns 0 on success, negative on failure
7483  **/
7484 static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
7485 {
7486         struct i40e_driver_version dv;
7487         struct i40e_pf *pf;
7488         struct i40e_hw *hw;
7489         static u16 pfs_found;
7490         u16 link_status;
7491         int err = 0;
7492         u32 len;
7493
7494         err = pci_enable_device_mem(pdev);
7495         if (err)
7496                 return err;
7497
7498         /* set up for high or low dma */
7499         if (!dma_set_mask(&pdev->dev, DMA_BIT_MASK(64))) {
7500                 /* coherent mask for the same size will always succeed if
7501                  * dma_set_mask does
7502                  */
7503                 dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
7504         } else if (!dma_set_mask(&pdev->dev, DMA_BIT_MASK(32))) {
7505                 dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
7506         } else {
7507                 dev_err(&pdev->dev, "DMA configuration failed: %d\n", err);
7508                 err = -EIO;
7509                 goto err_dma;
7510         }
7511
7512         /* set up pci connections */
7513         err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
7514                                            IORESOURCE_MEM), i40e_driver_name);
7515         if (err) {
7516                 dev_info(&pdev->dev,
7517                          "pci_request_selected_regions failed %d\n", err);
7518                 goto err_pci_reg;
7519         }
7520
7521         pci_enable_pcie_error_reporting(pdev);
7522         pci_set_master(pdev);
7523
7524         /* Now that we have a PCI connection, we need to do the
7525          * low level device setup.  This is primarily setting up
7526          * the Admin Queue structures and then querying for the
7527          * device's current profile information.
7528          */
7529         pf = kzalloc(sizeof(*pf), GFP_KERNEL);
7530         if (!pf) {
7531                 err = -ENOMEM;
7532                 goto err_pf_alloc;
7533         }
7534         pf->next_vsi = 0;
7535         pf->pdev = pdev;
7536         set_bit(__I40E_DOWN, &pf->state);
7537
7538         hw = &pf->hw;
7539         hw->back = pf;
7540         hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
7541                               pci_resource_len(pdev, 0));
7542         if (!hw->hw_addr) {
7543                 err = -EIO;
7544                 dev_info(&pdev->dev, "ioremap(0x%04x, 0x%04x) failed: 0x%x\n",
7545                          (unsigned int)pci_resource_start(pdev, 0),
7546                          (unsigned int)pci_resource_len(pdev, 0), err);
7547                 goto err_ioremap;
7548         }
7549         hw->vendor_id = pdev->vendor;
7550         hw->device_id = pdev->device;
7551         pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id);
7552         hw->subsystem_vendor_id = pdev->subsystem_vendor;
7553         hw->subsystem_device_id = pdev->subsystem_device;
7554         hw->bus.device = PCI_SLOT(pdev->devfn);
7555         hw->bus.func = PCI_FUNC(pdev->devfn);
7556         pf->instance = pfs_found;
7557
7558         /* do a special CORER for clearing PXE mode once at init */
7559         if (hw->revision_id == 0 &&
7560             (rd32(hw, I40E_GLLAN_RCTL_0) & I40E_GLLAN_RCTL_0_PXE_MODE_MASK)) {
7561                 wr32(hw, I40E_GLGEN_RTRIG, I40E_GLGEN_RTRIG_CORER_MASK);
7562                 i40e_flush(hw);
7563                 msleep(200);
7564                 pf->corer_count++;
7565
7566                 i40e_clear_pxe_mode(hw);
7567         }
7568
7569         /* Reset here to make sure all is clean and to define PF 'n' */
7570         err = i40e_pf_reset(hw);
7571         if (err) {
7572                 dev_info(&pdev->dev, "Initial pf_reset failed: %d\n", err);
7573                 goto err_pf_reset;
7574         }
7575         pf->pfr_count++;
7576
7577         hw->aq.num_arq_entries = I40E_AQ_LEN;
7578         hw->aq.num_asq_entries = I40E_AQ_LEN;
7579         hw->aq.arq_buf_size = I40E_MAX_AQ_BUF_SIZE;
7580         hw->aq.asq_buf_size = I40E_MAX_AQ_BUF_SIZE;
7581         pf->adminq_work_limit = I40E_AQ_WORK_LIMIT;
7582         snprintf(pf->misc_int_name, sizeof(pf->misc_int_name) - 1,
7583                  "%s-pf%d:misc",
7584                  dev_driver_string(&pf->pdev->dev), pf->hw.pf_id);
7585
7586         err = i40e_init_shared_code(hw);
7587         if (err) {
7588                 dev_info(&pdev->dev, "init_shared_code failed: %d\n", err);
7589                 goto err_pf_reset;
7590         }
7591
7592         /* set up a default setting for link flow control */
7593         pf->hw.fc.requested_mode = I40E_FC_NONE;
7594
7595         err = i40e_init_adminq(hw);
7596         dev_info(&pdev->dev, "%s\n", i40e_fw_version_str(hw));
7597         if (((hw->nvm.version & I40E_NVM_VERSION_HI_MASK)
7598                  >> I40E_NVM_VERSION_HI_SHIFT) != I40E_CURRENT_NVM_VERSION_HI) {
7599                 dev_info(&pdev->dev,
7600                          "warning: NVM version not supported, supported version: %02x.%02x\n",
7601                          I40E_CURRENT_NVM_VERSION_HI,
7602                          I40E_CURRENT_NVM_VERSION_LO);
7603         }
7604         if (err) {
7605                 dev_info(&pdev->dev,
7606                          "init_adminq failed: %d expecting API %02x.%02x\n",
7607                          err,
7608                          I40E_FW_API_VERSION_MAJOR, I40E_FW_API_VERSION_MINOR);
7609                 goto err_pf_reset;
7610         }
7611
7612         err = i40e_get_capabilities(pf);
7613         if (err)
7614                 goto err_adminq_setup;
7615
7616         err = i40e_sw_init(pf);
7617         if (err) {
7618                 dev_info(&pdev->dev, "sw_init failed: %d\n", err);
7619                 goto err_sw_init;
7620         }
7621
7622         err = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
7623                                 hw->func_caps.num_rx_qp,
7624                                 pf->fcoe_hmc_cntx_num, pf->fcoe_hmc_filt_num);
7625         if (err) {
7626                 dev_info(&pdev->dev, "init_lan_hmc failed: %d\n", err);
7627                 goto err_init_lan_hmc;
7628         }
7629
7630         err = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
7631         if (err) {
7632                 dev_info(&pdev->dev, "configure_lan_hmc failed: %d\n", err);
7633                 err = -ENOENT;
7634                 goto err_configure_lan_hmc;
7635         }
7636
7637         i40e_get_mac_addr(hw, hw->mac.addr);
7638         if (!is_valid_ether_addr(hw->mac.addr)) {
7639                 dev_info(&pdev->dev, "invalid MAC address %pM\n", hw->mac.addr);
7640                 err = -EIO;
7641                 goto err_mac_addr;
7642         }
7643         dev_info(&pdev->dev, "MAC address: %pM\n", hw->mac.addr);
7644         memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
7645
7646         pci_set_drvdata(pdev, pf);
7647         pci_save_state(pdev);
7648
7649         /* set up periodic task facility */
7650         setup_timer(&pf->service_timer, i40e_service_timer, (unsigned long)pf);
7651         pf->service_timer_period = HZ;
7652
7653         INIT_WORK(&pf->service_task, i40e_service_task);
7654         clear_bit(__I40E_SERVICE_SCHED, &pf->state);
7655         pf->flags |= I40E_FLAG_NEED_LINK_UPDATE;
7656         pf->link_check_timeout = jiffies;
7657
7658         /* WoL defaults to disabled */
7659         pf->wol_en = false;
7660         device_set_wakeup_enable(&pf->pdev->dev, pf->wol_en);
7661
7662         /* set up the main switch operations */
7663         i40e_determine_queue_usage(pf);
7664         i40e_init_interrupt_scheme(pf);
7665
7666         /* Set up the *vsi struct based on the number of VSIs in the HW,
7667          * and set up our local tracking of the MAIN PF vsi.
7668          */
7669         len = sizeof(struct i40e_vsi *) * pf->hw.func_caps.num_vsis;
7670         pf->vsi = kzalloc(len, GFP_KERNEL);
7671         if (!pf->vsi) {
7672                 err = -ENOMEM;
7673                 goto err_switch_setup;
7674         }
7675
7676         err = i40e_setup_pf_switch(pf, false);
7677         if (err) {
7678                 dev_info(&pdev->dev, "setup_pf_switch failed: %d\n", err);
7679                 goto err_vsis;
7680         }
7681
7682         /* The main driver is (mostly) up and happy. We need to set this state
7683          * before setting up the misc vector or we get a race and the vector
7684          * ends up disabled forever.
7685          */
7686         clear_bit(__I40E_DOWN, &pf->state);
7687
7688         /* In case of MSIX we are going to setup the misc vector right here
7689          * to handle admin queue events etc. In case of legacy and MSI
7690          * the misc functionality and queue processing is combined in
7691          * the same vector and that gets setup at open.
7692          */
7693         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
7694                 err = i40e_setup_misc_vector(pf);
7695                 if (err) {
7696                         dev_info(&pdev->dev,
7697                                  "setup of misc vector failed: %d\n", err);
7698                         goto err_vsis;
7699                 }
7700         }
7701
7702         /* prep for VF support */
7703         if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
7704             (pf->flags & I40E_FLAG_MSIX_ENABLED)) {
7705                 u32 val;
7706
7707                 /* disable link interrupts for VFs */
7708                 val = rd32(hw, I40E_PFGEN_PORTMDIO_NUM);
7709                 val &= ~I40E_PFGEN_PORTMDIO_NUM_VFLINK_STAT_ENA_MASK;
7710                 wr32(hw, I40E_PFGEN_PORTMDIO_NUM, val);
7711                 i40e_flush(hw);
7712         }
7713
7714         pfs_found++;
7715
7716         i40e_dbg_pf_init(pf);
7717
7718         /* tell the firmware that we're starting */
7719         dv.major_version = DRV_VERSION_MAJOR;
7720         dv.minor_version = DRV_VERSION_MINOR;
7721         dv.build_version = DRV_VERSION_BUILD;
7722         dv.subbuild_version = 0;
7723         i40e_aq_send_driver_version(&pf->hw, &dv, NULL);
7724
7725         /* since everything's happy, start the service_task timer */
7726         mod_timer(&pf->service_timer,
7727                   round_jiffies(jiffies + pf->service_timer_period));
7728
7729         /* Get the negotiated link width and speed from PCI config space */
7730         pcie_capability_read_word(pf->pdev, PCI_EXP_LNKSTA, &link_status);
7731
7732         i40e_set_pci_config_data(hw, link_status);
7733
7734         dev_info(&pdev->dev, "PCI Express: %s %s\n",
7735                 (hw->bus.speed == i40e_bus_speed_8000 ? "Speed 8.0GT/s" :
7736                  hw->bus.speed == i40e_bus_speed_5000 ? "Speed 5.0GT/s" :
7737                  hw->bus.speed == i40e_bus_speed_2500 ? "Speed 2.5GT/s" :
7738                  "Unknown"),
7739                 (hw->bus.width == i40e_bus_width_pcie_x8 ? "Width x8" :
7740                  hw->bus.width == i40e_bus_width_pcie_x4 ? "Width x4" :
7741                  hw->bus.width == i40e_bus_width_pcie_x2 ? "Width x2" :
7742                  hw->bus.width == i40e_bus_width_pcie_x1 ? "Width x1" :
7743                  "Unknown"));
7744
7745         if (hw->bus.width < i40e_bus_width_pcie_x8 ||
7746             hw->bus.speed < i40e_bus_speed_8000) {
7747                 dev_warn(&pdev->dev, "PCI-Express bandwidth available for this device may be insufficient for optimal performance.\n");
7748                 dev_warn(&pdev->dev, "Please move the device to a different PCI-e link with more lanes and/or higher transfer rate.\n");
7749         }
7750
7751         return 0;
7752
7753         /* Unwind what we've done if something failed in the setup */
7754 err_vsis:
7755         set_bit(__I40E_DOWN, &pf->state);
7756         i40e_clear_interrupt_scheme(pf);
7757         kfree(pf->vsi);
7758 err_switch_setup:
7759         i40e_reset_interrupt_capability(pf);
7760         del_timer_sync(&pf->service_timer);
7761 err_mac_addr:
7762 err_configure_lan_hmc:
7763         (void)i40e_shutdown_lan_hmc(hw);
7764 err_init_lan_hmc:
7765         kfree(pf->qp_pile);
7766         kfree(pf->irq_pile);
7767 err_sw_init:
7768 err_adminq_setup:
7769         (void)i40e_shutdown_adminq(hw);
7770 err_pf_reset:
7771         iounmap(hw->hw_addr);
7772 err_ioremap:
7773         kfree(pf);
7774 err_pf_alloc:
7775         pci_disable_pcie_error_reporting(pdev);
7776         pci_release_selected_regions(pdev,
7777                                      pci_select_bars(pdev, IORESOURCE_MEM));
7778 err_pci_reg:
7779 err_dma:
7780         pci_disable_device(pdev);
7781         return err;
7782 }
7783
7784 /**
7785  * i40e_remove - Device removal routine
7786  * @pdev: PCI device information struct
7787  *
7788  * i40e_remove is called by the PCI subsystem to alert the driver
7789  * that is should release a PCI device.  This could be caused by a
7790  * Hot-Plug event, or because the driver is going to be removed from
7791  * memory.
7792  **/
7793 static void i40e_remove(struct pci_dev *pdev)
7794 {
7795         struct i40e_pf *pf = pci_get_drvdata(pdev);
7796         i40e_status ret_code;
7797         u32 reg;
7798         int i;
7799
7800         i40e_dbg_pf_exit(pf);
7801
7802         if (pf->flags & I40E_FLAG_SRIOV_ENABLED) {
7803                 i40e_free_vfs(pf);
7804                 pf->flags &= ~I40E_FLAG_SRIOV_ENABLED;
7805         }
7806
7807         /* no more scheduling of any task */
7808         set_bit(__I40E_DOWN, &pf->state);
7809         del_timer_sync(&pf->service_timer);
7810         cancel_work_sync(&pf->service_task);
7811
7812         i40e_fdir_teardown(pf);
7813
7814         /* If there is a switch structure or any orphans, remove them.
7815          * This will leave only the PF's VSI remaining.
7816          */
7817         for (i = 0; i < I40E_MAX_VEB; i++) {
7818                 if (!pf->veb[i])
7819                         continue;
7820
7821                 if (pf->veb[i]->uplink_seid == pf->mac_seid ||
7822                     pf->veb[i]->uplink_seid == 0)
7823                         i40e_switch_branch_release(pf->veb[i]);
7824         }
7825
7826         /* Now we can shutdown the PF's VSI, just before we kill
7827          * adminq and hmc.
7828          */
7829         if (pf->vsi[pf->lan_vsi])
7830                 i40e_vsi_release(pf->vsi[pf->lan_vsi]);
7831
7832         i40e_stop_misc_vector(pf);
7833         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
7834                 synchronize_irq(pf->msix_entries[0].vector);
7835                 free_irq(pf->msix_entries[0].vector, pf);
7836         }
7837
7838         /* shutdown and destroy the HMC */
7839         ret_code = i40e_shutdown_lan_hmc(&pf->hw);
7840         if (ret_code)
7841                 dev_warn(&pdev->dev,
7842                          "Failed to destroy the HMC resources: %d\n", ret_code);
7843
7844         /* shutdown the adminq */
7845         ret_code = i40e_shutdown_adminq(&pf->hw);
7846         if (ret_code)
7847                 dev_warn(&pdev->dev,
7848                          "Failed to destroy the Admin Queue resources: %d\n",
7849                          ret_code);
7850
7851         /* Clear all dynamic memory lists of rings, q_vectors, and VSIs */
7852         i40e_clear_interrupt_scheme(pf);
7853         for (i = 0; i < pf->hw.func_caps.num_vsis; i++) {
7854                 if (pf->vsi[i]) {
7855                         i40e_vsi_clear_rings(pf->vsi[i]);
7856                         i40e_vsi_clear(pf->vsi[i]);
7857                         pf->vsi[i] = NULL;
7858                 }
7859         }
7860
7861         for (i = 0; i < I40E_MAX_VEB; i++) {
7862                 kfree(pf->veb[i]);
7863                 pf->veb[i] = NULL;
7864         }
7865
7866         kfree(pf->qp_pile);
7867         kfree(pf->irq_pile);
7868         kfree(pf->sw_config);
7869         kfree(pf->vsi);
7870
7871         /* force a PF reset to clean anything leftover */
7872         reg = rd32(&pf->hw, I40E_PFGEN_CTRL);
7873         wr32(&pf->hw, I40E_PFGEN_CTRL, (reg | I40E_PFGEN_CTRL_PFSWR_MASK));
7874         i40e_flush(&pf->hw);
7875
7876         iounmap(pf->hw.hw_addr);
7877         kfree(pf);
7878         pci_release_selected_regions(pdev,
7879                                      pci_select_bars(pdev, IORESOURCE_MEM));
7880
7881         pci_disable_pcie_error_reporting(pdev);
7882         pci_disable_device(pdev);
7883 }
7884
7885 /**
7886  * i40e_pci_error_detected - warning that something funky happened in PCI land
7887  * @pdev: PCI device information struct
7888  *
7889  * Called to warn that something happened and the error handling steps
7890  * are in progress.  Allows the driver to quiesce things, be ready for
7891  * remediation.
7892  **/
7893 static pci_ers_result_t i40e_pci_error_detected(struct pci_dev *pdev,
7894                                                 enum pci_channel_state error)
7895 {
7896         struct i40e_pf *pf = pci_get_drvdata(pdev);
7897
7898         dev_info(&pdev->dev, "%s: error %d\n", __func__, error);
7899
7900         /* shutdown all operations */
7901         if (!test_bit(__I40E_SUSPENDED, &pf->state)) {
7902                 rtnl_lock();
7903                 i40e_prep_for_reset(pf);
7904                 rtnl_unlock();
7905         }
7906
7907         /* Request a slot reset */
7908         return PCI_ERS_RESULT_NEED_RESET;
7909 }
7910
7911 /**
7912  * i40e_pci_error_slot_reset - a PCI slot reset just happened
7913  * @pdev: PCI device information struct
7914  *
7915  * Called to find if the driver can work with the device now that
7916  * the pci slot has been reset.  If a basic connection seems good
7917  * (registers are readable and have sane content) then return a
7918  * happy little PCI_ERS_RESULT_xxx.
7919  **/
7920 static pci_ers_result_t i40e_pci_error_slot_reset(struct pci_dev *pdev)
7921 {
7922         struct i40e_pf *pf = pci_get_drvdata(pdev);
7923         pci_ers_result_t result;
7924         int err;
7925         u32 reg;
7926
7927         dev_info(&pdev->dev, "%s\n", __func__);
7928         if (pci_enable_device_mem(pdev)) {
7929                 dev_info(&pdev->dev,
7930                          "Cannot re-enable PCI device after reset.\n");
7931                 result = PCI_ERS_RESULT_DISCONNECT;
7932         } else {
7933                 pci_set_master(pdev);
7934                 pci_restore_state(pdev);
7935                 pci_save_state(pdev);
7936                 pci_wake_from_d3(pdev, false);
7937
7938                 reg = rd32(&pf->hw, I40E_GLGEN_RTRIG);
7939                 if (reg == 0)
7940                         result = PCI_ERS_RESULT_RECOVERED;
7941                 else
7942                         result = PCI_ERS_RESULT_DISCONNECT;
7943         }
7944
7945         err = pci_cleanup_aer_uncorrect_error_status(pdev);
7946         if (err) {
7947                 dev_info(&pdev->dev,
7948                          "pci_cleanup_aer_uncorrect_error_status failed 0x%0x\n",
7949                          err);
7950                 /* non-fatal, continue */
7951         }
7952
7953         return result;
7954 }
7955
7956 /**
7957  * i40e_pci_error_resume - restart operations after PCI error recovery
7958  * @pdev: PCI device information struct
7959  *
7960  * Called to allow the driver to bring things back up after PCI error
7961  * and/or reset recovery has finished.
7962  **/
7963 static void i40e_pci_error_resume(struct pci_dev *pdev)
7964 {
7965         struct i40e_pf *pf = pci_get_drvdata(pdev);
7966
7967         dev_info(&pdev->dev, "%s\n", __func__);
7968         if (test_bit(__I40E_SUSPENDED, &pf->state))
7969                 return;
7970
7971         rtnl_lock();
7972         i40e_handle_reset_warning(pf);
7973         rtnl_lock();
7974 }
7975
7976 /**
7977  * i40e_shutdown - PCI callback for shutting down
7978  * @pdev: PCI device information struct
7979  **/
7980 static void i40e_shutdown(struct pci_dev *pdev)
7981 {
7982         struct i40e_pf *pf = pci_get_drvdata(pdev);
7983         struct i40e_hw *hw = &pf->hw;
7984
7985         set_bit(__I40E_SUSPENDED, &pf->state);
7986         set_bit(__I40E_DOWN, &pf->state);
7987         rtnl_lock();
7988         i40e_prep_for_reset(pf);
7989         rtnl_unlock();
7990
7991         wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
7992         wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
7993
7994         if (system_state == SYSTEM_POWER_OFF) {
7995                 pci_wake_from_d3(pdev, pf->wol_en);
7996                 pci_set_power_state(pdev, PCI_D3hot);
7997         }
7998 }
7999
8000 #ifdef CONFIG_PM
8001 /**
8002  * i40e_suspend - PCI callback for moving to D3
8003  * @pdev: PCI device information struct
8004  **/
8005 static int i40e_suspend(struct pci_dev *pdev, pm_message_t state)
8006 {
8007         struct i40e_pf *pf = pci_get_drvdata(pdev);
8008         struct i40e_hw *hw = &pf->hw;
8009
8010         set_bit(__I40E_SUSPENDED, &pf->state);
8011         set_bit(__I40E_DOWN, &pf->state);
8012         rtnl_lock();
8013         i40e_prep_for_reset(pf);
8014         rtnl_unlock();
8015
8016         wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
8017         wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
8018
8019         pci_wake_from_d3(pdev, pf->wol_en);
8020         pci_set_power_state(pdev, PCI_D3hot);
8021
8022         return 0;
8023 }
8024
8025 /**
8026  * i40e_resume - PCI callback for waking up from D3
8027  * @pdev: PCI device information struct
8028  **/
8029 static int i40e_resume(struct pci_dev *pdev)
8030 {
8031         struct i40e_pf *pf = pci_get_drvdata(pdev);
8032         u32 err;
8033
8034         pci_set_power_state(pdev, PCI_D0);
8035         pci_restore_state(pdev);
8036         /* pci_restore_state() clears dev->state_saves, so
8037          * call pci_save_state() again to restore it.
8038          */
8039         pci_save_state(pdev);
8040
8041         err = pci_enable_device_mem(pdev);
8042         if (err) {
8043                 dev_err(&pdev->dev,
8044                         "%s: Cannot enable PCI device from suspend\n",
8045                         __func__);
8046                 return err;
8047         }
8048         pci_set_master(pdev);
8049
8050         /* no wakeup events while running */
8051         pci_wake_from_d3(pdev, false);
8052
8053         /* handling the reset will rebuild the device state */
8054         if (test_and_clear_bit(__I40E_SUSPENDED, &pf->state)) {
8055                 clear_bit(__I40E_DOWN, &pf->state);
8056                 rtnl_lock();
8057                 i40e_reset_and_rebuild(pf, false);
8058                 rtnl_unlock();
8059         }
8060
8061         return 0;
8062 }
8063
8064 #endif
8065 static const struct pci_error_handlers i40e_err_handler = {
8066         .error_detected = i40e_pci_error_detected,
8067         .slot_reset = i40e_pci_error_slot_reset,
8068         .resume = i40e_pci_error_resume,
8069 };
8070
8071 static struct pci_driver i40e_driver = {
8072         .name     = i40e_driver_name,
8073         .id_table = i40e_pci_tbl,
8074         .probe    = i40e_probe,
8075         .remove   = i40e_remove,
8076 #ifdef CONFIG_PM
8077         .suspend  = i40e_suspend,
8078         .resume   = i40e_resume,
8079 #endif
8080         .shutdown = i40e_shutdown,
8081         .err_handler = &i40e_err_handler,
8082         .sriov_configure = i40e_pci_sriov_configure,
8083 };
8084
8085 /**
8086  * i40e_init_module - Driver registration routine
8087  *
8088  * i40e_init_module is the first routine called when the driver is
8089  * loaded. All it does is register with the PCI subsystem.
8090  **/
8091 static int __init i40e_init_module(void)
8092 {
8093         pr_info("%s: %s - version %s\n", i40e_driver_name,
8094                 i40e_driver_string, i40e_driver_version_str);
8095         pr_info("%s: %s\n", i40e_driver_name, i40e_copyright);
8096         i40e_dbg_init();
8097         return pci_register_driver(&i40e_driver);
8098 }
8099 module_init(i40e_init_module);
8100
8101 /**
8102  * i40e_exit_module - Driver exit cleanup routine
8103  *
8104  * i40e_exit_module is called just before the driver is removed
8105  * from memory.
8106  **/
8107 static void __exit i40e_exit_module(void)
8108 {
8109         pci_unregister_driver(&i40e_driver);
8110         i40e_dbg_exit();
8111 }
8112 module_exit(i40e_exit_module);