Pileus Git - ~andy/linux/blob - drivers/net/e1000/e1000_main.c

   1 /*******************************************************************************
   2
   3
   4   Copyright(c) 1999 - 2005 Intel Corporation. All rights reserved.
   5
   6   This program is free software; you can redistribute it and/or modify it
   7   under the terms of the GNU General Public License as published by the Free
   8   Software Foundation; either version 2 of the License, or (at your option)
   9   any later version.
  10
  11   This program is distributed in the hope that it will be useful, but WITHOUT
  12   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13   FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  14   more details.
  15
  16   You should have received a copy of the GNU General Public License along with
  17   this program; if not, write to the Free Software Foundation, Inc., 59
  18   Temple Place - Suite 330, Boston, MA  02111-1307, USA.
  19
  20   The full GNU General Public License is included in this distribution in the
  21   file called LICENSE.
  22
  23   Contact Information:
  24   Linux NICS <linux.nics@intel.com>
  25   Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  26
  27 *******************************************************************************/
  28
  29 #include "e1000.h"
  30
  31 /* Change Log
  32  * 6.0.58       4/20/05
  33  *   o Accepted ethtool cleanup patch from Stephen Hemminger
  34  * 6.0.44+      2/15/05
  35  *   o applied Anton's patch to resolve tx hang in hardware
  36  *   o Applied Andrew Mortons patch - e1000 stops working after resume
  37  */
  38
  39 char e1000_driver_name[] = "e1000";
  40 static char e1000_driver_string[] = "Intel(R) PRO/1000 Network Driver";
  41 #ifndef CONFIG_E1000_NAPI
  42 #define DRIVERNAPI
  43 #else
  44 #define DRIVERNAPI "-NAPI"
  45 #endif
  46 #define DRV_VERSION "6.3.9-k2"DRIVERNAPI
  47 char e1000_driver_version[] = DRV_VERSION;
  48 static char e1000_copyright[] = "Copyright (c) 1999-2005 Intel Corporation.";
  49
  50 /* e1000_pci_tbl - PCI Device ID Table
  51  *
  52  * Last entry must be all 0s
  53  *
  54  * Macro expands to...
  55  *   {PCI_DEVICE(PCI_VENDOR_ID_INTEL, device_id)}
  56  */
  57 static struct pci_device_id e1000_pci_tbl[] = {
  58         INTEL_E1000_ETHERNET_DEVICE(0x1000),
  59         INTEL_E1000_ETHERNET_DEVICE(0x1001),
  60         INTEL_E1000_ETHERNET_DEVICE(0x1004),
  61         INTEL_E1000_ETHERNET_DEVICE(0x1008),
  62         INTEL_E1000_ETHERNET_DEVICE(0x1009),
  63         INTEL_E1000_ETHERNET_DEVICE(0x100C),
  64         INTEL_E1000_ETHERNET_DEVICE(0x100D),
  65         INTEL_E1000_ETHERNET_DEVICE(0x100E),
  66         INTEL_E1000_ETHERNET_DEVICE(0x100F),
  67         INTEL_E1000_ETHERNET_DEVICE(0x1010),
  68         INTEL_E1000_ETHERNET_DEVICE(0x1011),
  69         INTEL_E1000_ETHERNET_DEVICE(0x1012),
  70         INTEL_E1000_ETHERNET_DEVICE(0x1013),
  71         INTEL_E1000_ETHERNET_DEVICE(0x1014),
  72         INTEL_E1000_ETHERNET_DEVICE(0x1015),
  73         INTEL_E1000_ETHERNET_DEVICE(0x1016),
  74         INTEL_E1000_ETHERNET_DEVICE(0x1017),
  75         INTEL_E1000_ETHERNET_DEVICE(0x1018),
  76         INTEL_E1000_ETHERNET_DEVICE(0x1019),
  77         INTEL_E1000_ETHERNET_DEVICE(0x101A),
  78         INTEL_E1000_ETHERNET_DEVICE(0x101D),
  79         INTEL_E1000_ETHERNET_DEVICE(0x101E),
  80         INTEL_E1000_ETHERNET_DEVICE(0x1026),
  81         INTEL_E1000_ETHERNET_DEVICE(0x1027),
  82         INTEL_E1000_ETHERNET_DEVICE(0x1028),
  83         INTEL_E1000_ETHERNET_DEVICE(0x105E),
  84         INTEL_E1000_ETHERNET_DEVICE(0x105F),
  85         INTEL_E1000_ETHERNET_DEVICE(0x1060),
  86         INTEL_E1000_ETHERNET_DEVICE(0x1075),
  87         INTEL_E1000_ETHERNET_DEVICE(0x1076),
  88         INTEL_E1000_ETHERNET_DEVICE(0x1077),
  89         INTEL_E1000_ETHERNET_DEVICE(0x1078),
  90         INTEL_E1000_ETHERNET_DEVICE(0x1079),
  91         INTEL_E1000_ETHERNET_DEVICE(0x107A),
  92         INTEL_E1000_ETHERNET_DEVICE(0x107B),
  93         INTEL_E1000_ETHERNET_DEVICE(0x107C),
  94         INTEL_E1000_ETHERNET_DEVICE(0x107D),
  95         INTEL_E1000_ETHERNET_DEVICE(0x107E),
  96         INTEL_E1000_ETHERNET_DEVICE(0x107F),
  97         INTEL_E1000_ETHERNET_DEVICE(0x108A),
  98         INTEL_E1000_ETHERNET_DEVICE(0x108B),
  99         INTEL_E1000_ETHERNET_DEVICE(0x108C),
 100         INTEL_E1000_ETHERNET_DEVICE(0x109A),
 101         /* required last entry */
 102         {0,}
 103 };
 104
 105 MODULE_DEVICE_TABLE(pci, e1000_pci_tbl);
 106
 107 int e1000_up(struct e1000_adapter *adapter);
 108 void e1000_down(struct e1000_adapter *adapter);
 109 void e1000_reset(struct e1000_adapter *adapter);
 110 int e1000_set_spd_dplx(struct e1000_adapter *adapter, uint16_t spddplx);
 111 int e1000_setup_all_tx_resources(struct e1000_adapter *adapter);
 112 int e1000_setup_all_rx_resources(struct e1000_adapter *adapter);
 113 void e1000_free_all_tx_resources(struct e1000_adapter *adapter);
 114 void e1000_free_all_rx_resources(struct e1000_adapter *adapter);
 115 static int e1000_setup_tx_resources(struct e1000_adapter *adapter,
 116                                     struct e1000_tx_ring *txdr);
 117 static int e1000_setup_rx_resources(struct e1000_adapter *adapter,
 118                                     struct e1000_rx_ring *rxdr);
 119 static void e1000_free_tx_resources(struct e1000_adapter *adapter,
 120                                     struct e1000_tx_ring *tx_ring);
 121 static void e1000_free_rx_resources(struct e1000_adapter *adapter,
 122                                     struct e1000_rx_ring *rx_ring);
 123 void e1000_update_stats(struct e1000_adapter *adapter);
 124
 125 /* Local Function Prototypes */
 126
 127 static int e1000_init_module(void);
 128 static void e1000_exit_module(void);
 129 static int e1000_probe(struct pci_dev *pdev, const struct pci_device_id *ent);
 130 static void __devexit e1000_remove(struct pci_dev *pdev);
 131 static int e1000_alloc_queues(struct e1000_adapter *adapter);
 132 #ifdef CONFIG_E1000_MQ
 133 static void e1000_setup_queue_mapping(struct e1000_adapter *adapter);
 134 #endif
 135 static int e1000_sw_init(struct e1000_adapter *adapter);
 136 static int e1000_open(struct net_device *netdev);
 137 static int e1000_close(struct net_device *netdev);
 138 static void e1000_configure_tx(struct e1000_adapter *adapter);
 139 static void e1000_configure_rx(struct e1000_adapter *adapter);
 140 static void e1000_setup_rctl(struct e1000_adapter *adapter);
 141 static void e1000_clean_all_tx_rings(struct e1000_adapter *adapter);
 142 static void e1000_clean_all_rx_rings(struct e1000_adapter *adapter);
 143 static void e1000_clean_tx_ring(struct e1000_adapter *adapter,
 144                                 struct e1000_tx_ring *tx_ring);
 145 static void e1000_clean_rx_ring(struct e1000_adapter *adapter,
 146                                 struct e1000_rx_ring *rx_ring);
 147 static void e1000_set_multi(struct net_device *netdev);
 148 static void e1000_update_phy_info(unsigned long data);
 149 static void e1000_watchdog(unsigned long data);
 150 static void e1000_watchdog_task(struct e1000_adapter *adapter);
 151 static void e1000_82547_tx_fifo_stall(unsigned long data);
 152 static int e1000_xmit_frame(struct sk_buff *skb, struct net_device *netdev);
 153 static struct net_device_stats * e1000_get_stats(struct net_device *netdev);
 154 static int e1000_change_mtu(struct net_device *netdev, int new_mtu);
 155 static int e1000_set_mac(struct net_device *netdev, void *p);
 156 static irqreturn_t e1000_intr(int irq, void *data, struct pt_regs *regs);
 157 static boolean_t e1000_clean_tx_irq(struct e1000_adapter *adapter,
 158                                     struct e1000_tx_ring *tx_ring);
 159 #ifdef CONFIG_E1000_NAPI
 160 static int e1000_clean(struct net_device *poll_dev, int *budget);
 161 static boolean_t e1000_clean_rx_irq(struct e1000_adapter *adapter,
 162                                     struct e1000_rx_ring *rx_ring,
 163                                     int *work_done, int work_to_do);
 164 static boolean_t e1000_clean_rx_irq_ps(struct e1000_adapter *adapter,
 165                                        struct e1000_rx_ring *rx_ring,
 166                                        int *work_done, int work_to_do);
 167 #else
 168 static boolean_t e1000_clean_rx_irq(struct e1000_adapter *adapter,
 169                                     struct e1000_rx_ring *rx_ring);
 170 static boolean_t e1000_clean_rx_irq_ps(struct e1000_adapter *adapter,
 171                                        struct e1000_rx_ring *rx_ring);
 172 #endif
 173 static void e1000_alloc_rx_buffers(struct e1000_adapter *adapter,
 174                                    struct e1000_rx_ring *rx_ring);
 175 static void e1000_alloc_rx_buffers_ps(struct e1000_adapter *adapter,
 176                                       struct e1000_rx_ring *rx_ring);
 177 static int e1000_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd);
 178 static int e1000_mii_ioctl(struct net_device *netdev, struct ifreq *ifr,
 179                            int cmd);
 180 void e1000_set_ethtool_ops(struct net_device *netdev);
 181 static void e1000_enter_82542_rst(struct e1000_adapter *adapter);
 182 static void e1000_leave_82542_rst(struct e1000_adapter *adapter);
 183 static void e1000_tx_timeout(struct net_device *dev);
 184 static void e1000_tx_timeout_task(struct net_device *dev);
 185 static void e1000_smartspeed(struct e1000_adapter *adapter);
 186 static inline int e1000_82547_fifo_workaround(struct e1000_adapter *adapter,
 187                                               struct sk_buff *skb);
 188
 189 static void e1000_vlan_rx_register(struct net_device *netdev, struct vlan_group *grp);
 190 static void e1000_vlan_rx_add_vid(struct net_device *netdev, uint16_t vid);
 191 static void e1000_vlan_rx_kill_vid(struct net_device *netdev, uint16_t vid);
 192 static void e1000_restore_vlan(struct e1000_adapter *adapter);
 193
 194 #ifdef CONFIG_PM
 195 static int e1000_suspend(struct pci_dev *pdev, pm_message_t state);
 196 static int e1000_resume(struct pci_dev *pdev);
 197 #endif
 198
 199 #ifdef CONFIG_NET_POLL_CONTROLLER
 200 /* for netdump / net console */
 201 static void e1000_netpoll (struct net_device *netdev);
 202 #endif
 203
 204 #ifdef CONFIG_E1000_MQ
 205 /* for multiple Rx queues */
 206 void e1000_rx_schedule(void *data);
 207 #endif
 208
 209 /* Exported from other modules */
 210
 211 extern void e1000_check_options(struct e1000_adapter *adapter);
 212
 213 static struct pci_driver e1000_driver = {
 214         .name     = e1000_driver_name,
 215         .id_table = e1000_pci_tbl,
 216         .probe    = e1000_probe,
 217         .remove   = __devexit_p(e1000_remove),
 218         /* Power Managment Hooks */
 219 #ifdef CONFIG_PM
 220         .suspend  = e1000_suspend,
 221         .resume   = e1000_resume
 222 #endif
 223 };
 224
 225 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
 226 MODULE_DESCRIPTION("Intel(R) PRO/1000 Network Driver");
 227 MODULE_LICENSE("GPL");
 228 MODULE_VERSION(DRV_VERSION);
 229
 230 static int debug = NETIF_MSG_DRV | NETIF_MSG_PROBE;
 231 module_param(debug, int, 0);
 232 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
 233
 234 /**
 235  * e1000_init_module - Driver Registration Routine
 236  *
 237  * e1000_init_module is the first routine called when the driver is
 238  * loaded. All it does is register with the PCI subsystem.
 239  **/
 240
 241 static int __init
 242 e1000_init_module(void)
 243 {
 244         int ret;
 245         printk(KERN_INFO "%s - version %s\n",
 246                e1000_driver_string, e1000_driver_version);
 247
 248         printk(KERN_INFO "%s\n", e1000_copyright);
 249
 250         ret = pci_module_init(&e1000_driver);
 251
 252         return ret;
 253 }
 254
 255 module_init(e1000_init_module);
 256
 257 /**
 258  * e1000_exit_module - Driver Exit Cleanup Routine
 259  *
 260  * e1000_exit_module is called just before the driver is removed
 261  * from memory.
 262  **/
 263
 264 static void __exit
 265 e1000_exit_module(void)
 266 {
 267         pci_unregister_driver(&e1000_driver);
 268 }
 269
 270 module_exit(e1000_exit_module);
 271
 272 /**
 273  * e1000_irq_disable - Mask off interrupt generation on the NIC
 274  * @adapter: board private structure
 275  **/
 276
 277 static inline void
 278 e1000_irq_disable(struct e1000_adapter *adapter)
 279 {
 280         atomic_inc(&adapter->irq_sem);
 281         E1000_WRITE_REG(&adapter->hw, IMC, ~0);
 282         E1000_WRITE_FLUSH(&adapter->hw);
 283         synchronize_irq(adapter->pdev->irq);
 284 }
 285
 286 /**
 287  * e1000_irq_enable - Enable default interrupt generation settings
 288  * @adapter: board private structure
 289  **/
 290
 291 static inline void
 292 e1000_irq_enable(struct e1000_adapter *adapter)
 293 {
 294         if(likely(atomic_dec_and_test(&adapter->irq_sem))) {
 295                 E1000_WRITE_REG(&adapter->hw, IMS, IMS_ENABLE_MASK);
 296                 E1000_WRITE_FLUSH(&adapter->hw);
 297         }
 298 }
 299
 300 static void
 301 e1000_update_mng_vlan(struct e1000_adapter *adapter)
 302 {
 303         struct net_device *netdev = adapter->netdev;
 304         uint16_t vid = adapter->hw.mng_cookie.vlan_id;
 305         uint16_t old_vid = adapter->mng_vlan_id;
 306         if(adapter->vlgrp) {
 307                 if(!adapter->vlgrp->vlan_devices[vid]) {
 308                         if(adapter->hw.mng_cookie.status &
 309                                 E1000_MNG_DHCP_COOKIE_STATUS_VLAN_SUPPORT) {
 310                                 e1000_vlan_rx_add_vid(netdev, vid);
 311                                 adapter->mng_vlan_id = vid;
 312                         } else
 313                                 adapter->mng_vlan_id = E1000_MNG_VLAN_NONE;
 314
 315                         if((old_vid != (uint16_t)E1000_MNG_VLAN_NONE) &&
 316                                         (vid != old_vid) &&
 317                                         !adapter->vlgrp->vlan_devices[old_vid])
 318                                 e1000_vlan_rx_kill_vid(netdev, old_vid);
 319                 }
 320         }
 321 }
 322
 323 /**
 324  * e1000_release_hw_control - release control of the h/w to f/w
 325  * @adapter: address of board private structure
 326  *
 327  * e1000_release_hw_control resets {CTRL_EXT|FWSM}:DRV_LOAD bit.
 328  * For ASF and Pass Through versions of f/w this means that the
 329  * driver is no longer loaded. For AMT version (only with 82573) i
 330  * of the f/w this means that the netowrk i/f is closed.
 331  *
 332  **/
 333
 334 static inline void
 335 e1000_release_hw_control(struct e1000_adapter *adapter)
 336 {
 337         uint32_t ctrl_ext;
 338         uint32_t swsm;
 339
 340         /* Let firmware taken over control of h/w */
 341         switch (adapter->hw.mac_type) {
 342         case e1000_82571:
 343         case e1000_82572:
 344                 ctrl_ext = E1000_READ_REG(&adapter->hw, CTRL_EXT);
 345                 E1000_WRITE_REG(&adapter->hw, CTRL_EXT,
 346                                 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
 347                 break;
 348         case e1000_82573:
 349                 swsm = E1000_READ_REG(&adapter->hw, SWSM);
 350                 E1000_WRITE_REG(&adapter->hw, SWSM,
 351                                 swsm & ~E1000_SWSM_DRV_LOAD);
 352         default:
 353                 break;
 354         }
 355 }
 356
 357 /**
 358  * e1000_get_hw_control - get control of the h/w from f/w
 359  * @adapter: address of board private structure
 360  *
 361  * e1000_get_hw_control sets {CTRL_EXT|FWSM}:DRV_LOAD bit.
 362  * For ASF and Pass Through versions of f/w this means that
 363  * the driver is loaded. For AMT version (only with 82573)
 364  * of the f/w this means that the netowrk i/f is open.
 365  *
 366  **/
 367
 368 static inline void
 369 e1000_get_hw_control(struct e1000_adapter *adapter)
 370 {
 371         uint32_t ctrl_ext;
 372         uint32_t swsm;
 373         /* Let firmware know the driver has taken over */
 374         switch (adapter->hw.mac_type) {
 375         case e1000_82571:
 376         case e1000_82572:
 377                 ctrl_ext = E1000_READ_REG(&adapter->hw, CTRL_EXT);
 378                 E1000_WRITE_REG(&adapter->hw, CTRL_EXT,
 379                                 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
 380                 break;
 381         case e1000_82573:
 382                 swsm = E1000_READ_REG(&adapter->hw, SWSM);
 383                 E1000_WRITE_REG(&adapter->hw, SWSM,
 384                                 swsm | E1000_SWSM_DRV_LOAD);
 385                 break;
 386         default:
 387                 break;
 388         }
 389 }
 390
 391 int
 392 e1000_up(struct e1000_adapter *adapter)
 393 {
 394         struct net_device *netdev = adapter->netdev;
 395         int i, err;
 396
 397         /* hardware has been reset, we need to reload some things */
 398
 399         /* Reset the PHY if it was previously powered down */
 400         if(adapter->hw.media_type == e1000_media_type_copper) {
 401                 uint16_t mii_reg;
 402                 e1000_read_phy_reg(&adapter->hw, PHY_CTRL, &mii_reg);
 403                 if(mii_reg & MII_CR_POWER_DOWN)
 404                         e1000_phy_reset(&adapter->hw);
 405         }
 406
 407         e1000_set_multi(netdev);
 408
 409         e1000_restore_vlan(adapter);
 410
 411         e1000_configure_tx(adapter);
 412         e1000_setup_rctl(adapter);
 413         e1000_configure_rx(adapter);
 414         for (i = 0; i < adapter->num_rx_queues; i++) {
 415                 adapter->alloc_rx_buf(adapter, &adapter->rx_ring[i]);
 416         }
 417
 418 #ifdef CONFIG_PCI_MSI
 419         if(adapter->hw.mac_type > e1000_82547_rev_2) {
 420                 adapter->have_msi = TRUE;
 421                 if((err = pci_enable_msi(adapter->pdev))) {
 422                         DPRINTK(PROBE, ERR,
 423                          "Unable to allocate MSI interrupt Error: %d\n", err);
 424                         adapter->have_msi = FALSE;
 425                 }
 426         }
 427 #endif
 428         if((err = request_irq(adapter->pdev->irq, &e1000_intr,
 429                               SA_SHIRQ | SA_SAMPLE_RANDOM,
 430                               netdev->name, netdev))) {
 431                 DPRINTK(PROBE, ERR,
 432                     "Unable to allocate interrupt Error: %d\n", err);
 433                 return err;
 434         }
 435
 436 #ifdef CONFIG_E1000_MQ
 437         e1000_setup_queue_mapping(adapter);
 438 #endif
 439
 440         adapter->tx_queue_len = netdev->tx_queue_len;
 441
 442         mod_timer(&adapter->watchdog_timer, jiffies);
 443
 444 #ifdef CONFIG_E1000_NAPI
 445         netif_poll_enable(netdev);
 446 #endif
 447         e1000_irq_enable(adapter);
 448
 449         return 0;
 450 }
 451
 452 void
 453 e1000_down(struct e1000_adapter *adapter)
 454 {
 455         struct net_device *netdev = adapter->netdev;
 456         boolean_t mng_mode_enabled = (adapter->hw.mac_type >= e1000_82571) &&
 457                                      e1000_check_mng_mode(&adapter->hw);
 458
 459         e1000_irq_disable(adapter);
 460 #ifdef CONFIG_E1000_MQ
 461         while (atomic_read(&adapter->rx_sched_call_data.count) != 0);
 462 #endif
 463         free_irq(adapter->pdev->irq, netdev);
 464 #ifdef CONFIG_PCI_MSI
 465         if(adapter->hw.mac_type > e1000_82547_rev_2 &&
 466            adapter->have_msi == TRUE)
 467                 pci_disable_msi(adapter->pdev);
 468 #endif
 469         del_timer_sync(&adapter->tx_fifo_stall_timer);
 470         del_timer_sync(&adapter->watchdog_timer);
 471         del_timer_sync(&adapter->phy_info_timer);
 472
 473 #ifdef CONFIG_E1000_NAPI
 474         netif_poll_disable(netdev);
 475 #endif
 476         netdev->tx_queue_len = adapter->tx_queue_len;
 477         adapter->link_speed = 0;
 478         adapter->link_duplex = 0;
 479         netif_carrier_off(netdev);
 480         netif_stop_queue(netdev);
 481
 482         e1000_reset(adapter);
 483         e1000_clean_all_tx_rings(adapter);
 484         e1000_clean_all_rx_rings(adapter);
 485
 486         /* Power down the PHY so no link is implied when interface is down *
 487          * The PHY cannot be powered down if any of the following is TRUE *
 488          * (a) WoL is enabled
 489          * (b) AMT is active
 490          * (c) SoL/IDER session is active */
 491         if (!adapter->wol && adapter->hw.mac_type >= e1000_82540 &&
 492            adapter->hw.media_type == e1000_media_type_copper &&
 493            !(E1000_READ_REG(&adapter->hw, MANC) & E1000_MANC_SMBUS_EN) &&
 494            !mng_mode_enabled &&
 495            !e1000_check_phy_reset_block(&adapter->hw)) {
 496                 uint16_t mii_reg;
 497                 e1000_read_phy_reg(&adapter->hw, PHY_CTRL, &mii_reg);
 498                 mii_reg |= MII_CR_POWER_DOWN;
 499                 e1000_write_phy_reg(&adapter->hw, PHY_CTRL, mii_reg);
 500                 mdelay(1);
 501         }
 502 }
 503
 504 void
 505 e1000_reset(struct e1000_adapter *adapter)
 506 {
 507         struct net_device *netdev = adapter->netdev;
 508         uint32_t pba, manc;
 509         uint16_t fc_high_water_mark = E1000_FC_HIGH_DIFF;
 510         uint16_t fc_low_water_mark = E1000_FC_LOW_DIFF;
 511
 512         /* Repartition Pba for greater than 9k mtu
 513          * To take effect CTRL.RST is required.
 514          */
 515
 516         switch (adapter->hw.mac_type) {
 517         case e1000_82547:
 518         case e1000_82547_rev_2:
 519                 pba = E1000_PBA_30K;
 520                 break;
 521         case e1000_82571:
 522         case e1000_82572:
 523                 pba = E1000_PBA_38K;
 524                 break;
 525         case e1000_82573:
 526                 pba = E1000_PBA_12K;
 527                 break;
 528         default:
 529                 pba = E1000_PBA_48K;
 530                 break;
 531         }
 532
 533         if((adapter->hw.mac_type != e1000_82573) &&
 534            (adapter->netdev->mtu > E1000_RXBUFFER_8192)) {
 535                 pba -= 8; /* allocate more FIFO for Tx */
 536                 /* send an XOFF when there is enough space in the
 537                  * Rx FIFO to hold one extra full size Rx packet
 538                 */
 539                 fc_high_water_mark = netdev->mtu + ENET_HEADER_SIZE +
 540                                         ETHERNET_FCS_SIZE + 1;
 541                 fc_low_water_mark = fc_high_water_mark + 8;
 542         }
 543
 544
 545         if(adapter->hw.mac_type == e1000_82547) {
 546                 adapter->tx_fifo_head = 0;
 547                 adapter->tx_head_addr = pba << E1000_TX_HEAD_ADDR_SHIFT;
 548                 adapter->tx_fifo_size =
 549                         (E1000_PBA_40K - pba) << E1000_PBA_BYTES_SHIFT;
 550                 atomic_set(&adapter->tx_fifo_stall, 0);
 551         }
 552
 553         E1000_WRITE_REG(&adapter->hw, PBA, pba);
 554
 555         /* flow control settings */
 556         adapter->hw.fc_high_water = (pba << E1000_PBA_BYTES_SHIFT) -
 557                                     fc_high_water_mark;
 558         adapter->hw.fc_low_water = (pba << E1000_PBA_BYTES_SHIFT) -
 559                                    fc_low_water_mark;
 560         adapter->hw.fc_pause_time = E1000_FC_PAUSE_TIME;
 561         adapter->hw.fc_send_xon = 1;
 562         adapter->hw.fc = adapter->hw.original_fc;
 563
 564         /* Allow time for pending master requests to run */
 565         e1000_reset_hw(&adapter->hw);
 566         if(adapter->hw.mac_type >= e1000_82544)
 567                 E1000_WRITE_REG(&adapter->hw, WUC, 0);
 568         if(e1000_init_hw(&adapter->hw))
 569                 DPRINTK(PROBE, ERR, "Hardware Error\n");
 570         e1000_update_mng_vlan(adapter);
 571         /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
 572         E1000_WRITE_REG(&adapter->hw, VET, ETHERNET_IEEE_VLAN_TYPE);
 573
 574         e1000_reset_adaptive(&adapter->hw);
 575         e1000_phy_get_info(&adapter->hw, &adapter->phy_info);
 576         if (adapter->en_mng_pt) {
 577                 manc = E1000_READ_REG(&adapter->hw, MANC);
 578                 manc |= (E1000_MANC_ARP_EN | E1000_MANC_EN_MNG2HOST);
 579                 E1000_WRITE_REG(&adapter->hw, MANC, manc);
 580         }
 581 }
 582
 583 /**
 584  * e1000_probe - Device Initialization Routine
 585  * @pdev: PCI device information struct
 586  * @ent: entry in e1000_pci_tbl
 587  *
 588  * Returns 0 on success, negative on failure
 589  *
 590  * e1000_probe initializes an adapter identified by a pci_dev structure.
 591  * The OS initialization, configuring of the adapter private structure,
 592  * and a hardware reset occur.
 593  **/
 594
 595 static int __devinit
 596 e1000_probe(struct pci_dev *pdev,
 597             const struct pci_device_id *ent)
 598 {
 599         struct net_device *netdev;
 600         struct e1000_adapter *adapter;
 601         unsigned long mmio_start, mmio_len;
 602
 603         static int cards_found = 0;
 604         int i, err, pci_using_dac;
 605         uint16_t eeprom_data;
 606         uint16_t eeprom_apme_mask = E1000_EEPROM_APME;
 607         if((err = pci_enable_device(pdev)))
 608                 return err;
 609
 610         if(!(err = pci_set_dma_mask(pdev, DMA_64BIT_MASK))) {
 611                 pci_using_dac = 1;
 612         } else {
 613                 if((err = pci_set_dma_mask(pdev, DMA_32BIT_MASK))) {
 614                         E1000_ERR("No usable DMA configuration, aborting\n");
 615                         return err;
 616                 }
 617                 pci_using_dac = 0;
 618         }
 619
 620         if((err = pci_request_regions(pdev, e1000_driver_name)))
 621                 return err;
 622
 623         pci_set_master(pdev);
 624
 625         netdev = alloc_etherdev(sizeof(struct e1000_adapter));
 626         if(!netdev) {
 627                 err = -ENOMEM;
 628                 goto err_alloc_etherdev;
 629         }
 630
 631         SET_MODULE_OWNER(netdev);
 632         SET_NETDEV_DEV(netdev, &pdev->dev);
 633
 634         pci_set_drvdata(pdev, netdev);
 635         adapter = netdev_priv(netdev);
 636         adapter->netdev = netdev;
 637         adapter->pdev = pdev;
 638         adapter->hw.back = adapter;
 639         adapter->msg_enable = (1 << debug) - 1;
 640
 641         mmio_start = pci_resource_start(pdev, BAR_0);
 642         mmio_len = pci_resource_len(pdev, BAR_0);
 643
 644         adapter->hw.hw_addr = ioremap(mmio_start, mmio_len);
 645         if(!adapter->hw.hw_addr) {
 646                 err = -EIO;
 647                 goto err_ioremap;
 648         }
 649
 650         for(i = BAR_1; i <= BAR_5; i++) {
 651                 if(pci_resource_len(pdev, i) == 0)
 652                         continue;
 653                 if(pci_resource_flags(pdev, i) & IORESOURCE_IO) {
 654                         adapter->hw.io_base = pci_resource_start(pdev, i);
 655                         break;
 656                 }
 657         }
 658
 659         netdev->open = &e1000_open;
 660         netdev->stop = &e1000_close;
 661         netdev->hard_start_xmit = &e1000_xmit_frame;
 662         netdev->get_stats = &e1000_get_stats;
 663         netdev->set_multicast_list = &e1000_set_multi;
 664         netdev->set_mac_address = &e1000_set_mac;
 665         netdev->change_mtu = &e1000_change_mtu;
 666         netdev->do_ioctl = &e1000_ioctl;
 667         e1000_set_ethtool_ops(netdev);
 668         netdev->tx_timeout = &e1000_tx_timeout;
 669         netdev->watchdog_timeo = 5 * HZ;
 670 #ifdef CONFIG_E1000_NAPI
 671         netdev->poll = &e1000_clean;
 672         netdev->weight = 64;
 673 #endif
 674         netdev->vlan_rx_register = e1000_vlan_rx_register;
 675         netdev->vlan_rx_add_vid = e1000_vlan_rx_add_vid;
 676         netdev->vlan_rx_kill_vid = e1000_vlan_rx_kill_vid;
 677 #ifdef CONFIG_NET_POLL_CONTROLLER
 678         netdev->poll_controller = e1000_netpoll;
 679 #endif
 680         strcpy(netdev->name, pci_name(pdev));
 681
 682         netdev->mem_start = mmio_start;
 683         netdev->mem_end = mmio_start + mmio_len;
 684         netdev->base_addr = adapter->hw.io_base;
 685
 686         adapter->bd_number = cards_found;
 687
 688         /* setup the private structure */
 689
 690         if((err = e1000_sw_init(adapter)))
 691                 goto err_sw_init;
 692
 693         if((err = e1000_check_phy_reset_block(&adapter->hw)))
 694                 DPRINTK(PROBE, INFO, "PHY reset is blocked due to SOL/IDER session.\n");
 695
 696         if(adapter->hw.mac_type >= e1000_82543) {
 697                 netdev->features = NETIF_F_SG |
 698                                    NETIF_F_HW_CSUM |
 699                                    NETIF_F_HW_VLAN_TX |
 700                                    NETIF_F_HW_VLAN_RX |
 701                                    NETIF_F_HW_VLAN_FILTER;
 702         }
 703
 704 #ifdef NETIF_F_TSO
 705         if((adapter->hw.mac_type >= e1000_82544) &&
 706            (adapter->hw.mac_type != e1000_82547))
 707                 netdev->features |= NETIF_F_TSO;
 708
 709 #ifdef NETIF_F_TSO_IPV6
 710         if(adapter->hw.mac_type > e1000_82547_rev_2)
 711                 netdev->features |= NETIF_F_TSO_IPV6;
 712 #endif
 713 #endif
 714         if(pci_using_dac)
 715                 netdev->features |= NETIF_F_HIGHDMA;
 716
 717         /* hard_start_xmit is safe against parallel locking */
 718         netdev->features |= NETIF_F_LLTX;
 719
 720         adapter->en_mng_pt = e1000_enable_mng_pass_thru(&adapter->hw);
 721
 722         /* before reading the EEPROM, reset the controller to
 723          * put the device in a known good starting state */
 724
 725         e1000_reset_hw(&adapter->hw);
 726
 727         /* make sure the EEPROM is good */
 728
 729         if(e1000_validate_eeprom_checksum(&adapter->hw) < 0) {
 730                 DPRINTK(PROBE, ERR, "The EEPROM Checksum Is Not Valid\n");
 731                 err = -EIO;
 732                 goto err_eeprom;
 733         }
 734
 735         /* copy the MAC address out of the EEPROM */
 736
 737         if(e1000_read_mac_addr(&adapter->hw))
 738                 DPRINTK(PROBE, ERR, "EEPROM Read Error\n");
 739         memcpy(netdev->dev_addr, adapter->hw.mac_addr, netdev->addr_len);
 740         memcpy(netdev->perm_addr, adapter->hw.mac_addr, netdev->addr_len);
 741
 742         if(!is_valid_ether_addr(netdev->perm_addr)) {
 743                 DPRINTK(PROBE, ERR, "Invalid MAC Address\n");
 744                 err = -EIO;
 745                 goto err_eeprom;
 746         }
 747
 748         e1000_read_part_num(&adapter->hw, &(adapter->part_num));
 749
 750         e1000_get_bus_info(&adapter->hw);
 751
 752         init_timer(&adapter->tx_fifo_stall_timer);
 753         adapter->tx_fifo_stall_timer.function = &e1000_82547_tx_fifo_stall;
 754         adapter->tx_fifo_stall_timer.data = (unsigned long) adapter;
 755
 756         init_timer(&adapter->watchdog_timer);
 757         adapter->watchdog_timer.function = &e1000_watchdog;
 758         adapter->watchdog_timer.data = (unsigned long) adapter;
 759
 760         INIT_WORK(&adapter->watchdog_task,
 761                 (void (*)(void *))e1000_watchdog_task, adapter);
 762
 763         init_timer(&adapter->phy_info_timer);
 764         adapter->phy_info_timer.function = &e1000_update_phy_info;
 765         adapter->phy_info_timer.data = (unsigned long) adapter;
 766
 767         INIT_WORK(&adapter->tx_timeout_task,
 768                 (void (*)(void *))e1000_tx_timeout_task, netdev);
 769
 770         /* we're going to reset, so assume we have no link for now */
 771
 772         netif_carrier_off(netdev);
 773         netif_stop_queue(netdev);
 774
 775         e1000_check_options(adapter);
 776
 777         /* Initial Wake on LAN setting
 778          * If APM wake is enabled in the EEPROM,
 779          * enable the ACPI Magic Packet filter
 780          */
 781
 782         switch(adapter->hw.mac_type) {
 783         case e1000_82542_rev2_0:
 784         case e1000_82542_rev2_1:
 785         case e1000_82543:
 786                 break;
 787         case e1000_82544:
 788                 e1000_read_eeprom(&adapter->hw,
 789                         EEPROM_INIT_CONTROL2_REG, 1, &eeprom_data);
 790                 eeprom_apme_mask = E1000_EEPROM_82544_APM;
 791                 break;
 792         case e1000_82546:
 793         case e1000_82546_rev_3:
 794         case e1000_82571:
 795                 if((E1000_READ_REG(&adapter->hw, STATUS) & E1000_STATUS_FUNC_1)
 796                    && (adapter->hw.media_type == e1000_media_type_copper)) {
 797                         e1000_read_eeprom(&adapter->hw,
 798                                 EEPROM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
 799                         break;
 800                 }
 801                 /* Fall Through */
 802         default:
 803                 e1000_read_eeprom(&adapter->hw,
 804                         EEPROM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
 805                 break;
 806         }
 807         if(eeprom_data & eeprom_apme_mask)
 808                 adapter->wol |= E1000_WUFC_MAG;
 809
 810         /* reset the hardware with the new settings */
 811         e1000_reset(adapter);
 812
 813         /* If the controller is 82573 and f/w is AMT, do not set
 814          * DRV_LOAD until the interface is up.  For all other cases,
 815          * let the f/w know that the h/w is now under the control
 816          * of the driver. */
 817         if (adapter->hw.mac_type != e1000_82573 ||
 818             !e1000_check_mng_mode(&adapter->hw))
 819                 e1000_get_hw_control(adapter);
 820
 821         strcpy(netdev->name, "eth%d");
 822         if((err = register_netdev(netdev)))
 823                 goto err_register;
 824
 825         DPRINTK(PROBE, INFO, "Intel(R) PRO/1000 Network Connection\n");
 826
 827         cards_found++;
 828         return 0;
 829
 830 err_register:
 831 err_sw_init:
 832 err_eeprom:
 833         iounmap(adapter->hw.hw_addr);
 834 err_ioremap:
 835         free_netdev(netdev);
 836 err_alloc_etherdev:
 837         pci_release_regions(pdev);
 838         return err;
 839 }
 840
 841 /**
 842  * e1000_remove - Device Removal Routine
 843  * @pdev: PCI device information struct
 844  *
 845  * e1000_remove is called by the PCI subsystem to alert the driver
 846  * that it should release a PCI device.  The could be caused by a
 847  * Hot-Plug event, or because the driver is going to be removed from
 848  * memory.
 849  **/
 850
 851 static void __devexit
 852 e1000_remove(struct pci_dev *pdev)
 853 {
 854         struct net_device *netdev = pci_get_drvdata(pdev);
 855         struct e1000_adapter *adapter = netdev_priv(netdev);
 856         uint32_t manc;
 857 #ifdef CONFIG_E1000_NAPI
 858         int i;
 859 #endif
 860
 861         flush_scheduled_work();
 862
 863         if(adapter->hw.mac_type >= e1000_82540 &&
 864            adapter->hw.media_type == e1000_media_type_copper) {
 865                 manc = E1000_READ_REG(&adapter->hw, MANC);
 866                 if(manc & E1000_MANC_SMBUS_EN) {
 867                         manc |= E1000_MANC_ARP_EN;
 868                         E1000_WRITE_REG(&adapter->hw, MANC, manc);
 869                 }
 870         }
 871
 872         /* Release control of h/w to f/w.  If f/w is AMT enabled, this
 873          * would have already happened in close and is redundant. */
 874         e1000_release_hw_control(adapter);
 875
 876         unregister_netdev(netdev);
 877 #ifdef CONFIG_E1000_NAPI
 878         for (i = 0; i < adapter->num_rx_queues; i++)
 879                 __dev_put(&adapter->polling_netdev[i]);
 880 #endif
 881
 882         if(!e1000_check_phy_reset_block(&adapter->hw))
 883                 e1000_phy_hw_reset(&adapter->hw);
 884
 885         kfree(adapter->tx_ring);
 886         kfree(adapter->rx_ring);
 887 #ifdef CONFIG_E1000_NAPI
 888         kfree(adapter->polling_netdev);
 889 #endif
 890
 891         iounmap(adapter->hw.hw_addr);
 892         pci_release_regions(pdev);
 893
 894 #ifdef CONFIG_E1000_MQ
 895         free_percpu(adapter->cpu_netdev);
 896         free_percpu(adapter->cpu_tx_ring);
 897 #endif
 898         free_netdev(netdev);
 899
 900         pci_disable_device(pdev);
 901 }
 902
 903 /**
 904  * e1000_sw_init - Initialize general software structures (struct e1000_adapter)
 905  * @adapter: board private structure to initialize
 906  *
 907  * e1000_sw_init initializes the Adapter private data structure.
 908  * Fields are initialized based on PCI device information and
 909  * OS network device settings (MTU size).
 910  **/
 911
 912 static int __devinit
 913 e1000_sw_init(struct e1000_adapter *adapter)
 914 {
 915         struct e1000_hw *hw = &adapter->hw;
 916         struct net_device *netdev = adapter->netdev;
 917         struct pci_dev *pdev = adapter->pdev;
 918 #ifdef CONFIG_E1000_NAPI
 919         int i;
 920 #endif
 921
 922         /* PCI config space info */
 923
 924         hw->vendor_id = pdev->vendor;
 925         hw->device_id = pdev->device;
 926         hw->subsystem_vendor_id = pdev->subsystem_vendor;
 927         hw->subsystem_id = pdev->subsystem_device;
 928
 929         pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id);
 930
 931         pci_read_config_word(pdev, PCI_COMMAND, &hw->pci_cmd_word);
 932
 933         adapter->rx_buffer_len = E1000_RXBUFFER_2048;
 934         adapter->rx_ps_bsize0 = E1000_RXBUFFER_256;
 935         hw->max_frame_size = netdev->mtu +
 936                              ENET_HEADER_SIZE + ETHERNET_FCS_SIZE;
 937         hw->min_frame_size = MINIMUM_ETHERNET_FRAME_SIZE;
 938
 939         /* identify the MAC */
 940
 941         if(e1000_set_mac_type(hw)) {
 942                 DPRINTK(PROBE, ERR, "Unknown MAC Type\n");
 943                 return -EIO;
 944         }
 945
 946         /* initialize eeprom parameters */
 947
 948         if(e1000_init_eeprom_params(hw)) {
 949                 E1000_ERR("EEPROM initialization failed\n");
 950                 return -EIO;
 951         }
 952
 953         switch(hw->mac_type) {
 954         default:
 955                 break;
 956         case e1000_82541:
 957         case e1000_82547:
 958         case e1000_82541_rev_2:
 959         case e1000_82547_rev_2:
 960                 hw->phy_init_script = 1;
 961                 break;
 962         }
 963
 964         e1000_set_media_type(hw);
 965
 966         hw->wait_autoneg_complete = FALSE;
 967         hw->tbi_compatibility_en = TRUE;
 968         hw->adaptive_ifs = TRUE;
 969
 970         /* Copper options */
 971
 972         if(hw->media_type == e1000_media_type_copper) {
 973                 hw->mdix = AUTO_ALL_MODES;
 974                 hw->disable_polarity_correction = FALSE;
 975                 hw->master_slave = E1000_MASTER_SLAVE;
 976         }
 977
 978 #ifdef CONFIG_E1000_MQ
 979         /* Number of supported queues */
 980         switch (hw->mac_type) {
 981         case e1000_82571:
 982         case e1000_82572:
 983                 /* These controllers support 2 tx queues, but with a single
 984                  * qdisc implementation, multiple tx queues aren't quite as
 985                  * interesting.  If we can find a logical way of mapping
 986                  * flows to a queue, then perhaps we can up the num_tx_queue
 987                  * count back to its default.  Until then, we run the risk of
 988                  * terrible performance due to SACK overload. */
 989                 adapter->num_tx_queues = 1;
 990                 adapter->num_rx_queues = 2;
 991                 break;
 992         default:
 993                 adapter->num_tx_queues = 1;
 994                 adapter->num_rx_queues = 1;
 995                 break;
 996         }
 997         adapter->num_rx_queues = min(adapter->num_rx_queues, num_online_cpus());
 998         adapter->num_tx_queues = min(adapter->num_tx_queues, num_online_cpus());
 999         DPRINTK(DRV, INFO, "Multiqueue Enabled: Rx Queue count = %u %s\n",
1000                 adapter->num_rx_queues,
1001                 ((adapter->num_rx_queues == 1)
1002                  ? ((num_online_cpus() > 1)
1003                         ? "(due to unsupported feature in current adapter)"
1004                         : "(due to unsupported system configuration)")
1005                  : ""));
1006         DPRINTK(DRV, INFO, "Multiqueue Enabled: Tx Queue count = %u\n",
1007                 adapter->num_tx_queues);
1008 #else
1009         adapter->num_tx_queues = 1;
1010         adapter->num_rx_queues = 1;
1011 #endif
1012
1013         if (e1000_alloc_queues(adapter)) {
1014                 DPRINTK(PROBE, ERR, "Unable to allocate memory for queues\n");
1015                 return -ENOMEM;
1016         }
1017
1018 #ifdef CONFIG_E1000_NAPI
1019         for (i = 0; i < adapter->num_rx_queues; i++) {
1020                 adapter->polling_netdev[i].priv = adapter;
1021                 adapter->polling_netdev[i].poll = &e1000_clean;
1022                 adapter->polling_netdev[i].weight = 64;
1023                 dev_hold(&adapter->polling_netdev[i]);
1024                 set_bit(__LINK_STATE_START, &adapter->polling_netdev[i].state);
1025         }
1026         spin_lock_init(&adapter->tx_queue_lock);
1027 #endif
1028
1029         atomic_set(&adapter->irq_sem, 1);
1030         spin_lock_init(&adapter->stats_lock);
1031
1032         return 0;
1033 }
1034
1035 /**
1036  * e1000_alloc_queues - Allocate memory for all rings
1037  * @adapter: board private structure to initialize
1038  *
1039  * We allocate one ring per queue at run-time since we don't know the
1040  * number of queues at compile-time.  The polling_netdev array is
1041  * intended for Multiqueue, but should work fine with a single queue.
1042  **/
1043
1044 static int __devinit
1045 e1000_alloc_queues(struct e1000_adapter *adapter)
1046 {
1047         int size;
1048
1049         size = sizeof(struct e1000_tx_ring) * adapter->num_tx_queues;
1050         adapter->tx_ring = kmalloc(size, GFP_KERNEL);
1051         if (!adapter->tx_ring)
1052                 return -ENOMEM;
1053         memset(adapter->tx_ring, 0, size);
1054
1055         size = sizeof(struct e1000_rx_ring) * adapter->num_rx_queues;
1056         adapter->rx_ring = kmalloc(size, GFP_KERNEL);
1057         if (!adapter->rx_ring) {
1058                 kfree(adapter->tx_ring);
1059                 return -ENOMEM;
1060         }
1061         memset(adapter->rx_ring, 0, size);
1062
1063 #ifdef CONFIG_E1000_NAPI
1064         size = sizeof(struct net_device) * adapter->num_rx_queues;
1065         adapter->polling_netdev = kmalloc(size, GFP_KERNEL);
1066         if (!adapter->polling_netdev) {
1067                 kfree(adapter->tx_ring);
1068                 kfree(adapter->rx_ring);
1069                 return -ENOMEM;
1070         }
1071         memset(adapter->polling_netdev, 0, size);
1072 #endif
1073
1074 #ifdef CONFIG_E1000_MQ
1075         adapter->rx_sched_call_data.func = e1000_rx_schedule;
1076         adapter->rx_sched_call_data.info = adapter->netdev;
1077
1078         adapter->cpu_netdev = alloc_percpu(struct net_device *);
1079         adapter->cpu_tx_ring = alloc_percpu(struct e1000_tx_ring *);
1080 #endif
1081
1082         return E1000_SUCCESS;
1083 }
1084
1085 #ifdef CONFIG_E1000_MQ
1086 static void __devinit
1087 e1000_setup_queue_mapping(struct e1000_adapter *adapter)
1088 {
1089         int i, cpu;
1090
1091         adapter->rx_sched_call_data.func = e1000_rx_schedule;
1092         adapter->rx_sched_call_data.info = adapter->netdev;
1093         cpus_clear(adapter->rx_sched_call_data.cpumask);
1094
1095         adapter->cpu_netdev = alloc_percpu(struct net_device *);
1096         adapter->cpu_tx_ring = alloc_percpu(struct e1000_tx_ring *);
1097
1098         lock_cpu_hotplug();
1099         i = 0;
1100         for_each_online_cpu(cpu) {
1101                 *per_cpu_ptr(adapter->cpu_tx_ring, cpu) = &adapter->tx_ring[i % adapter->num_tx_queues];
1102                 /* This is incomplete because we'd like to assign separate
1103                  * physical cpus to these netdev polling structures and
1104                  * avoid saturating a subset of cpus.
1105                  */
1106                 if (i < adapter->num_rx_queues) {
1107                         *per_cpu_ptr(adapter->cpu_netdev, cpu) = &adapter->polling_netdev[i];
1108                         adapter->rx_ring[i].cpu = cpu;
1109                         cpu_set(cpu, adapter->cpumask);
1110                 } else
1111                         *per_cpu_ptr(adapter->cpu_netdev, cpu) = NULL;
1112
1113                 i++;
1114         }
1115         unlock_cpu_hotplug();
1116 }
1117 #endif
1118
1119 /**
1120  * e1000_open - Called when a network interface is made active
1121  * @netdev: network interface device structure
1122  *
1123  * Returns 0 on success, negative value on failure
1124  *
1125  * The open entry point is called when a network interface is made
1126  * active by the system (IFF_UP).  At this point all resources needed
1127  * for transmit and receive operations are allocated, the interrupt
1128  * handler is registered with the OS, the watchdog timer is started,
1129  * and the stack is notified that the interface is ready.
1130  **/
1131
1132 static int
1133 e1000_open(struct net_device *netdev)
1134 {
1135         struct e1000_adapter *adapter = netdev_priv(netdev);
1136         int err;
1137
1138         /* allocate transmit descriptors */
1139
1140         if ((err = e1000_setup_all_tx_resources(adapter)))
1141                 goto err_setup_tx;
1142
1143         /* allocate receive descriptors */
1144
1145         if ((err = e1000_setup_all_rx_resources(adapter)))
1146                 goto err_setup_rx;
1147
1148         if((err = e1000_up(adapter)))
1149                 goto err_up;
1150         adapter->mng_vlan_id = E1000_MNG_VLAN_NONE;
1151         if((adapter->hw.mng_cookie.status &
1152                           E1000_MNG_DHCP_COOKIE_STATUS_VLAN_SUPPORT)) {
1153                 e1000_update_mng_vlan(adapter);
1154         }
1155
1156         /* If AMT is enabled, let the firmware know that the network
1157          * interface is now open */
1158         if (adapter->hw.mac_type == e1000_82573 &&
1159             e1000_check_mng_mode(&adapter->hw))
1160                 e1000_get_hw_control(adapter);
1161
1162         return E1000_SUCCESS;
1163
1164 err_up:
1165         e1000_free_all_rx_resources(adapter);
1166 err_setup_rx:
1167         e1000_free_all_tx_resources(adapter);
1168 err_setup_tx:
1169         e1000_reset(adapter);
1170
1171         return err;
1172 }
1173
1174 /**
1175  * e1000_close - Disables a network interface
1176  * @netdev: network interface device structure
1177  *
1178  * Returns 0, this is not allowed to fail
1179  *
1180  * The close entry point is called when an interface is de-activated
1181  * by the OS.  The hardware is still under the drivers control, but
1182  * needs to be disabled.  A global MAC reset is issued to stop the
1183  * hardware, and all transmit and receive resources are freed.
1184  **/
1185
1186 static int
1187 e1000_close(struct net_device *netdev)
1188 {
1189         struct e1000_adapter *adapter = netdev_priv(netdev);
1190
1191         e1000_down(adapter);
1192
1193         e1000_free_all_tx_resources(adapter);
1194         e1000_free_all_rx_resources(adapter);
1195
1196         if((adapter->hw.mng_cookie.status &
1197                           E1000_MNG_DHCP_COOKIE_STATUS_VLAN_SUPPORT)) {
1198                 e1000_vlan_rx_kill_vid(netdev, adapter->mng_vlan_id);
1199         }
1200
1201         /* If AMT is enabled, let the firmware know that the network
1202          * interface is now closed */
1203         if (adapter->hw.mac_type == e1000_82573 &&
1204             e1000_check_mng_mode(&adapter->hw))
1205                 e1000_release_hw_control(adapter);
1206
1207         return 0;
1208 }
1209
1210 /**
1211  * e1000_check_64k_bound - check that memory doesn't cross 64kB boundary
1212  * @adapter: address of board private structure
1213  * @start: address of beginning of memory
1214  * @len: length of memory
1215  **/
1216 static inline boolean_t
1217 e1000_check_64k_bound(struct e1000_adapter *adapter,
1218                       void *start, unsigned long len)
1219 {
1220         unsigned long begin = (unsigned long) start;
1221         unsigned long end = begin + len;
1222
1223         /* First rev 82545 and 82546 need to not allow any memory
1224          * write location to cross 64k boundary due to errata 23 */
1225         if (adapter->hw.mac_type == e1000_82545 ||
1226             adapter->hw.mac_type == e1000_82546) {
1227                 return ((begin ^ (end - 1)) >> 16) != 0 ? FALSE : TRUE;
1228         }
1229
1230         return TRUE;
1231 }
1232
1233 /**
1234  * e1000_setup_tx_resources - allocate Tx resources (Descriptors)
1235  * @adapter: board private structure
1236  * @txdr:    tx descriptor ring (for a specific queue) to setup
1237  *
1238  * Return 0 on success, negative on failure
1239  **/
1240
1241 static int
1242 e1000_setup_tx_resources(struct e1000_adapter *adapter,
1243                          struct e1000_tx_ring *txdr)
1244 {
1245         struct pci_dev *pdev = adapter->pdev;
1246         int size;
1247
1248         size = sizeof(struct e1000_buffer) * txdr->count;
1249
1250         txdr->buffer_info = vmalloc_node(size, pcibus_to_node(pdev->bus));
1251         if(!txdr->buffer_info) {
1252                 DPRINTK(PROBE, ERR,
1253                 "Unable to allocate memory for the transmit descriptor ring\n");
1254                 return -ENOMEM;
1255         }
1256         memset(txdr->buffer_info, 0, size);
1257
1258         /* round up to nearest 4K */
1259
1260         txdr->size = txdr->count * sizeof(struct e1000_tx_desc);
1261         E1000_ROUNDUP(txdr->size, 4096);
1262
1263         txdr->desc = pci_alloc_consistent(pdev, txdr->size, &txdr->dma);
1264         if(!txdr->desc) {
1265 setup_tx_desc_die:
1266                 vfree(txdr->buffer_info);
1267                 DPRINTK(PROBE, ERR,
1268                 "Unable to allocate memory for the transmit descriptor ring\n");
1269                 return -ENOMEM;
1270         }
1271
1272         /* Fix for errata 23, can't cross 64kB boundary */
1273         if (!e1000_check_64k_bound(adapter, txdr->desc, txdr->size)) {
1274                 void *olddesc = txdr->desc;
1275                 dma_addr_t olddma = txdr->dma;
1276                 DPRINTK(TX_ERR, ERR, "txdr align check failed: %u bytes "
1277                                      "at %p\n", txdr->size, txdr->desc);
1278                 /* Try again, without freeing the previous */
1279                 txdr->desc = pci_alloc_consistent(pdev, txdr->size, &txdr->dma);
1280                 if(!txdr->desc) {
1281                 /* Failed allocation, critical failure */
1282                         pci_free_consistent(pdev, txdr->size, olddesc, olddma);
1283                         goto setup_tx_desc_die;
1284                 }
1285
1286                 if (!e1000_check_64k_bound(adapter, txdr->desc, txdr->size)) {
1287                         /* give up */
1288                         pci_free_consistent(pdev, txdr->size, txdr->desc,
1289                                             txdr->dma);
1290                         pci_free_consistent(pdev, txdr->size, olddesc, olddma);
1291                         DPRINTK(PROBE, ERR,
1292                                 "Unable to allocate aligned memory "
1293                                 "for the transmit descriptor ring\n");
1294                         vfree(txdr->buffer_info);
1295                         return -ENOMEM;
1296                 } else {
1297                         /* Free old allocation, new allocation was successful */
1298                         pci_free_consistent(pdev, txdr->size, olddesc, olddma);
1299                 }
1300         }
1301         memset(txdr->desc, 0, txdr->size);
1302
1303         txdr->next_to_use = 0;
1304         txdr->next_to_clean = 0;
1305         spin_lock_init(&txdr->tx_lock);
1306
1307         return 0;
1308 }
1309
1310 /**
1311  * e1000_setup_all_tx_resources - wrapper to allocate Tx resources
1312  *                                (Descriptors) for all queues
1313  * @adapter: board private structure
1314  *
1315  * If this function returns with an error, then it's possible one or
1316  * more of the rings is populated (while the rest are not).  It is the
1317  * callers duty to clean those orphaned rings.
1318  *
1319  * Return 0 on success, negative on failure
1320  **/
1321
1322 int
1323 e1000_setup_all_tx_resources(struct e1000_adapter *adapter)
1324 {
1325         int i, err = 0;
1326
1327         for (i = 0; i < adapter->num_tx_queues; i++) {
1328                 err = e1000_setup_tx_resources(adapter, &adapter->tx_ring[i]);
1329                 if (err) {
1330                         DPRINTK(PROBE, ERR,
1331                                 "Allocation for Tx Queue %u failed\n", i);
1332                         break;
1333                 }
1334         }
1335
1336         return err;
1337 }
1338
1339 /**
1340  * e1000_configure_tx - Configure 8254x Transmit Unit after Reset
1341  * @adapter: board private structure
1342  *
1343  * Configure the Tx unit of the MAC after a reset.
1344  **/
1345
1346 static void
1347 e1000_configure_tx(struct e1000_adapter *adapter)
1348 {
1349         uint64_t tdba;
1350         struct e1000_hw *hw = &adapter->hw;
1351         uint32_t tdlen, tctl, tipg, tarc;
1352
1353         /* Setup the HW Tx Head and Tail descriptor pointers */
1354
1355         switch (adapter->num_tx_queues) {
1356         case 2:
1357                 tdba = adapter->tx_ring[1].dma;
1358                 tdlen = adapter->tx_ring[1].count *
1359                         sizeof(struct e1000_tx_desc);
1360                 E1000_WRITE_REG(hw, TDBAL1, (tdba & 0x00000000ffffffffULL));
1361                 E1000_WRITE_REG(hw, TDBAH1, (tdba >> 32));
1362                 E1000_WRITE_REG(hw, TDLEN1, tdlen);
1363                 E1000_WRITE_REG(hw, TDH1, 0);
1364                 E1000_WRITE_REG(hw, TDT1, 0);
1365                 adapter->tx_ring[1].tdh = E1000_TDH1;
1366                 adapter->tx_ring[1].tdt = E1000_TDT1;
1367                 /* Fall Through */
1368         case 1:
1369         default:
1370                 tdba = adapter->tx_ring[0].dma;
1371                 tdlen = adapter->tx_ring[0].count *
1372                         sizeof(struct e1000_tx_desc);
1373                 E1000_WRITE_REG(hw, TDBAL, (tdba & 0x00000000ffffffffULL));
1374                 E1000_WRITE_REG(hw, TDBAH, (tdba >> 32));
1375                 E1000_WRITE_REG(hw, TDLEN, tdlen);
1376                 E1000_WRITE_REG(hw, TDH, 0);
1377                 E1000_WRITE_REG(hw, TDT, 0);
1378                 adapter->tx_ring[0].tdh = E1000_TDH;
1379                 adapter->tx_ring[0].tdt = E1000_TDT;
1380                 break;
1381         }
1382
1383         /* Set the default values for the Tx Inter Packet Gap timer */
1384
1385         switch (hw->mac_type) {
1386         case e1000_82542_rev2_0:
1387         case e1000_82542_rev2_1:
1388                 tipg = DEFAULT_82542_TIPG_IPGT;
1389                 tipg |= DEFAULT_82542_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
1390                 tipg |= DEFAULT_82542_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
1391                 break;
1392         default:
1393                 if (hw->media_type == e1000_media_type_fiber ||
1394                     hw->media_type == e1000_media_type_internal_serdes)
1395                         tipg = DEFAULT_82543_TIPG_IPGT_FIBER;
1396                 else
1397                         tipg = DEFAULT_82543_TIPG_IPGT_COPPER;
1398                 tipg |= DEFAULT_82543_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
1399                 tipg |= DEFAULT_82543_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
1400         }
1401         E1000_WRITE_REG(hw, TIPG, tipg);
1402
1403         /* Set the Tx Interrupt Delay register */
1404
1405         E1000_WRITE_REG(hw, TIDV, adapter->tx_int_delay);
1406         if (hw->mac_type >= e1000_82540)
1407                 E1000_WRITE_REG(hw, TADV, adapter->tx_abs_int_delay);
1408
1409         /* Program the Transmit Control Register */
1410
1411         tctl = E1000_READ_REG(hw, TCTL);
1412
1413         tctl &= ~E1000_TCTL_CT;
1414         tctl |= E1000_TCTL_EN | E1000_TCTL_PSP | E1000_TCTL_RTLC |
1415                 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
1416
1417         E1000_WRITE_REG(hw, TCTL, tctl);
1418
1419         if (hw->mac_type == e1000_82571 || hw->mac_type == e1000_82572) {
1420                 tarc = E1000_READ_REG(hw, TARC0);
1421                 tarc |= ((1 << 25) | (1 << 21));
1422                 E1000_WRITE_REG(hw, TARC0, tarc);
1423                 tarc = E1000_READ_REG(hw, TARC1);
1424                 tarc |= (1 << 25);
1425                 if (tctl & E1000_TCTL_MULR)
1426                         tarc &= ~(1 << 28);
1427                 else
1428                         tarc |= (1 << 28);
1429                 E1000_WRITE_REG(hw, TARC1, tarc);
1430         }
1431
1432         e1000_config_collision_dist(hw);
1433
1434         /* Setup Transmit Descriptor Settings for eop descriptor */
1435         adapter->txd_cmd = E1000_TXD_CMD_IDE | E1000_TXD_CMD_EOP |
1436                 E1000_TXD_CMD_IFCS;
1437
1438         if (hw->mac_type < e1000_82543)
1439                 adapter->txd_cmd |= E1000_TXD_CMD_RPS;
1440         else
1441                 adapter->txd_cmd |= E1000_TXD_CMD_RS;
1442
1443         /* Cache if we're 82544 running in PCI-X because we'll
1444          * need this to apply a workaround later in the send path. */
1445         if (hw->mac_type == e1000_82544 &&
1446             hw->bus_type == e1000_bus_type_pcix)
1447                 adapter->pcix_82544 = 1;
1448 }
1449
1450 /**
1451  * e1000_setup_rx_resources - allocate Rx resources (Descriptors)
1452  * @adapter: board private structure
1453  * @rxdr:    rx descriptor ring (for a specific queue) to setup
1454  *
1455  * Returns 0 on success, negative on failure
1456  **/
1457
1458 static int
1459 e1000_setup_rx_resources(struct e1000_adapter *adapter,
1460                          struct e1000_rx_ring *rxdr)
1461 {
1462         struct pci_dev *pdev = adapter->pdev;
1463         int size, desc_len;
1464
1465         size = sizeof(struct e1000_buffer) * rxdr->count;
1466         rxdr->buffer_info = vmalloc_node(size, pcibus_to_node(pdev->bus));
1467         if (!rxdr->buffer_info) {
1468                 DPRINTK(PROBE, ERR,
1469                 "Unable to allocate memory for the receive descriptor ring\n");
1470                 return -ENOMEM;
1471         }
1472         memset(rxdr->buffer_info, 0, size);
1473
1474         size = sizeof(struct e1000_ps_page) * rxdr->count;
1475         rxdr->ps_page = kmalloc(size, GFP_KERNEL);
1476         if(!rxdr->ps_page) {
1477                 vfree(rxdr->buffer_info);
1478                 DPRINTK(PROBE, ERR,
1479                 "Unable to allocate memory for the receive descriptor ring\n");
1480                 return -ENOMEM;
1481         }
1482         memset(rxdr->ps_page, 0, size);
1483
1484         size = sizeof(struct e1000_ps_page_dma) * rxdr->count;
1485         rxdr->ps_page_dma = kmalloc(size, GFP_KERNEL);
1486         if(!rxdr->ps_page_dma) {
1487                 vfree(rxdr->buffer_info);
1488                 kfree(rxdr->ps_page);
1489                 DPRINTK(PROBE, ERR,
1490                 "Unable to allocate memory for the receive descriptor ring\n");
1491                 return -ENOMEM;
1492         }
1493         memset(rxdr->ps_page_dma, 0, size);
1494
1495         if(adapter->hw.mac_type <= e1000_82547_rev_2)
1496                 desc_len = sizeof(struct e1000_rx_desc);
1497         else
1498                 desc_len = sizeof(union e1000_rx_desc_packet_split);
1499
1500         /* Round up to nearest 4K */
1501
1502         rxdr->size = rxdr->count * desc_len;
1503         E1000_ROUNDUP(rxdr->size, 4096);
1504
1505         rxdr->desc = pci_alloc_consistent(pdev, rxdr->size, &rxdr->dma);
1506
1507         if (!rxdr->desc) {
1508                 DPRINTK(PROBE, ERR,
1509                 "Unable to allocate memory for the receive descriptor ring\n");
1510 setup_rx_desc_die:
1511                 vfree(rxdr->buffer_info);
1512                 kfree(rxdr->ps_page);
1513                 kfree(rxdr->ps_page_dma);
1514                 return -ENOMEM;
1515         }
1516
1517         /* Fix for errata 23, can't cross 64kB boundary */
1518         if (!e1000_check_64k_bound(adapter, rxdr->desc, rxdr->size)) {
1519                 void *olddesc = rxdr->desc;
1520                 dma_addr_t olddma = rxdr->dma;
1521                 DPRINTK(RX_ERR, ERR, "rxdr align check failed: %u bytes "
1522                                      "at %p\n", rxdr->size, rxdr->desc);
1523                 /* Try again, without freeing the previous */
1524                 rxdr->desc = pci_alloc_consistent(pdev, rxdr->size, &rxdr->dma);
1525                 /* Failed allocation, critical failure */
1526                 if (!rxdr->desc) {
1527                         pci_free_consistent(pdev, rxdr->size, olddesc, olddma);
1528                         DPRINTK(PROBE, ERR,
1529                                 "Unable to allocate memory "
1530                                 "for the receive descriptor ring\n");
1531                         goto setup_rx_desc_die;
1532                 }
1533
1534                 if (!e1000_check_64k_bound(adapter, rxdr->desc, rxdr->size)) {
1535                         /* give up */
1536                         pci_free_consistent(pdev, rxdr->size, rxdr->desc,
1537                                             rxdr->dma);
1538                         pci_free_consistent(pdev, rxdr->size, olddesc, olddma);
1539                         DPRINTK(PROBE, ERR,
1540                                 "Unable to allocate aligned memory "
1541                                 "for the receive descriptor ring\n");
1542                         goto setup_rx_desc_die;
1543                 } else {
1544                         /* Free old allocation, new allocation was successful */
1545                         pci_free_consistent(pdev, rxdr->size, olddesc, olddma);
1546                 }
1547         }
1548         memset(rxdr->desc, 0, rxdr->size);
1549
1550         rxdr->next_to_clean = 0;
1551         rxdr->next_to_use = 0;
1552
1553         return 0;
1554 }
1555
1556 /**
1557  * e1000_setup_all_rx_resources - wrapper to allocate Rx resources
1558  *                                (Descriptors) for all queues
1559  * @adapter: board private structure
1560  *
1561  * If this function returns with an error, then it's possible one or
1562  * more of the rings is populated (while the rest are not).  It is the
1563  * callers duty to clean those orphaned rings.
1564  *
1565  * Return 0 on success, negative on failure
1566  **/
1567
1568 int
1569 e1000_setup_all_rx_resources(struct e1000_adapter *adapter)
1570 {
1571         int i, err = 0;
1572
1573         for (i = 0; i < adapter->num_rx_queues; i++) {
1574                 err = e1000_setup_rx_resources(adapter, &adapter->rx_ring[i]);
1575                 if (err) {
1576                         DPRINTK(PROBE, ERR,
1577                                 "Allocation for Rx Queue %u failed\n", i);
1578                         break;
1579                 }
1580         }
1581
1582         return err;
1583 }
1584
1585 /**
1586  * e1000_setup_rctl - configure the receive control registers
1587  * @adapter: Board private structure
1588  **/
1589 #define PAGE_USE_COUNT(S) (((S) >> PAGE_SHIFT) + \
1590                         (((S) & (PAGE_SIZE - 1)) ? 1 : 0))
1591 static void
1592 e1000_setup_rctl(struct e1000_adapter *adapter)
1593 {
1594         uint32_t rctl, rfctl;
1595         uint32_t psrctl = 0;
1596 #ifdef CONFIG_E1000_PACKET_SPLIT
1597         uint32_t pages = 0;
1598 #endif
1599
1600         rctl = E1000_READ_REG(&adapter->hw, RCTL);
1601
1602         rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
1603
1604         rctl |= E1000_RCTL_EN | E1000_RCTL_BAM |
1605                 E1000_RCTL_LBM_NO | E1000_RCTL_RDMTS_HALF |
1606                 (adapter->hw.mc_filter_type << E1000_RCTL_MO_SHIFT);
1607
1608         if(adapter->hw.tbi_compatibility_on == 1)
1609                 rctl |= E1000_RCTL_SBP;
1610         else
1611                 rctl &= ~E1000_RCTL_SBP;
1612
1613         if (adapter->netdev->mtu <= ETH_DATA_LEN)
1614                 rctl &= ~E1000_RCTL_LPE;
1615         else
1616                 rctl |= E1000_RCTL_LPE;
1617
1618         /* Setup buffer sizes */
1619         if(adapter->hw.mac_type >= e1000_82571) {
1620                 /* We can now specify buffers in 1K increments.
1621                  * BSIZE and BSEX are ignored in this case. */
1622                 rctl |= adapter->rx_buffer_len << 0x11;
1623         } else {
1624                 rctl &= ~E1000_RCTL_SZ_4096;
1625                 rctl |= E1000_RCTL_BSEX;
1626                 switch (adapter->rx_buffer_len) {
1627                 case E1000_RXBUFFER_2048:
1628                 default:
1629                         rctl |= E1000_RCTL_SZ_2048;
1630                         rctl &= ~E1000_RCTL_BSEX;
1631                         break;
1632                 case E1000_RXBUFFER_4096:
1633                         rctl |= E1000_RCTL_SZ_4096;
1634                         break;
1635                 case E1000_RXBUFFER_8192:
1636                         rctl |= E1000_RCTL_SZ_8192;
1637                         break;
1638                 case E1000_RXBUFFER_16384:
1639                         rctl |= E1000_RCTL_SZ_16384;
1640                         break;
1641                 }
1642         }
1643
1644 #ifdef CONFIG_E1000_PACKET_SPLIT
1645         /* 82571 and greater support packet-split where the protocol
1646          * header is placed in skb->data and the packet data is
1647          * placed in pages hanging off of skb_shinfo(skb)->nr_frags.
1648          * In the case of a non-split, skb->data is linearly filled,
1649          * followed by the page buffers.  Therefore, skb->data is
1650          * sized to hold the largest protocol header.
1651          */
1652         pages = PAGE_USE_COUNT(adapter->netdev->mtu);
1653         if ((adapter->hw.mac_type > e1000_82547_rev_2) && (pages <= 3) &&
1654             PAGE_SIZE <= 16384)
1655                 adapter->rx_ps_pages = pages;
1656         else
1657                 adapter->rx_ps_pages = 0;
1658 #endif
1659         if (adapter->rx_ps_pages) {
1660                 /* Configure extra packet-split registers */
1661                 rfctl = E1000_READ_REG(&adapter->hw, RFCTL);
1662                 rfctl |= E1000_RFCTL_EXTEN;
1663                 /* disable IPv6 packet split support */
1664                 rfctl |= E1000_RFCTL_IPV6_DIS;
1665                 E1000_WRITE_REG(&adapter->hw, RFCTL, rfctl);
1666
1667                 rctl |= E1000_RCTL_DTYP_PS | E1000_RCTL_SECRC;
1668
1669                 psrctl |= adapter->rx_ps_bsize0 >>
1670                         E1000_PSRCTL_BSIZE0_SHIFT;
1671
1672                 switch (adapter->rx_ps_pages) {
1673                 case 3:
1674                         psrctl |= PAGE_SIZE <<
1675                                 E1000_PSRCTL_BSIZE3_SHIFT;
1676                 case 2:
1677                         psrctl |= PAGE_SIZE <<
1678                                 E1000_PSRCTL_BSIZE2_SHIFT;
1679                 case 1:
1680                         psrctl |= PAGE_SIZE >>
1681                                 E1000_PSRCTL_BSIZE1_SHIFT;
1682                         break;
1683                 }
1684
1685                 E1000_WRITE_REG(&adapter->hw, PSRCTL, psrctl);
1686         }
1687
1688         E1000_WRITE_REG(&adapter->hw, RCTL, rctl);
1689 }
1690
1691 /**
1692  * e1000_configure_rx - Configure 8254x Receive Unit after Reset
1693  * @adapter: board private structure
1694  *
1695  * Configure the Rx unit of the MAC after a reset.
1696  **/
1697
1698 static void
1699 e1000_configure_rx(struct e1000_adapter *adapter)
1700 {
1701         uint64_t rdba;
1702         struct e1000_hw *hw = &adapter->hw;
1703         uint32_t rdlen, rctl, rxcsum, ctrl_ext;
1704 #ifdef CONFIG_E1000_MQ
1705         uint32_t reta, mrqc;
1706         int i;
1707 #endif
1708
1709         if (adapter->rx_ps_pages) {
1710                 rdlen = adapter->rx_ring[0].count *
1711                         sizeof(union e1000_rx_desc_packet_split);
1712                 adapter->clean_rx = e1000_clean_rx_irq_ps;
1713                 adapter->alloc_rx_buf = e1000_alloc_rx_buffers_ps;
1714         } else {
1715                 rdlen = adapter->rx_ring[0].count *
1716                         sizeof(struct e1000_rx_desc);
1717                 adapter->clean_rx = e1000_clean_rx_irq;
1718                 adapter->alloc_rx_buf = e1000_alloc_rx_buffers;
1719         }
1720
1721         /* disable receives while setting up the descriptors */
1722         rctl = E1000_READ_REG(hw, RCTL);
1723         E1000_WRITE_REG(hw, RCTL, rctl & ~E1000_RCTL_EN);
1724
1725         /* set the Receive Delay Timer Register */
1726         E1000_WRITE_REG(hw, RDTR, adapter->rx_int_delay);
1727
1728         if (hw->mac_type >= e1000_82540) {
1729                 E1000_WRITE_REG(hw, RADV, adapter->rx_abs_int_delay);
1730                 if(adapter->itr > 1)
1731                         E1000_WRITE_REG(hw, ITR,
1732                                 1000000000 / (adapter->itr * 256));
1733         }
1734
1735         if (hw->mac_type >= e1000_82571) {
1736                 /* Reset delay timers after every interrupt */
1737                 ctrl_ext = E1000_READ_REG(hw, CTRL_EXT);
1738                 ctrl_ext |= E1000_CTRL_EXT_CANC;
1739                 E1000_WRITE_REG(hw, CTRL_EXT, ctrl_ext);
1740                 E1000_WRITE_FLUSH(hw);
1741         }
1742
1743         /* Setup the HW Rx Head and Tail Descriptor Pointers and
1744          * the Base and Length of the Rx Descriptor Ring */
1745         switch (adapter->num_rx_queues) {
1746 #ifdef CONFIG_E1000_MQ
1747         case 2:
1748                 rdba = adapter->rx_ring[1].dma;
1749                 E1000_WRITE_REG(hw, RDBAL1, (rdba & 0x00000000ffffffffULL));
1750                 E1000_WRITE_REG(hw, RDBAH1, (rdba >> 32));
1751                 E1000_WRITE_REG(hw, RDLEN1, rdlen);
1752                 E1000_WRITE_REG(hw, RDH1, 0);
1753                 E1000_WRITE_REG(hw, RDT1, 0);
1754                 adapter->rx_ring[1].rdh = E1000_RDH1;
1755                 adapter->rx_ring[1].rdt = E1000_RDT1;
1756                 /* Fall Through */
1757 #endif
1758         case 1:
1759         default:
1760                 rdba = adapter->rx_ring[0].dma;
1761                 E1000_WRITE_REG(hw, RDBAL, (rdba & 0x00000000ffffffffULL));
1762                 E1000_WRITE_REG(hw, RDBAH, (rdba >> 32));
1763                 E1000_WRITE_REG(hw, RDLEN, rdlen);
1764                 E1000_WRITE_REG(hw, RDH, 0);
1765                 E1000_WRITE_REG(hw, RDT, 0);
1766                 adapter->rx_ring[0].rdh = E1000_RDH;
1767                 adapter->rx_ring[0].rdt = E1000_RDT;
1768                 break;
1769         }
1770
1771 #ifdef CONFIG_E1000_MQ
1772         if (adapter->num_rx_queues > 1) {
1773                 uint32_t random[10];
1774
1775                 get_random_bytes(&random[0], 40);
1776
1777                 if (hw->mac_type <= e1000_82572) {
1778                         E1000_WRITE_REG(hw, RSSIR, 0);
1779                         E1000_WRITE_REG(hw, RSSIM, 0);
1780                 }
1781
1782                 switch (adapter->num_rx_queues) {
1783                 case 2:
1784                 default:
1785                         reta = 0x00800080;
1786                         mrqc = E1000_MRQC_ENABLE_RSS_2Q;
1787                         break;
1788                 }
1789
1790                 /* Fill out redirection table */
1791                 for (i = 0; i < 32; i++)
1792                         E1000_WRITE_REG_ARRAY(hw, RETA, i, reta);
1793                 /* Fill out hash function seeds */
1794                 for (i = 0; i < 10; i++)
1795                         E1000_WRITE_REG_ARRAY(hw, RSSRK, i, random[i]);
1796
1797                 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 |
1798                          E1000_MRQC_RSS_FIELD_IPV4_TCP);
1799                 E1000_WRITE_REG(hw, MRQC, mrqc);
1800         }
1801
1802         /* Multiqueue and packet checksumming are mutually exclusive. */
1803         if (hw->mac_type >= e1000_82571) {
1804                 rxcsum = E1000_READ_REG(hw, RXCSUM);
1805                 rxcsum |= E1000_RXCSUM_PCSD;
1806                 E1000_WRITE_REG(hw, RXCSUM, rxcsum);
1807         }
1808
1809 #else
1810
1811         /* Enable 82543 Receive Checksum Offload for TCP and UDP */
1812         if (hw->mac_type >= e1000_82543) {
1813                 rxcsum = E1000_READ_REG(hw, RXCSUM);
1814                 if(adapter->rx_csum == TRUE) {
1815                         rxcsum |= E1000_RXCSUM_TUOFL;
1816
1817                         /* Enable 82571 IPv4 payload checksum for UDP fragments
1818                          * Must be used in conjunction with packet-split. */
1819                         if ((hw->mac_type >= e1000_82571) &&
1820                            (adapter->rx_ps_pages)) {
1821                                 rxcsum |= E1000_RXCSUM_IPPCSE;
1822                         }
1823                 } else {
1824                         rxcsum &= ~E1000_RXCSUM_TUOFL;
1825                         /* don't need to clear IPPCSE as it defaults to 0 */
1826                 }
1827                 E1000_WRITE_REG(hw, RXCSUM, rxcsum);
1828         }
1829 #endif /* CONFIG_E1000_MQ */
1830
1831         if (hw->mac_type == e1000_82573)
1832                 E1000_WRITE_REG(hw, ERT, 0x0100);
1833
1834         /* Enable Receives */
1835         E1000_WRITE_REG(hw, RCTL, rctl);
1836 }
1837
1838 /**
1839  * e1000_free_tx_resources - Free Tx Resources per Queue
1840  * @adapter: board private structure
1841  * @tx_ring: Tx descriptor ring for a specific queue
1842  *
1843  * Free all transmit software resources
1844  **/
1845
1846 static void
1847 e1000_free_tx_resources(struct e1000_adapter *adapter,
1848                         struct e1000_tx_ring *tx_ring)
1849 {
1850         struct pci_dev *pdev = adapter->pdev;
1851
1852         e1000_clean_tx_ring(adapter, tx_ring);
1853
1854         vfree(tx_ring->buffer_info);
1855         tx_ring->buffer_info = NULL;
1856
1857         pci_free_consistent(pdev, tx_ring->size, tx_ring->desc, tx_ring->dma);
1858
1859         tx_ring->desc = NULL;
1860 }
1861
1862 /**
1863  * e1000_free_all_tx_resources - Free Tx Resources for All Queues
1864  * @adapter: board private structure
1865  *
1866  * Free all transmit software resources
1867  **/
1868
1869 void
1870 e1000_free_all_tx_resources(struct e1000_adapter *adapter)
1871 {
1872         int i;
1873
1874         for (i = 0; i < adapter->num_tx_queues; i++)
1875                 e1000_free_tx_resources(adapter, &adapter->tx_ring[i]);
1876 }
1877
1878 static inline void
1879 e1000_unmap_and_free_tx_resource(struct e1000_adapter *adapter,
1880                         struct e1000_buffer *buffer_info)
1881 {
1882         if(buffer_info->dma) {
1883                 pci_unmap_page(adapter->pdev,
1884                                 buffer_info->dma,
1885                                 buffer_info->length,
1886                                 PCI_DMA_TODEVICE);
1887                 buffer_info->dma = 0;
1888         }
1889         if(buffer_info->skb) {
1890                 dev_kfree_skb_any(buffer_info->skb);
1891                 buffer_info->skb = NULL;
1892         }
1893 }
1894
1895 /**
1896  * e1000_clean_tx_ring - Free Tx Buffers
1897  * @adapter: board private structure
1898  * @tx_ring: ring to be cleaned
1899  **/
1900
1901 static void
1902 e1000_clean_tx_ring(struct e1000_adapter *adapter,
1903                     struct e1000_tx_ring *tx_ring)
1904 {
1905         struct e1000_buffer *buffer_info;
1906         unsigned long size;
1907         unsigned int i;
1908
1909         /* Free all the Tx ring sk_buffs */
1910
1911         for(i = 0; i < tx_ring->count; i++) {
1912                 buffer_info = &tx_ring->buffer_info[i];
1913                 e1000_unmap_and_free_tx_resource(adapter, buffer_info);
1914         }
1915
1916         size = sizeof(struct e1000_buffer) * tx_ring->count;
1917         memset(tx_ring->buffer_info, 0, size);
1918
1919         /* Zero out the descriptor ring */
1920
1921         memset(tx_ring->desc, 0, tx_ring->size);
1922
1923         tx_ring->next_to_use = 0;
1924         tx_ring->next_to_clean = 0;
1925         tx_ring->last_tx_tso = 0;
1926
1927         writel(0, adapter->hw.hw_addr + tx_ring->tdh);
1928         writel(0, adapter->hw.hw_addr + tx_ring->tdt);
1929 }
1930
1931 /**
1932  * e1000_clean_all_tx_rings - Free Tx Buffers for all queues
1933  * @adapter: board private structure
1934  **/
1935
1936 static void
1937 e1000_clean_all_tx_rings(struct e1000_adapter *adapter)
1938 {
1939         int i;
1940
1941         for (i = 0; i < adapter->num_tx_queues; i++)
1942                 e1000_clean_tx_ring(adapter, &adapter->tx_ring[i]);
1943 }
1944
1945 /**
1946  * e1000_free_rx_resources - Free Rx Resources
1947  * @adapter: board private structure
1948  * @rx_ring: ring to clean the resources from
1949  *
1950  * Free all receive software resources
1951  **/
1952
1953 static void
1954 e1000_free_rx_resources(struct e1000_adapter *adapter,
1955                         struct e1000_rx_ring *rx_ring)
1956 {
1957         struct pci_dev *pdev = adapter->pdev;
1958
1959         e1000_clean_rx_ring(adapter, rx_ring);
1960
1961         vfree(rx_ring->buffer_info);
1962         rx_ring->buffer_info = NULL;
1963         kfree(rx_ring->ps_page);
1964         rx_ring->ps_page = NULL;
1965         kfree(rx_ring->ps_page_dma);
1966         rx_ring->ps_page_dma = NULL;
1967
1968         pci_free_consistent(pdev, rx_ring->size, rx_ring->desc, rx_ring->dma);
1969
1970         rx_ring->desc = NULL;
1971 }
1972
1973 /**
1974  * e1000_free_all_rx_resources - Free Rx Resources for All Queues
1975  * @adapter: board private structure
1976  *
1977  * Free all receive software resources
1978  **/
1979
1980 void
1981 e1000_free_all_rx_resources(struct e1000_adapter *adapter)
1982 {
1983         int i;
1984
1985         for (i = 0; i < adapter->num_rx_queues; i++)
1986                 e1000_free_rx_resources(adapter, &adapter->rx_ring[i]);
1987 }
1988
1989 /**
1990  * e1000_clean_rx_ring - Free Rx Buffers per Queue
1991  * @adapter: board private structure
1992  * @rx_ring: ring to free buffers from
1993  **/
1994
1995 static void
1996 e1000_clean_rx_ring(struct e1000_adapter *adapter,
1997                     struct e1000_rx_ring *rx_ring)
1998 {
1999         struct e1000_buffer *buffer_info;
2000         struct e1000_ps_page *ps_page;
2001         struct e1000_ps_page_dma *ps_page_dma;
2002         struct pci_dev *pdev = adapter->pdev;
2003         unsigned long size;
2004         unsigned int i, j;
2005
2006         /* Free all the Rx ring sk_buffs */
2007
2008         for(i = 0; i < rx_ring->count; i++) {
2009                 buffer_info = &rx_ring->buffer_info[i];
2010                 if(buffer_info->skb) {
2011                         ps_page = &rx_ring->ps_page[i];
2012                         ps_page_dma = &rx_ring->ps_page_dma[i];
2013                         pci_unmap_single(pdev,
2014                                          buffer_info->dma,
2015                                          buffer_info->length,
2016                                          PCI_DMA_FROMDEVICE);
2017
2018                         dev_kfree_skb(buffer_info->skb);
2019                         buffer_info->skb = NULL;
2020
2021                         for(j = 0; j < adapter->rx_ps_pages; j++) {
2022                                 if(!ps_page->ps_page[j]) break;
2023                                 pci_unmap_single(pdev,
2024                                                  ps_page_dma->ps_page_dma[j],
2025                                                  PAGE_SIZE, PCI_DMA_FROMDEVICE);
2026                                 ps_page_dma->ps_page_dma[j] = 0;
2027                                 put_page(ps_page->ps_page[j]);
2028                                 ps_page->ps_page[j] = NULL;
2029                         }
2030                 }
2031         }
2032
2033         size = sizeof(struct e1000_buffer) * rx_ring->count;
2034         memset(rx_ring->buffer_info, 0, size);
2035         size = sizeof(struct e1000_ps_page) * rx_ring->count;
2036         memset(rx_ring->ps_page, 0, size);
2037         size = sizeof(struct e1000_ps_page_dma) * rx_ring->count;
2038         memset(rx_ring->ps_page_dma, 0, size);
2039
2040         /* Zero out the descriptor ring */
2041
2042         memset(rx_ring->desc, 0, rx_ring->size);
2043
2044         rx_ring->next_to_clean = 0;
2045         rx_ring->next_to_use = 0;
2046
2047         writel(0, adapter->hw.hw_addr + rx_ring->rdh);
2048         writel(0, adapter->hw.hw_addr + rx_ring->rdt);
2049 }
2050
2051 /**
2052  * e1000_clean_all_rx_rings - Free Rx Buffers for all queues
2053  * @adapter: board private structure
2054  **/
2055
2056 static void
2057 e1000_clean_all_rx_rings(struct e1000_adapter *adapter)
2058 {
2059         int i;
2060
2061         for (i = 0; i < adapter->num_rx_queues; i++)
2062                 e1000_clean_rx_ring(adapter, &adapter->rx_ring[i]);
2063 }
2064
2065 /* The 82542 2.0 (revision 2) needs to have the receive unit in reset
2066  * and memory write and invalidate disabled for certain operations
2067  */
2068 static void
2069 e1000_enter_82542_rst(struct e1000_adapter *adapter)
2070 {
2071         struct net_device *netdev = adapter->netdev;
2072         uint32_t rctl;
2073
2074         e1000_pci_clear_mwi(&adapter->hw);
2075
2076         rctl = E1000_READ_REG(&adapter->hw, RCTL);
2077         rctl |= E1000_RCTL_RST;
2078         E1000_WRITE_REG(&adapter->hw, RCTL, rctl);
2079         E1000_WRITE_FLUSH(&adapter->hw);
2080         mdelay(5);
2081
2082         if(netif_running(netdev))
2083                 e1000_clean_all_rx_rings(adapter);
2084 }
2085
2086 static void
2087 e1000_leave_82542_rst(struct e1000_adapter *adapter)
2088 {
2089         struct net_device *netdev = adapter->netdev;
2090         uint32_t rctl;
2091
2092         rctl = E1000_READ_REG(&adapter->hw, RCTL);
2093         rctl &= ~E1000_RCTL_RST;
2094         E1000_WRITE_REG(&adapter->hw, RCTL, rctl);
2095         E1000_WRITE_FLUSH(&adapter->hw);
2096         mdelay(5);
2097
2098         if(adapter->hw.pci_cmd_word & PCI_COMMAND_INVALIDATE)
2099                 e1000_pci_set_mwi(&adapter->hw);
2100
2101         if(netif_running(netdev)) {
2102                 e1000_configure_rx(adapter);
2103                 e1000_alloc_rx_buffers(adapter, &adapter->rx_ring[0]);
2104         }
2105 }
2106
2107 /**
2108  * e1000_set_mac - Change the Ethernet Address of the NIC
2109  * @netdev: network interface device structure
2110  * @p: pointer to an address structure
2111  *
2112  * Returns 0 on success, negative on failure
2113  **/
2114
2115 static int
2116 e1000_set_mac(struct net_device *netdev, void *p)
2117 {
2118         struct e1000_adapter *adapter = netdev_priv(netdev);
2119         struct sockaddr *addr = p;
2120
2121         if(!is_valid_ether_addr(addr->sa_data))
2122                 return -EADDRNOTAVAIL;
2123
2124         /* 82542 2.0 needs to be in reset to write receive address registers */
2125
2126         if(adapter->hw.mac_type == e1000_82542_rev2_0)
2127                 e1000_enter_82542_rst(adapter);
2128
2129         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
2130         memcpy(adapter->hw.mac_addr, addr->sa_data, netdev->addr_len);
2131
2132         e1000_rar_set(&adapter->hw, adapter->hw.mac_addr, 0);
2133
2134         /* With 82571 controllers, LAA may be overwritten (with the default)
2135          * due to controller reset from the other port. */
2136         if (adapter->hw.mac_type == e1000_82571) {
2137                 /* activate the work around */
2138                 adapter->hw.laa_is_present = 1;
2139
2140                 /* Hold a copy of the LAA in RAR[14] This is done so that
2141                  * between the time RAR[0] gets clobbered  and the time it
2142                  * gets fixed (in e1000_watchdog), the actual LAA is in one
2143                  * of the RARs and no incoming packets directed to this port
2144                  * are dropped. Eventaully the LAA will be in RAR[0] and
2145                  * RAR[14] */
2146                 e1000_rar_set(&adapter->hw, adapter->hw.mac_addr,
2147                                         E1000_RAR_ENTRIES - 1);
2148         }
2149
2150         if(adapter->hw.mac_type == e1000_82542_rev2_0)
2151                 e1000_leave_82542_rst(adapter);
2152
2153         return 0;
2154 }
2155
2156 /**
2157  * e1000_set_multi - Multicast and Promiscuous mode set
2158  * @netdev: network interface device structure
2159  *
2160  * The set_multi entry point is called whenever the multicast address
2161  * list or the network interface flags are updated.  This routine is
2162  * responsible for configuring the hardware for proper multicast,
2163  * promiscuous mode, and all-multi behavior.
2164  **/
2165
2166 static void
2167 e1000_set_multi(struct net_device *netdev)
2168 {
2169         struct e1000_adapter *adapter = netdev_priv(netdev);
2170         struct e1000_hw *hw = &adapter->hw;
2171         struct dev_mc_list *mc_ptr;
2172         uint32_t rctl;
2173         uint32_t hash_value;
2174         int i, rar_entries = E1000_RAR_ENTRIES;
2175
2176         /* reserve RAR[14] for LAA over-write work-around */
2177         if (adapter->hw.mac_type == e1000_82571)
2178                 rar_entries--;
2179
2180         /* Check for Promiscuous and All Multicast modes */
2181
2182         rctl = E1000_READ_REG(hw, RCTL);
2183
2184         if(netdev->flags & IFF_PROMISC) {
2185                 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
2186         } else if(netdev->flags & IFF_ALLMULTI) {
2187                 rctl |= E1000_RCTL_MPE;
2188                 rctl &= ~E1000_RCTL_UPE;
2189         } else {
2190                 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE);
2191         }
2192
2193         E1000_WRITE_REG(hw, RCTL, rctl);
2194
2195         /* 82542 2.0 needs to be in reset to write receive address registers */
2196
2197         if(hw->mac_type == e1000_82542_rev2_0)
2198                 e1000_enter_82542_rst(adapter);
2199
2200         /* load the first 14 multicast address into the exact filters 1-14
2201          * RAR 0 is used for the station MAC adddress
2202          * if there are not 14 addresses, go ahead and clear the filters
2203          * -- with 82571 controllers only 0-13 entries are filled here
2204          */
2205         mc_ptr = netdev->mc_list;
2206
2207         for(i = 1; i < rar_entries; i++) {
2208                 if (mc_ptr) {
2209                         e1000_rar_set(hw, mc_ptr->dmi_addr, i);
2210                         mc_ptr = mc_ptr->next;
2211                 } else {
2212                         E1000_WRITE_REG_ARRAY(hw, RA, i << 1, 0);
2213                         E1000_WRITE_REG_ARRAY(hw, RA, (i << 1) + 1, 0);
2214                 }
2215         }
2216
2217         /* clear the old settings from the multicast hash table */
2218
2219         for(i = 0; i < E1000_NUM_MTA_REGISTERS; i++)
2220                 E1000_WRITE_REG_ARRAY(hw, MTA, i, 0);
2221
2222         /* load any remaining addresses into the hash table */
2223
2224         for(; mc_ptr; mc_ptr = mc_ptr->next) {
2225                 hash_value = e1000_hash_mc_addr(hw, mc_ptr->dmi_addr);
2226                 e1000_mta_set(hw, hash_value);
2227         }
2228
2229         if(hw->mac_type == e1000_82542_rev2_0)
2230                 e1000_leave_82542_rst(adapter);
2231 }
2232
2233 /* Need to wait a few seconds after link up to get diagnostic information from
2234  * the phy */
2235
2236 static void
2237 e1000_update_phy_info(unsigned long data)
2238 {
2239         struct e1000_adapter *adapter = (struct e1000_adapter *) data;
2240         e1000_phy_get_info(&adapter->hw, &adapter->phy_info);
2241 }
2242
2243 /**
2244  * e1000_82547_tx_fifo_stall - Timer Call-back
2245  * @data: pointer to adapter cast into an unsigned long
2246  **/
2247
2248 static void
2249 e1000_82547_tx_fifo_stall(unsigned long data)
2250 {
2251         struct e1000_adapter *adapter = (struct e1000_adapter *) data;
2252         struct net_device *netdev = adapter->netdev;
2253         uint32_t tctl;
2254
2255         if(atomic_read(&adapter->tx_fifo_stall)) {
2256                 if((E1000_READ_REG(&adapter->hw, TDT) ==
2257                     E1000_READ_REG(&adapter->hw, TDH)) &&
2258                    (E1000_READ_REG(&adapter->hw, TDFT) ==
2259                     E1000_READ_REG(&adapter->hw, TDFH)) &&
2260                    (E1000_READ_REG(&adapter->hw, TDFTS) ==
2261                     E1000_READ_REG(&adapter->hw, TDFHS))) {
2262                         tctl = E1000_READ_REG(&adapter->hw, TCTL);
2263                         E1000_WRITE_REG(&adapter->hw, TCTL,
2264                                         tctl & ~E1000_TCTL_EN);
2265                         E1000_WRITE_REG(&adapter->hw, TDFT,
2266                                         adapter->tx_head_addr);
2267                         E1000_WRITE_REG(&adapter->hw, TDFH,
2268                                         adapter->tx_head_addr);
2269                         E1000_WRITE_REG(&adapter->hw, TDFTS,
2270                                         adapter->tx_head_addr);
2271                         E1000_WRITE_REG(&adapter->hw, TDFHS,
2272                                         adapter->tx_head_addr);
2273                         E1000_WRITE_REG(&adapter->hw, TCTL, tctl);
2274                         E1000_WRITE_FLUSH(&adapter->hw);
2275
2276                         adapter->tx_fifo_head = 0;
2277                         atomic_set(&adapter->tx_fifo_stall, 0);
2278                         netif_wake_queue(netdev);
2279                 } else {
2280                         mod_timer(&adapter->tx_fifo_stall_timer, jiffies + 1);
2281                 }
2282         }
2283 }
2284
2285 /**
2286  * e1000_watchdog - Timer Call-back
2287  * @data: pointer to adapter cast into an unsigned long
2288  **/
2289 static void
2290 e1000_watchdog(unsigned long data)
2291 {
2292         struct e1000_adapter *adapter = (struct e1000_adapter *) data;
2293
2294         /* Do the rest outside of interrupt context */
2295         schedule_work(&adapter->watchdog_task);
2296 }
2297
2298 static void
2299 e1000_watchdog_task(struct e1000_adapter *adapter)
2300 {
2301         struct net_device *netdev = adapter->netdev;
2302         struct e1000_tx_ring *txdr = adapter->tx_ring;
2303         uint32_t link;
2304
2305         e1000_check_for_link(&adapter->hw);
2306         if (adapter->hw.mac_type == e1000_82573) {
2307                 e1000_enable_tx_pkt_filtering(&adapter->hw);
2308                 if(adapter->mng_vlan_id != adapter->hw.mng_cookie.vlan_id)
2309                         e1000_update_mng_vlan(adapter);
2310         }
2311
2312         if((adapter->hw.media_type == e1000_media_type_internal_serdes) &&
2313            !(E1000_READ_REG(&adapter->hw, TXCW) & E1000_TXCW_ANE))
2314                 link = !adapter->hw.serdes_link_down;
2315         else
2316                 link = E1000_READ_REG(&adapter->hw, STATUS) & E1000_STATUS_LU;
2317
2318         if(link) {
2319                 if(!netif_carrier_ok(netdev)) {
2320                         e1000_get_speed_and_duplex(&adapter->hw,
2321                                                    &adapter->link_speed,
2322                                                    &adapter->link_duplex);
2323
2324                         DPRINTK(LINK, INFO, "NIC Link is Up %d Mbps %s\n",
2325                                adapter->link_speed,
2326                                adapter->link_duplex == FULL_DUPLEX ?
2327                                "Full Duplex" : "Half Duplex");
2328
2329                         netif_carrier_on(netdev);
2330                         netif_wake_queue(netdev);
2331                         mod_timer(&adapter->phy_info_timer, jiffies + 2 * HZ);
2332                         adapter->smartspeed = 0;
2333                 }
2334         } else {
2335                 if(netif_carrier_ok(netdev)) {
2336                         adapter->link_speed = 0;
2337                         adapter->link_duplex = 0;
2338                         DPRINTK(LINK, INFO, "NIC Link is Down\n");
2339                         netif_carrier_off(netdev);
2340                         netif_stop_queue(netdev);
2341                         mod_timer(&adapter->phy_info_timer, jiffies + 2 * HZ);
2342                 }
2343
2344                 e1000_smartspeed(adapter);
2345         }
2346
2347         e1000_update_stats(adapter);
2348
2349         adapter->hw.tx_packet_delta = adapter->stats.tpt - adapter->tpt_old;
2350         adapter->tpt_old = adapter->stats.tpt;
2351         adapter->hw.collision_delta = adapter->stats.colc - adapter->colc_old;
2352         adapter->colc_old = adapter->stats.colc;
2353
2354         adapter->gorcl = adapter->stats.gorcl - adapter->gorcl_old;
2355         adapter->gorcl_old = adapter->stats.gorcl;
2356         adapter->gotcl = adapter->stats.gotcl - adapter->gotcl_old;
2357         adapter->gotcl_old = adapter->stats.gotcl;
2358
2359         e1000_update_adaptive(&adapter->hw);
2360
2361 #ifdef CONFIG_E1000_MQ
2362         txdr = *per_cpu_ptr(adapter->cpu_tx_ring, smp_processor_id());
2363 #endif
2364         if (!netif_carrier_ok(netdev)) {
2365                 if (E1000_DESC_UNUSED(txdr) + 1 < txdr->count) {
2366                         /* We've lost link, so the controller stops DMA,
2367                          * but we've got queued Tx work that's never going
2368                          * to get done, so reset controller to flush Tx.
2369                          * (Do the reset outside of interrupt context). */
2370                         schedule_work(&adapter->tx_timeout_task);
2371                 }
2372         }
2373
2374         /* Dynamic mode for Interrupt Throttle Rate (ITR) */
2375         if(adapter->hw.mac_type >= e1000_82540 && adapter->itr == 1) {
2376                 /* Symmetric Tx/Rx gets a reduced ITR=2000; Total
2377                  * asymmetrical Tx or Rx gets ITR=8000; everyone
2378                  * else is between 2000-8000. */
2379                 uint32_t goc = (adapter->gotcl + adapter->gorcl) / 10000;
2380                 uint32_t dif = (adapter->gotcl > adapter->gorcl ?
2381                         adapter->gotcl - adapter->gorcl :
2382                         adapter->gorcl - adapter->gotcl) / 10000;
2383                 uint32_t itr = goc > 0 ? (dif * 6000 / goc + 2000) : 8000;
2384                 E1000_WRITE_REG(&adapter->hw, ITR, 1000000000 / (itr * 256));
2385         }
2386
2387         /* Cause software interrupt to ensure rx ring is cleaned */
2388         E1000_WRITE_REG(&adapter->hw, ICS, E1000_ICS_RXDMT0);
2389
2390         /* Force detection of hung controller every watchdog period */
2391         adapter->detect_tx_hung = TRUE;
2392
2393         /* With 82571 controllers, LAA may be overwritten due to controller
2394          * reset from the other port. Set the appropriate LAA in RAR[0] */
2395         if (adapter->hw.mac_type == e1000_82571 && adapter->hw.laa_is_present)
2396                 e1000_rar_set(&adapter->hw, adapter->hw.mac_addr, 0);
2397
2398         /* Reset the timer */
2399         mod_timer(&adapter->watchdog_timer, jiffies + 2 * HZ);
2400 }
2401
2402 #define E1000_TX_FLAGS_CSUM             0x00000001
2403 #define E1000_TX_FLAGS_VLAN             0x00000002
2404 #define E1000_TX_FLAGS_TSO              0x00000004
2405 #define E1000_TX_FLAGS_IPV4             0x00000008
2406 #define E1000_TX_FLAGS_VLAN_MASK        0xffff0000
2407 #define E1000_TX_FLAGS_VLAN_SHIFT       16
2408
2409 static inline int
2410 e1000_tso(struct e1000_adapter *adapter, struct e1000_tx_ring *tx_ring,
2411           struct sk_buff *skb)
2412 {
2413 #ifdef NETIF_F_TSO
2414         struct e1000_context_desc *context_desc;
2415         struct e1000_buffer *buffer_info;
2416         unsigned int i;
2417         uint32_t cmd_length = 0;
2418         uint16_t ipcse = 0, tucse, mss;
2419         uint8_t ipcss, ipcso, tucss, tucso, hdr_len;
2420         int err;
2421
2422         if(skb_shinfo(skb)->tso_size) {
2423                 if (skb_header_cloned(skb)) {
2424                         err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2425                         if (err)
2426                                 return err;
2427                 }
2428
2429                 hdr_len = ((skb->h.raw - skb->data) + (skb->h.th->doff << 2));
2430                 mss = skb_shinfo(skb)->tso_size;
2431                 if(skb->protocol == ntohs(ETH_P_IP)) {
2432                         skb->nh.iph->tot_len = 0;
2433                         skb->nh.iph->check = 0;
2434                         skb->h.th->check =
2435                                 ~csum_tcpudp_magic(skb->nh.iph->saddr,
2436                                                    skb->nh.iph->daddr,
2437                                                    0,
2438                                                    IPPROTO_TCP,
2439                                                    0);
2440                         cmd_length = E1000_TXD_CMD_IP;
2441                         ipcse = skb->h.raw - skb->data - 1;
2442 #ifdef NETIF_F_TSO_IPV6
2443                 } else if(skb->protocol == ntohs(ETH_P_IPV6)) {
2444                         skb->nh.ipv6h->payload_len = 0;
2445                         skb->h.th->check =
2446                                 ~csum_ipv6_magic(&skb->nh.ipv6h->saddr,
2447                                                  &skb->nh.ipv6h->daddr,
2448                                                  0,
2449                                                  IPPROTO_TCP,
2450                                                  0);
2451                         ipcse = 0;
2452 #endif
2453                 }
2454                 ipcss = skb->nh.raw - skb->data;
2455                 ipcso = (void *)&(skb->nh.iph->check) - (void *)skb->data;
2456                 tucss = skb->h.raw - skb->data;
2457                 tucso = (void *)&(skb->h.th->check) - (void *)skb->data;
2458                 tucse = 0;
2459
2460                 cmd_length |= (E1000_TXD_CMD_DEXT | E1000_TXD_CMD_TSE |
2461                                E1000_TXD_CMD_TCP | (skb->len - (hdr_len)));
2462
2463                 i = tx_ring->next_to_use;
2464                 context_desc = E1000_CONTEXT_DESC(*tx_ring, i);
2465                 buffer_info = &tx_ring->buffer_info[i];
2466
2467                 context_desc->lower_setup.ip_fields.ipcss  = ipcss;
2468                 context_desc->lower_setup.ip_fields.ipcso  = ipcso;
2469                 context_desc->lower_setup.ip_fields.ipcse  = cpu_to_le16(ipcse);
2470                 context_desc->upper_setup.tcp_fields.tucss = tucss;
2471                 context_desc->upper_setup.tcp_fields.tucso = tucso;
2472                 context_desc->upper_setup.tcp_fields.tucse = cpu_to_le16(tucse);
2473                 context_desc->tcp_seg_setup.fields.mss     = cpu_to_le16(mss);
2474                 context_desc->tcp_seg_setup.fields.hdr_len = hdr_len;
2475                 context_desc->cmd_and_length = cpu_to_le32(cmd_length);
2476
2477                 buffer_info->time_stamp = jiffies;
2478
2479                 if (++i == tx_ring->count) i = 0;
2480                 tx_ring->next_to_use = i;
2481
2482                 return 1;
2483         }
2484 #endif
2485
2486         return 0;
2487 }
2488
2489 static inline boolean_t
2490 e1000_tx_csum(struct e1000_adapter *adapter, struct e1000_tx_ring *tx_ring,
2491               struct sk_buff *skb)
2492 {
2493         struct e1000_context_desc *context_desc;
2494         struct e1000_buffer *buffer_info;
2495         unsigned int i;
2496         uint8_t css;
2497
2498         if(likely(skb->ip_summed == CHECKSUM_HW)) {
2499                 css = skb->h.raw - skb->data;
2500
2501                 i = tx_ring->next_to_use;
2502                 buffer_info = &tx_ring->buffer_info[i];
2503                 context_desc = E1000_CONTEXT_DESC(*tx_ring, i);
2504
2505                 context_desc->upper_setup.tcp_fields.tucss = css;
2506                 context_desc->upper_setup.tcp_fields.tucso = css + skb->csum;
2507                 context_desc->upper_setup.tcp_fields.tucse = 0;
2508                 context_desc->tcp_seg_setup.data = 0;
2509                 context_desc->cmd_and_length = cpu_to_le32(E1000_TXD_CMD_DEXT);
2510
2511                 buffer_info->time_stamp = jiffies;
2512
2513                 if (unlikely(++i == tx_ring->count)) i = 0;
2514                 tx_ring->next_to_use = i;
2515
2516                 return TRUE;
2517         }
2518
2519         return FALSE;
2520 }
2521
2522 #define E1000_MAX_TXD_PWR       12
2523 #define E1000_MAX_DATA_PER_TXD  (1<<E1000_MAX_TXD_PWR)
2524
2525 static inline int
2526 e1000_tx_map(struct e1000_adapter *adapter, struct e1000_tx_ring *tx_ring,
2527              struct sk_buff *skb, unsigned int first, unsigned int max_per_txd,
2528              unsigned int nr_frags, unsigned int mss)
2529 {
2530         struct e1000_buffer *buffer_info;
2531         unsigned int len = skb->len;
2532         unsigned int offset = 0, size, count = 0, i;
2533         unsigned int f;
2534         len -= skb->data_len;
2535
2536         i = tx_ring->next_to_use;
2537
2538         while(len) {
2539                 buffer_info = &tx_ring->buffer_info[i];
2540                 size = min(len, max_per_txd);
2541 #ifdef NETIF_F_TSO
2542                 /* Workaround for Controller erratum --
2543                  * descriptor for non-tso packet in a linear SKB that follows a
2544                  * tso gets written back prematurely before the data is fully
2545                  * DMAd to the controller */
2546                 if (!skb->data_len && tx_ring->last_tx_tso &&
2547                                 !skb_shinfo(skb)->tso_size) {
2548                         tx_ring->last_tx_tso = 0;
2549                         size -= 4;
2550                 }
2551
2552                 /* Workaround for premature desc write-backs
2553                  * in TSO mode.  Append 4-byte sentinel desc */
2554                 if(unlikely(mss && !nr_frags && size == len && size > 8))
2555                         size -= 4;
2556 #endif
2557                 /* work-around for errata 10 and it applies
2558                  * to all controllers in PCI-X mode
2559                  * The fix is to make sure that the first descriptor of a
2560                  * packet is smaller than 2048 - 16 - 16 (or 2016) bytes
2561                  */
2562                 if(unlikely((adapter->hw.bus_type == e1000_bus_type_pcix) &&
2563                                 (size > 2015) && count == 0))
2564                         size = 2015;
2565
2566                 /* Workaround for potential 82544 hang in PCI-X.  Avoid
2567                  * terminating buffers within evenly-aligned dwords. */
2568                 if(unlikely(adapter->pcix_82544 &&
2569                    !((unsigned long)(skb->data + offset + size - 1) & 4) &&
2570                    size > 4))
2571                         size -= 4;
2572
2573                 buffer_info->length = size;
2574                 buffer_info->dma =
2575                         pci_map_single(adapter->pdev,
2576                                 skb->data + offset,
2577                                 size,
2578                                 PCI_DMA_TODEVICE);
2579                 buffer_info->time_stamp = jiffies;
2580
2581                 len -= size;
2582                 offset += size;
2583                 count++;
2584                 if(unlikely(++i == tx_ring->count)) i = 0;
2585         }
2586
2587         for(f = 0; f < nr_frags; f++) {
2588                 struct skb_frag_struct *frag;
2589
2590                 frag = &skb_shinfo(skb)->frags[f];
2591                 len = frag->size;
2592                 offset = frag->page_offset;
2593
2594                 while(len) {
2595                         buffer_info = &tx_ring->buffer_info[i];
2596                         size = min(len, max_per_txd);
2597 #ifdef NETIF_F_TSO
2598                         /* Workaround for premature desc write-backs
2599                          * in TSO mode.  Append 4-byte sentinel desc */
2600                         if(unlikely(mss && f == (nr_frags-1) && size == len && size > 8))
2601                                 size -= 4;
2602 #endif
2603                         /* Workaround for potential 82544 hang in PCI-X.
2604                          * Avoid terminating buffers within evenly-aligned
2605                          * dwords. */
2606                         if(unlikely(adapter->pcix_82544 &&
2607                            !((unsigned long)(frag->page+offset+size-1) & 4) &&
2608                            size > 4))
2609                                 size -= 4;
2610
2611                         buffer_info->length = size;
2612                         buffer_info->dma =
2613                                 pci_map_page(adapter->pdev,
2614                                         frag->page,
2615                                         offset,
2616                                         size,
2617                                         PCI_DMA_TODEVICE);
2618                         buffer_info->time_stamp = jiffies;
2619
2620                         len -= size;
2621                         offset += size;
2622                         count++;
2623                         if(unlikely(++i == tx_ring->count)) i = 0;
2624                 }
2625         }
2626
2627         i = (i == 0) ? tx_ring->count - 1 : i - 1;
2628         tx_ring->buffer_info[i].skb = skb;
2629         tx_ring->buffer_info[first].next_to_watch = i;
2630
2631         return count;
2632 }
2633
2634 static inline void
2635 e1000_tx_queue(struct e1000_adapter *adapter, struct e1000_tx_ring *tx_ring,
2636                int tx_flags, int count)
2637 {
2638         struct e1000_tx_desc *tx_desc = NULL;
2639         struct e1000_buffer *buffer_info;
2640         uint32_t txd_upper = 0, txd_lower = E1000_TXD_CMD_IFCS;
2641         unsigned int i;
2642
2643         if(likely(tx_flags & E1000_TX_FLAGS_TSO)) {
2644                 txd_lower |= E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D |
2645                              E1000_TXD_CMD_TSE;
2646                 txd_upper |= E1000_TXD_POPTS_TXSM << 8;
2647
2648                 if(likely(tx_flags & E1000_TX_FLAGS_IPV4))
2649                         txd_upper |= E1000_TXD_POPTS_IXSM << 8;
2650         }
2651
2652         if(likely(tx_flags & E1000_TX_FLAGS_CSUM)) {
2653                 txd_lower |= E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
2654                 txd_upper |= E1000_TXD_POPTS_TXSM << 8;
2655         }
2656
2657         if(unlikely(tx_flags & E1000_TX_FLAGS_VLAN)) {
2658                 txd_lower |= E1000_TXD_CMD_VLE;
2659                 txd_upper |= (tx_flags & E1000_TX_FLAGS_VLAN_MASK);
2660         }
2661
2662         i = tx_ring->next_to_use;
2663
2664         while(count--) {
2665                 buffer_info = &tx_ring->buffer_info[i];
2666                 tx_desc = E1000_TX_DESC(*tx_ring, i);
2667                 tx_desc->buffer_addr = cpu_to_le64(buffer_info->dma);
2668                 tx_desc->lower.data =
2669                         cpu_to_le32(txd_lower | buffer_info->length);
2670                 tx_desc->upper.data = cpu_to_le32(txd_upper);
2671                 if(unlikely(++i == tx_ring->count)) i = 0;
2672         }
2673
2674         tx_desc->lower.data |= cpu_to_le32(adapter->txd_cmd);
2675
2676         /* Force memory writes to complete before letting h/w
2677          * know there are new descriptors to fetch.  (Only
2678          * applicable for weak-ordered memory model archs,
2679          * such as IA-64). */
2680         wmb();
2681
2682         tx_ring->next_to_use = i;
2683         writel(i, adapter->hw.hw_addr + tx_ring->tdt);
2684 }
2685
2686 /**
2687  * 82547 workaround to avoid controller hang in half-duplex environment.
2688  * The workaround is to avoid queuing a large packet that would span
2689  * the internal Tx FIFO ring boundary by notifying the stack to resend
2690  * the packet at a later time.  This gives the Tx FIFO an opportunity to
2691  * flush all packets.  When that occurs, we reset the Tx FIFO pointers
2692  * to the beginning of the Tx FIFO.
2693  **/
2694
2695 #define E1000_FIFO_HDR                  0x10
2696 #define E1000_82547_PAD_LEN             0x3E0
2697
2698 static inline int
2699 e1000_82547_fifo_workaround(struct e1000_adapter *adapter, struct sk_buff *skb)
2700 {
2701         uint32_t fifo_space = adapter->tx_fifo_size - adapter->tx_fifo_head;
2702         uint32_t skb_fifo_len = skb->len + E1000_FIFO_HDR;
2703
2704         E1000_ROUNDUP(skb_fifo_len, E1000_FIFO_HDR);
2705
2706         if(adapter->link_duplex != HALF_DUPLEX)
2707                 goto no_fifo_stall_required;
2708
2709         if(atomic_read(&adapter->tx_fifo_stall))
2710                 return 1;
2711
2712         if(skb_fifo_len >= (E1000_82547_PAD_LEN + fifo_space)) {
2713                 atomic_set(&adapter->tx_fifo_stall, 1);
2714                 return 1;
2715         }
2716
2717 no_fifo_stall_required:
2718         adapter->tx_fifo_head += skb_fifo_len;
2719         if(adapter->tx_fifo_head >= adapter->tx_fifo_size)
2720                 adapter->tx_fifo_head -= adapter->tx_fifo_size;
2721         return 0;
2722 }
2723
2724 #define MINIMUM_DHCP_PACKET_SIZE 282
2725 static inline int
2726 e1000_transfer_dhcp_info(struct e1000_adapter *adapter, struct sk_buff *skb)
2727 {
2728         struct e1000_hw *hw =  &adapter->hw;
2729         uint16_t length, offset;
2730         if(vlan_tx_tag_present(skb)) {
2731                 if(!((vlan_tx_tag_get(skb) == adapter->hw.mng_cookie.vlan_id) &&
2732                         ( adapter->hw.mng_cookie.status &
2733                           E1000_MNG_DHCP_COOKIE_STATUS_VLAN_SUPPORT)) )
2734                         return 0;
2735         }
2736         if ((skb->len > MINIMUM_DHCP_PACKET_SIZE) && (!skb->protocol)) {
2737                 struct ethhdr *eth = (struct ethhdr *) skb->data;
2738                 if((htons(ETH_P_IP) == eth->h_proto)) {
2739                         const struct iphdr *ip =
2740                                 (struct iphdr *)((uint8_t *)skb->data+14);
2741                         if(IPPROTO_UDP == ip->protocol) {
2742                                 struct udphdr *udp =
2743                                         (struct udphdr *)((uint8_t *)ip +
2744                                                 (ip->ihl << 2));
2745                                 if(ntohs(udp->dest) == 67) {
2746                                         offset = (uint8_t *)udp + 8 - skb->data;
2747                                         length = skb->len - offset;
2748
2749                                         return e1000_mng_write_dhcp_info(hw,
2750                                                         (uint8_t *)udp + 8,
2751                                                         length);
2752                                 }
2753                         }
2754                 }
2755         }
2756         return 0;
2757 }
2758
2759 #define TXD_USE_COUNT(S, X) (((S) >> (X)) + 1 )
2760 static int
2761 e1000_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
2762 {
2763         struct e1000_adapter *adapter = netdev_priv(netdev);
2764         struct e1000_tx_ring *tx_ring;
2765         unsigned int first, max_per_txd = E1000_MAX_DATA_PER_TXD;
2766         unsigned int max_txd_pwr = E1000_MAX_TXD_PWR;
2767         unsigned int tx_flags = 0;
2768         unsigned int len = skb->len;
2769         unsigned long flags;
2770         unsigned int nr_frags = 0;
2771         unsigned int mss = 0;
2772         int count = 0;
2773         int tso;
2774         unsigned int f;
2775         len -= skb->data_len;
2776
2777 #ifdef CONFIG_E1000_MQ
2778         tx_ring = *per_cpu_ptr(adapter->cpu_tx_ring, smp_processor_id());
2779 #else
2780         tx_ring = adapter->tx_ring;
2781 #endif
2782
2783         if (unlikely(skb->len <= 0)) {
2784                 dev_kfree_skb_any(skb);
2785                 return NETDEV_TX_OK;
2786         }
2787
2788 #ifdef NETIF_F_TSO
2789         mss = skb_shinfo(skb)->tso_size;
2790         /* The controller does a simple calculation to
2791          * make sure there is enough room in the FIFO before
2792          * initiating the DMA for each buffer.  The calc is:
2793          * 4 = ceil(buffer len/mss).  To make sure we don't
2794          * overrun the FIFO, adjust the max buffer len if mss
2795          * drops. */
2796         if(mss) {
2797                 uint8_t hdr_len;
2798                 max_per_txd = min(mss << 2, max_per_txd);
2799                 max_txd_pwr = fls(max_per_txd) - 1;
2800
2801         /* TSO Workaround for 82571/2 Controllers -- if skb->data
2802          * points to just header, pull a few bytes of payload from
2803          * frags into skb->data */
2804                 hdr_len = ((skb->h.raw - skb->data) + (skb->h.th->doff << 2));
2805                 if (skb->data_len && (hdr_len == (skb->len - skb->data_len)) &&
2806                         (adapter->hw.mac_type == e1000_82571 ||
2807                         adapter->hw.mac_type == e1000_82572)) {
2808                         len = skb->len - skb->data_len;
2809                 }
2810         }
2811
2812         if((mss) || (skb->ip_summed == CHECKSUM_HW))
2813         /* reserve a descriptor for the offload context */
2814                 count++;
2815         count++;
2816 #else
2817         if(skb->ip_summed == CHECKSUM_HW)
2818                 count++;
2819 #endif
2820
2821 #ifdef NETIF_F_TSO
2822         /* Controller Erratum workaround */
2823         if (!skb->data_len && tx_ring->last_tx_tso &&
2824                 !skb_shinfo(skb)->tso_size)
2825                 count++;
2826 #endif
2827
2828         count += TXD_USE_COUNT(len, max_txd_pwr);
2829
2830         if(adapter->pcix_82544)
2831                 count++;
2832
2833         /* work-around for errata 10 and it applies to all controllers
2834          * in PCI-X mode, so add one more descriptor to the count
2835          */
2836         if(unlikely((adapter->hw.bus_type == e1000_bus_type_pcix) &&
2837                         (len > 2015)))
2838                 count++;
2839
2840         nr_frags = skb_shinfo(skb)->nr_frags;
2841         for(f = 0; f < nr_frags; f++)
2842                 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size,
2843                                        max_txd_pwr);
2844         if(adapter->pcix_82544)
2845                 count += nr_frags;
2846
2847                 unsigned int pull_size;
2848                 pull_size = min((unsigned int)4, skb->data_len);
2849                 if (!__pskb_pull_tail(skb, pull_size)) {
2850                         printk(KERN_ERR "__pskb_pull_tail failed.\n");
2851                         dev_kfree_skb_any(skb);
2852                         return -EFAULT;
2853                 }
2854
2855         if(adapter->hw.tx_pkt_filtering && (adapter->hw.mac_type == e1000_82573) )
2856                 e1000_transfer_dhcp_info(adapter, skb);
2857
2858         local_irq_save(flags);
2859         if (!spin_trylock(&tx_ring->tx_lock)) {
2860                 /* Collision - tell upper layer to requeue */
2861                 local_irq_restore(flags);
2862                 return NETDEV_TX_LOCKED;
2863         }
2864
2865         /* need: count + 2 desc gap to keep tail from touching
2866          * head, otherwise try next time */
2867         if (unlikely(E1000_DESC_UNUSED(tx_ring) < count + 2)) {
2868                 netif_stop_queue(netdev);
2869                 spin_unlock_irqrestore(&tx_ring->tx_lock, flags);
2870                 return NETDEV_TX_BUSY;
2871         }
2872
2873         if(unlikely(adapter->hw.mac_type == e1000_82547)) {
2874                 if(unlikely(e1000_82547_fifo_workaround(adapter, skb))) {
2875                         netif_stop_queue(netdev);
2876                         mod_timer(&adapter->tx_fifo_stall_timer, jiffies);
2877                         spin_unlock_irqrestore(&tx_ring->tx_lock, flags);
2878                         return NETDEV_TX_BUSY;
2879                 }
2880         }
2881
2882         if(unlikely(adapter->vlgrp && vlan_tx_tag_present(skb))) {
2883                 tx_flags |= E1000_TX_FLAGS_VLAN;
2884                 tx_flags |= (vlan_tx_tag_get(skb) << E1000_TX_FLAGS_VLAN_SHIFT);
2885         }
2886
2887         first = tx_ring->next_to_use;
2888
2889         tso = e1000_tso(adapter, tx_ring, skb);
2890         if (tso < 0) {
2891                 dev_kfree_skb_any(skb);
2892                 spin_unlock_irqrestore(&tx_ring->tx_lock, flags);
2893                 return NETDEV_TX_OK;
2894         }
2895
2896         if (likely(tso)) {
2897                 tx_ring->last_tx_tso = 1;
2898                 tx_flags |= E1000_TX_FLAGS_TSO;
2899         } else if (likely(e1000_tx_csum(adapter, tx_ring, skb)))
2900                 tx_flags |= E1000_TX_FLAGS_CSUM;
2901
2902         /* Old method was to assume IPv4 packet by default if TSO was enabled.
2903          * 82571 hardware supports TSO capabilities for IPv6 as well...
2904          * no longer assume, we must. */
2905         if (likely(skb->protocol == ntohs(ETH_P_IP)))
2906                 tx_flags |= E1000_TX_FLAGS_IPV4;
2907
2908         e1000_tx_queue(adapter, tx_ring, tx_flags,
2909                        e1000_tx_map(adapter, tx_ring, skb, first,
2910                                     max_per_txd, nr_frags, mss));
2911
2912         netdev->trans_start = jiffies;
2913
2914         /* Make sure there is space in the ring for the next send. */
2915         if (unlikely(E1000_DESC_UNUSED(tx_ring) < MAX_SKB_FRAGS + 2))
2916                 netif_stop_queue(netdev);
2917
2918         spin_unlock_irqrestore(&tx_ring->tx_lock, flags);
2919         return NETDEV_TX_OK;
2920 }
2921
2922 /**
2923  * e1000_tx_timeout - Respond to a Tx Hang
2924  * @netdev: network interface device structure
2925  **/
2926
2927 static void
2928 e1000_tx_timeout(struct net_device *netdev)
2929 {
2930         struct e1000_adapter *adapter = netdev_priv(netdev);
2931
2932         /* Do the reset outside of interrupt context */
2933         schedule_work(&adapter->tx_timeout_task);
2934 }
2935
2936 static void
2937 e1000_tx_timeout_task(struct net_device *netdev)
2938 {
2939         struct e1000_adapter *adapter = netdev_priv(netdev);
2940
2941         adapter->tx_timeout_count++;
2942         e1000_down(adapter);
2943         e1000_up(adapter);
2944 }
2945
2946 /**
2947  * e1000_get_stats - Get System Network Statistics
2948  * @netdev: network interface device structure
2949  *
2950  * Returns the address of the device statistics structure.
2951  * The statistics are actually updated from the timer callback.
2952  **/
2953
2954 static struct net_device_stats *
2955 e1000_get_stats(struct net_device *netdev)
2956 {
2957         struct e1000_adapter *adapter = netdev_priv(netdev);
2958
2959         /* only return the current stats */
2960         return &adapter->net_stats;
2961 }
2962
2963 /**
2964  * e1000_change_mtu - Change the Maximum Transfer Unit
2965  * @netdev: network interface device structure
2966  * @new_mtu: new value for maximum frame size
2967  *
2968  * Returns 0 on success, negative on failure
2969  **/
2970
2971 static int
2972 e1000_change_mtu(struct net_device *netdev, int new_mtu)
2973 {
2974         struct e1000_adapter *adapter = netdev_priv(netdev);
2975         int max_frame = new_mtu + ENET_HEADER_SIZE + ETHERNET_FCS_SIZE;
2976
2977         if((max_frame < MINIMUM_ETHERNET_FRAME_SIZE) ||
2978                 (max_frame > MAX_JUMBO_FRAME_SIZE)) {
2979                         DPRINTK(PROBE, ERR, "Invalid MTU setting\n");
2980                         return -EINVAL;
2981         }
2982
2983 #define MAX_STD_JUMBO_FRAME_SIZE 9234
2984         /* might want this to be bigger enum check... */
2985         /* 82571 controllers limit jumbo frame size to 10500 bytes */
2986         if ((adapter->hw.mac_type == e1000_82571 ||
2987              adapter->hw.mac_type == e1000_82572) &&
2988             max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
2989                 DPRINTK(PROBE, ERR, "MTU > 9216 bytes not supported "
2990                                     "on 82571 and 82572 controllers.\n");
2991                 return -EINVAL;
2992         }
2993
2994         if(adapter->hw.mac_type == e1000_82573 &&
2995             max_frame > MAXIMUM_ETHERNET_FRAME_SIZE) {
2996                 DPRINTK(PROBE, ERR, "Jumbo Frames not supported "
2997                                     "on 82573\n");
2998                 return -EINVAL;
2999         }
3000
3001         if(adapter->hw.mac_type > e1000_82547_rev_2) {
3002                 adapter->rx_buffer_len = max_frame;
3003                 E1000_ROUNDUP(adapter->rx_buffer_len, 1024);
3004         } else {
3005                 if(unlikely((adapter->hw.mac_type < e1000_82543) &&
3006                    (max_frame > MAXIMUM_ETHERNET_FRAME_SIZE))) {
3007                         DPRINTK(PROBE, ERR, "Jumbo Frames not supported "
3008                                             "on 82542\n");
3009                         return -EINVAL;
3010
3011                 } else {
3012                         if(max_frame <= E1000_RXBUFFER_2048) {
3013                                 adapter->rx_buffer_len = E1000_RXBUFFER_2048;
3014                         } else if(max_frame <= E1000_RXBUFFER_4096) {
3015                                 adapter->rx_buffer_len = E1000_RXBUFFER_4096;
3016                         } else if(max_frame <= E1000_RXBUFFER_8192) {
3017                                 adapter->rx_buffer_len = E1000_RXBUFFER_8192;
3018                         } else if(max_frame <= E1000_RXBUFFER_16384) {
3019                                 adapter->rx_buffer_len = E1000_RXBUFFER_16384;
3020                         }
3021                 }
3022         }
3023
3024         netdev->mtu = new_mtu;
3025
3026         if(netif_running(netdev)) {
3027                 e1000_down(adapter);
3028                 e1000_up(adapter);
3029         }
3030
3031         adapter->hw.max_frame_size = max_frame;
3032
3033         return 0;
3034 }
3035
3036 /**
3037  * e1000_update_stats - Update the board statistics counters
3038  * @adapter: board private structure
3039  **/
3040
3041 void
3042 e1000_update_stats(struct e1000_adapter *adapter)
3043 {
3044         struct e1000_hw *hw = &adapter->hw;
3045         unsigned long flags;
3046         uint16_t phy_tmp;
3047
3048 #define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
3049
3050         spin_lock_irqsave(&adapter->stats_lock, flags);
3051
3052         /* these counters are modified from e1000_adjust_tbi_stats,
3053          * called from the interrupt context, so they must only
3054          * be written while holding adapter->stats_lock
3055          */
3056
3057         adapter->stats.crcerrs += E1000_READ_REG(hw, CRCERRS);
3058         adapter->stats.gprc += E1000_READ_REG(hw, GPRC);
3059         adapter->stats.gorcl += E1000_READ_REG(hw, GORCL);
3060         adapter->stats.gorch += E1000_READ_REG(hw, GORCH);
3061         adapter->stats.bprc += E1000_READ_REG(hw, BPRC);
3062         adapter->stats.mprc += E1000_READ_REG(hw, MPRC);
3063         adapter->stats.roc += E1000_READ_REG(hw, ROC);
3064         adapter->stats.prc64 += E1000_READ_REG(hw, PRC64);
3065         adapter->stats.prc127 += E1000_READ_REG(hw, PRC127);
3066         adapter->stats.prc255 += E1000_READ_REG(hw, PRC255);
3067         adapter->stats.prc511 += E1000_READ_REG(hw, PRC511);
3068         adapter->stats.prc1023 += E1000_READ_REG(hw, PRC1023);
3069         adapter->stats.prc1522 += E1000_READ_REG(hw, PRC1522);
3070
3071         adapter->stats.symerrs += E1000_READ_REG(hw, SYMERRS);
3072         adapter->stats.mpc += E1000_READ_REG(hw, MPC);
3073         adapter->stats.scc += E1000_READ_REG(hw, SCC);
3074         adapter->stats.ecol += E1000_READ_REG(hw, ECOL);
3075         adapter->stats.mcc += E1000_READ_REG(hw, MCC);
3076         adapter->stats.latecol += E1000_READ_REG(hw, LATECOL);
3077         adapter->stats.dc += E1000_READ_REG(hw, DC);
3078         adapter->stats.sec += E1000_READ_REG(hw, SEC);
3079         adapter->stats.rlec += E1000_READ_REG(hw, RLEC);
3080         adapter->stats.xonrxc += E1000_READ_REG(hw, XONRXC);
3081         adapter->stats.xontxc += E1000_READ_REG(hw, XONTXC);
3082         adapter->stats.xoffrxc += E1000_READ_REG(hw, XOFFRXC);
3083         adapter->stats.xofftxc += E1000_READ_REG(hw, XOFFTXC);
3084         adapter->stats.fcruc += E1000_READ_REG(hw, FCRUC);
3085         adapter->stats.gptc += E1000_READ_REG(hw, GPTC);
3086         adapter->stats.gotcl += E1000_READ_REG(hw, GOTCL);
3087         adapter->stats.gotch += E1000_READ_REG(hw, GOTCH);
3088         adapter->stats.rnbc += E1000_READ_REG(hw, RNBC);
3089         adapter->stats.ruc += E1000_READ_REG(hw, RUC);
3090         adapter->stats.rfc += E1000_READ_REG(hw, RFC);
3091         adapter->stats.rjc += E1000_READ_REG(hw, RJC);
3092         adapter->stats.torl += E1000_READ_REG(hw, TORL);
3093         adapter->stats.torh += E1000_READ_REG(hw, TORH);
3094         adapter->stats.totl += E1000_READ_REG(hw, TOTL);
3095         adapter->stats.toth += E1000_READ_REG(hw, TOTH);
3096         adapter->stats.tpr += E1000_READ_REG(hw, TPR);
3097         adapter->stats.ptc64 += E1000_READ_REG(hw, PTC64);
3098         adapter->stats.ptc127 += E1000_READ_REG(hw, PTC127);
3099         adapter->stats.ptc255 += E1000_READ_REG(hw, PTC255);
3100         adapter->stats.ptc511 += E1000_READ_REG(hw, PTC511);
3101         adapter->stats.ptc1023 += E1000_READ_REG(hw, PTC1023);
3102         adapter->stats.ptc1522 += E1000_READ_REG(hw, PTC1522);
3103         adapter->stats.mptc += E1000_READ_REG(hw, MPTC);
3104         adapter->stats.bptc += E1000_READ_REG(hw, BPTC);
3105
3106         /* used for adaptive IFS */
3107
3108         hw->tx_packet_delta = E1000_READ_REG(hw, TPT);
3109         adapter->stats.tpt += hw->tx_packet_delta;
3110         hw->collision_delta = E1000_READ_REG(hw, COLC);
3111         adapter->stats.colc += hw->collision_delta;
3112
3113         if(hw->mac_type >= e1000_82543) {
3114                 adapter->stats.algnerrc += E1000_READ_REG(hw, ALGNERRC);
3115                 adapter->stats.rxerrc += E1000_READ_REG(hw, RXERRC);
3116                 adapter->stats.tncrs += E1000_READ_REG(hw, TNCRS);
3117                 adapter->stats.cexterr += E1000_READ_REG(hw, CEXTERR);
3118                 adapter->stats.tsctc += E1000_READ_REG(hw, TSCTC);
3119                 adapter->stats.tsctfc += E1000_READ_REG(hw, TSCTFC);
3120         }
3121         if(hw->mac_type > e1000_82547_rev_2) {
3122                 adapter->stats.iac += E1000_READ_REG(hw, IAC);
3123                 adapter->stats.icrxoc += E1000_READ_REG(hw, ICRXOC);
3124                 adapter->stats.icrxptc += E1000_READ_REG(hw, ICRXPTC);
3125                 adapter->stats.icrxatc += E1000_READ_REG(hw, ICRXATC);
3126                 adapter->stats.ictxptc += E1000_READ_REG(hw, ICTXPTC);
3127                 adapter->stats.ictxatc += E1000_READ_REG(hw, ICTXATC);
3128                 adapter->stats.ictxqec += E1000_READ_REG(hw, ICTXQEC);
3129                 adapter->stats.ictxqmtc += E1000_READ_REG(hw, ICTXQMTC);
3130                 adapter->stats.icrxdmtc += E1000_READ_REG(hw, ICRXDMTC);
3131         }
3132
3133         /* Fill out the OS statistics structure */
3134
3135         adapter->net_stats.rx_packets = adapter->stats.gprc;
3136         adapter->net_stats.tx_packets = adapter->stats.gptc;
3137         adapter->net_stats.rx_bytes = adapter->stats.gorcl;
3138         adapter->net_stats.tx_bytes = adapter->stats.gotcl;
3139         adapter->net_stats.multicast = adapter->stats.mprc;
3140         adapter->net_stats.collisions = adapter->stats.colc;
3141
3142         /* Rx Errors */
3143
3144         adapter->net_stats.rx_errors = adapter->stats.rxerrc +
3145                 adapter->stats.crcerrs + adapter->stats.algnerrc +
3146                 adapter->stats.rlec + adapter->stats.cexterr;
3147         adapter->net_stats.rx_dropped = 0;
3148         adapter->net_stats.rx_length_errors = adapter->stats.rlec;
3149         adapter->net_stats.rx_crc_errors = adapter->stats.crcerrs;
3150         adapter->net_stats.rx_frame_errors = adapter->stats.algnerrc;
3151         adapter->net_stats.rx_missed_errors = adapter->stats.mpc;
3152
3153         /* Tx Errors */
3154
3155         adapter->net_stats.tx_errors = adapter->stats.ecol +
3156                                        adapter->stats.latecol;
3157         adapter->net_stats.tx_aborted_errors = adapter->stats.ecol;
3158         adapter->net_stats.tx_window_errors = adapter->stats.latecol;
3159         adapter->net_stats.tx_carrier_errors = adapter->stats.tncrs;
3160
3161         /* Tx Dropped needs to be maintained elsewhere */
3162
3163         /* Phy Stats */
3164
3165         if(hw->media_type == e1000_media_type_copper) {
3166                 if((adapter->link_speed == SPEED_1000) &&
3167                    (!e1000_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
3168                         phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
3169                         adapter->phy_stats.idle_errors += phy_tmp;
3170                 }
3171
3172                 if((hw->mac_type <= e1000_82546) &&
3173                    (hw->phy_type == e1000_phy_m88) &&
3174                    !e1000_read_phy_reg(hw, M88E1000_RX_ERR_CNTR, &phy_tmp))
3175                         adapter->phy_stats.receive_errors += phy_tmp;
3176         }
3177
3178         spin_unlock_irqrestore(&adapter->stats_lock, flags);
3179 }
3180
3181 #ifdef CONFIG_E1000_MQ
3182 void
3183 e1000_rx_schedule(void *data)
3184 {
3185         struct net_device *poll_dev, *netdev = data;
3186         struct e1000_adapter *adapter = netdev->priv;
3187         int this_cpu = get_cpu();
3188
3189         poll_dev = *per_cpu_ptr(adapter->cpu_netdev, this_cpu);
3190         if (poll_dev == NULL) {
3191                 put_cpu();
3192                 return;
3193         }
3194
3195         if (likely(netif_rx_schedule_prep(poll_dev)))
3196                 __netif_rx_schedule(poll_dev);
3197         else
3198                 e1000_irq_enable(adapter);
3199
3200         put_cpu();
3201 }
3202 #endif
3203
3204 /**
3205  * e1000_intr - Interrupt Handler
3206  * @irq: interrupt number
3207  * @data: pointer to a network interface device structure
3208  * @pt_regs: CPU registers structure
3209  **/
3210
3211 static irqreturn_t
3212 e1000_intr(int irq, void *data, struct pt_regs *regs)
3213 {
3214         struct net_device *netdev = data;
3215         struct e1000_adapter *adapter = netdev_priv(netdev);
3216         struct e1000_hw *hw = &adapter->hw;
3217         uint32_t icr = E1000_READ_REG(hw, ICR);
3218 #if defined(CONFIG_E1000_NAPI) && defined(CONFIG_E1000_MQ) || !defined(CONFIG_E1000_NAPI)
3219         int i;
3220 #endif
3221
3222         if(unlikely(!icr))
3223                 return IRQ_NONE;  /* Not our interrupt */
3224
3225         if(unlikely(icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC))) {
3226                 hw->get_link_status = 1;
3227                 mod_timer(&adapter->watchdog_timer, jiffies);
3228         }
3229
3230 #ifdef CONFIG_E1000_NAPI
3231         atomic_inc(&adapter->irq_sem);
3232         E1000_WRITE_REG(hw, IMC, ~0);
3233         E1000_WRITE_FLUSH(hw);
3234 #ifdef CONFIG_E1000_MQ
3235         if (atomic_read(&adapter->rx_sched_call_data.count) == 0) {
3236                 /* We must setup the cpumask once count == 0 since
3237                  * each cpu bit is cleared when the work is done. */
3238                 adapter->rx_sched_call_data.cpumask = adapter->cpumask;
3239                 atomic_add(adapter->num_rx_queues - 1, &adapter->irq_sem);
3240                 atomic_set(&adapter->rx_sched_call_data.count,
3241                            adapter->num_rx_queues);
3242                 smp_call_async_mask(&adapter->rx_sched_call_data);
3243         } else {
3244                 printk("call_data.count == %u\n", atomic_read(&adapter->rx_sched_call_data.count));
3245         }
3246 #else /* if !CONFIG_E1000_MQ */
3247         if (likely(netif_rx_schedule_prep(&adapter->polling_netdev[0])))
3248                 __netif_rx_schedule(&adapter->polling_netdev[0]);
3249         else
3250                 e1000_irq_enable(adapter);
3251 #endif /* CONFIG_E1000_MQ */
3252
3253 #else /* if !CONFIG_E1000_NAPI */
3254         /* Writing IMC and IMS is needed for 82547.
3255            Due to Hub Link bus being occupied, an interrupt
3256            de-assertion message is not able to be sent.
3257            When an interrupt assertion message is generated later,
3258            two messages are re-ordered and sent out.
3259            That causes APIC to think 82547 is in de-assertion
3260            state, while 82547 is in assertion state, resulting
3261            in dead lock. Writing IMC forces 82547 into
3262            de-assertion state.
3263         */
3264         if(hw->mac_type == e1000_82547 || hw->mac_type == e1000_82547_rev_2){
3265                 atomic_inc(&adapter->irq_sem);
3266                 E1000_WRITE_REG(hw, IMC, ~0);
3267         }
3268
3269         for(i = 0; i < E1000_MAX_INTR; i++)
3270                 if(unlikely(!adapter->clean_rx(adapter, adapter->rx_ring) &
3271                    !e1000_clean_tx_irq(adapter, adapter->tx_ring)))
3272                         break;
3273
3274         if(hw->mac_type == e1000_82547 || hw->mac_type == e1000_82547_rev_2)
3275                 e1000_irq_enable(adapter);
3276
3277 #endif /* CONFIG_E1000_NAPI */
3278
3279         return IRQ_HANDLED;
3280 }
3281
3282 #ifdef CONFIG_E1000_NAPI
3283 /**
3284  * e1000_clean - NAPI Rx polling callback
3285  * @adapter: board private structure
3286  **/
3287
3288 static int
3289 e1000_clean(struct net_device *poll_dev, int *budget)
3290 {
3291         struct e1000_adapter *adapter;
3292         int work_to_do = min(*budget, poll_dev->quota);
3293         int tx_cleaned, i = 0, work_done = 0;
3294
3295         /* Must NOT use netdev_priv macro here. */
3296         adapter = poll_dev->priv;
3297
3298         /* Keep link state information with original netdev */
3299         if (!netif_carrier_ok(adapter->netdev))
3300                 goto quit_polling;
3301
3302         while (poll_dev != &adapter->polling_netdev[i]) {
3303                 i++;
3304                 if (unlikely(i == adapter->num_rx_queues))
3305                         BUG();
3306         }
3307
3308         tx_cleaned = e1000_clean_tx_irq(adapter, &adapter->tx_ring[i]);
3309         adapter->clean_rx(adapter, &adapter->rx_ring[i],
3310                           &work_done, work_to_do);
3311
3312         *budget -= work_done;
3313         poll_dev->quota -= work_done;
3314
3315         /* If no Tx and not enough Rx work done, exit the polling mode */
3316         if((!tx_cleaned && (work_done == 0)) ||
3317            !netif_running(adapter->netdev)) {
3318 quit_polling:
3319                 netif_rx_complete(poll_dev);
3320                 e1000_irq_enable(adapter);
3321                 return 0;
3322         }
3323
3324         return 1;
3325 }
3326
3327 #endif
3328 /**
3329  * e1000_clean_tx_irq - Reclaim resources after transmit completes
3330  * @adapter: board private structure
3331  **/
3332
3333 static boolean_t
3334 e1000_clean_tx_irq(struct e1000_adapter *adapter,
3335                    struct e1000_tx_ring *tx_ring)
3336 {
3337         struct net_device *netdev = adapter->netdev;
3338         struct e1000_tx_desc *tx_desc, *eop_desc;
3339         struct e1000_buffer *buffer_info;
3340         unsigned int i, eop;
3341         boolean_t cleaned = FALSE;
3342
3343         i = tx_ring->next_to_clean;
3344         eop = tx_ring->buffer_info[i].next_to_watch;
3345         eop_desc = E1000_TX_DESC(*tx_ring, eop);
3346
3347         while (eop_desc->upper.data & cpu_to_le32(E1000_TXD_STAT_DD)) {
3348                 for(cleaned = FALSE; !cleaned; ) {
3349                         tx_desc = E1000_TX_DESC(*tx_ring, i);
3350                         buffer_info = &tx_ring->buffer_info[i];
3351                         cleaned = (i == eop);
3352
3353                         e1000_unmap_and_free_tx_resource(adapter, buffer_info);
3354
3355                         tx_desc->buffer_addr = 0;
3356                         tx_desc->lower.data = 0;
3357                         tx_desc->upper.data = 0;
3358
3359                         if(unlikely(++i == tx_ring->count)) i = 0;
3360                 }
3361
3362 #ifdef CONFIG_E1000_MQ
3363                 tx_ring->tx_stats.packets++;
3364 #endif
3365
3366                 eop = tx_ring->buffer_info[i].next_to_watch;
3367                 eop_desc = E1000_TX_DESC(*tx_ring, eop);
3368         }
3369
3370         tx_ring->next_to_clean = i;
3371
3372         spin_lock(&tx_ring->tx_lock);
3373
3374         if(unlikely(cleaned && netif_queue_stopped(netdev) &&
3375                     netif_carrier_ok(netdev)))
3376                 netif_wake_queue(netdev);
3377
3378         spin_unlock(&tx_ring->tx_lock);
3379
3380         if (adapter->detect_tx_hung) {
3381                 /* Detect a transmit hang in hardware, this serializes the
3382                  * check with the clearing of time_stamp and movement of i */
3383                 adapter->detect_tx_hung = FALSE;
3384                 if (tx_ring->buffer_info[i].dma &&
3385                     time_after(jiffies, tx_ring->buffer_info[i].time_stamp + HZ)
3386                     && !(E1000_READ_REG(&adapter->hw, STATUS) &
3387                         E1000_STATUS_TXOFF)) {
3388
3389                         /* detected Tx unit hang */
3390                         i = tx_ring->next_to_clean;
3391                         eop = tx_ring->buffer_info[i].next_to_watch;
3392                         eop_desc = E1000_TX_DESC(*tx_ring, eop);
3393                         DPRINTK(DRV, ERR, "Detected Tx Unit Hang\n"
3394                                         "  Tx Queue             <%lu>\n"
3395                                         "  TDH                  <%x>\n"
3396                                         "  TDT                  <%x>\n"
3397                                         "  next_to_use          <%x>\n"
3398                                         "  next_to_clean        <%x>\n"
3399                                         "buffer_info[next_to_clean]\n"
3400                                         "  dma                  <%llx>\n"
3401                                         "  time_stamp           <%lx>\n"
3402                                         "  next_to_watch        <%x>\n"
3403                                         "  jiffies              <%lx>\n"
3404                                         "  next_to_watch.status <%x>\n",
3405                                 (unsigned long)((tx_ring - adapter->tx_ring) /
3406                                         sizeof(struct e1000_tx_ring)),
3407                                 readl(adapter->hw.hw_addr + tx_ring->tdh),
3408                                 readl(adapter->hw.hw_addr + tx_ring->tdt),
3409                                 tx_ring->next_to_use,
3410                                 i,
3411                                 (unsigned long long)tx_ring->buffer_info[i].dma,
3412                                 tx_ring->buffer_info[i].time_stamp,
3413                                 eop,
3414                                 jiffies,
3415                                 eop_desc->upper.fields.status);
3416                         netif_stop_queue(netdev);
3417                 }
3418         }
3419         return cleaned;
3420 }
3421
3422 /**
3423  * e1000_rx_checksum - Receive Checksum Offload for 82543
3424  * @adapter:     board private structure
3425  * @status_err:  receive descriptor status and error fields
3426  * @csum:        receive descriptor csum field
3427  * @sk_buff:     socket buffer with received data
3428  **/
3429
3430 static inline void
3431 e1000_rx_checksum(struct e1000_adapter *adapter,
3432                   uint32_t status_err, uint32_t csum,
3433                   struct sk_buff *skb)
3434 {
3435         uint16_t status = (uint16_t)status_err;
3436         uint8_t errors = (uint8_t)(status_err >> 24);
3437         skb->ip_summed = CHECKSUM_NONE;
3438
3439         /* 82543 or newer only */
3440         if(unlikely(adapter->hw.mac_type < e1000_82543)) return;
3441         /* Ignore Checksum bit is set */
3442         if(unlikely(status & E1000_RXD_STAT_IXSM)) return;
3443         /* TCP/UDP checksum error bit is set */
3444         if(unlikely(errors & E1000_RXD_ERR_TCPE)) {
3445                 /* let the stack verify checksum errors */
3446                 adapter->hw_csum_err++;
3447                 return;
3448         }
3449         /* TCP/UDP Checksum has not been calculated */
3450         if(adapter->hw.mac_type <= e1000_82547_rev_2) {
3451                 if(!(status & E1000_RXD_STAT_TCPCS))
3452                         return;
3453         } else {
3454                 if(!(status & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS)))
3455                         return;
3456         }
3457         /* It must be a TCP or UDP packet with a valid checksum */
3458         if (likely(status & E1000_RXD_STAT_TCPCS)) {
3459                 /* TCP checksum is good */
3460                 skb->ip_summed = CHECKSUM_UNNECESSARY;
3461         } else if (adapter->hw.mac_type > e1000_82547_rev_2) {
3462                 /* IP fragment with UDP payload */
3463                 /* Hardware complements the payload checksum, so we undo it
3464                  * and then put the value in host order for further stack use.
3465                  */
3466                 csum = ntohl(csum ^ 0xFFFF);
3467                 skb->csum = csum;
3468                 skb->ip_summed = CHECKSUM_HW;
3469         }
3470         adapter->hw_csum_good++;
3471 }
3472
3473 /**
3474  * e1000_clean_rx_irq - Send received data up the network stack; legacy
3475  * @adapter: board private structure
3476  **/
3477
3478 static boolean_t
3479 #ifdef CONFIG_E1000_NAPI
3480 e1000_clean_rx_irq(struct e1000_adapter *adapter,
3481                    struct e1000_rx_ring *rx_ring,
3482                    int *work_done, int work_to_do)
3483 #else
3484 e1000_clean_rx_irq(struct e1000_adapter *adapter,
3485                    struct e1000_rx_ring *rx_ring)
3486 #endif
3487 {
3488         struct net_device *netdev = adapter->netdev;
3489         struct pci_dev *pdev = adapter->pdev;
3490         struct e1000_rx_desc *rx_desc;
3491         struct e1000_buffer *buffer_info;
3492         struct sk_buff *skb;
3493         unsigned long flags;
3494         uint32_t length;
3495         uint8_t last_byte;
3496         unsigned int i;
3497         boolean_t cleaned = FALSE;
3498
3499         i = rx_ring->next_to_clean;
3500         rx_desc = E1000_RX_DESC(*rx_ring, i);
3501
3502         while(rx_desc->status & E1000_RXD_STAT_DD) {
3503                 buffer_info = &rx_ring->buffer_info[i];
3504 #ifdef CONFIG_E1000_NAPI
3505                 if(*work_done >= work_to_do)
3506                         break;
3507                 (*work_done)++;
3508 #endif
3509                 cleaned = TRUE;
3510
3511                 pci_unmap_single(pdev,
3512                                  buffer_info->dma,
3513                                  buffer_info->length,
3514                                  PCI_DMA_FROMDEVICE);
3515
3516                 skb = buffer_info->skb;
3517                 length = le16_to_cpu(rx_desc->length);
3518
3519                 if(unlikely(!(rx_desc->status & E1000_RXD_STAT_EOP))) {
3520                         /* All receives must fit into a single buffer */
3521                         E1000_DBG("%s: Receive packet consumed multiple"
3522                                   " buffers\n", netdev->name);
3523                         dev_kfree_skb_irq(skb);
3524                         goto next_desc;
3525                 }
3526
3527                 if(unlikely(rx_desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK)) {
3528                         last_byte = *(skb->data + length - 1);
3529                         if(TBI_ACCEPT(&adapter->hw, rx_desc->status,
3530                                       rx_desc->errors, length, last_byte)) {
3531                                 spin_lock_irqsave(&adapter->stats_lock, flags);
3532                                 e1000_tbi_adjust_stats(&adapter->hw,
3533                                                        &adapter->stats,
3534                                                        length, skb->data);
3535                                 spin_unlock_irqrestore(&adapter->stats_lock,
3536                                                        flags);
3537                                 length--;
3538                         } else {
3539                                 dev_kfree_skb_irq(skb);
3540                                 goto next_desc;
3541                         }
3542                 }
3543
3544                 /* Good Receive */
3545                 skb_put(skb, length - ETHERNET_FCS_SIZE);
3546
3547                 /* Receive Checksum Offload */
3548                 e1000_rx_checksum(adapter,
3549                                   (uint32_t)(rx_desc->status) |
3550                                   ((uint32_t)(rx_desc->errors) << 24),
3551                                   rx_desc->csum, skb);
3552                 skb->protocol = eth_type_trans(skb, netdev);
3553 #ifdef CONFIG_E1000_NAPI
3554                 if(unlikely(adapter->vlgrp &&
3555                             (rx_desc->status & E1000_RXD_STAT_VP))) {
3556                         vlan_hwaccel_receive_skb(skb, adapter->vlgrp,
3557                                                  le16_to_cpu(rx_desc->special) &
3558                                                  E1000_RXD_SPC_VLAN_MASK);
3559                 } else {
3560                         netif_receive_skb(skb);
3561                 }
3562 #else /* CONFIG_E1000_NAPI */
3563                 if(unlikely(adapter->vlgrp &&
3564                             (rx_desc->status & E1000_RXD_STAT_VP))) {
3565                         vlan_hwaccel_rx(skb, adapter->vlgrp,
3566                                         le16_to_cpu(rx_desc->special) &
3567                                         E1000_RXD_SPC_VLAN_MASK);
3568                 } else {
3569                         netif_rx(skb);
3570                 }
3571 #endif /* CONFIG_E1000_NAPI */
3572                 netdev->last_rx = jiffies;
3573 #ifdef CONFIG_E1000_MQ
3574                 rx_ring->rx_stats.packets++;
3575                 rx_ring->rx_stats.bytes += length;
3576 #endif
3577
3578 next_desc:
3579                 rx_desc->status = 0;
3580                 buffer_info->skb = NULL;
3581                 if(unlikely(++i == rx_ring->count)) i = 0;
3582
3583                 rx_desc = E1000_RX_DESC(*rx_ring, i);
3584         }
3585         rx_ring->next_to_clean = i;
3586         adapter->alloc_rx_buf(adapter, rx_ring);
3587
3588         return cleaned;
3589 }
3590
3591 /**
3592  * e1000_clean_rx_irq_ps - Send received data up the network stack; packet split
3593  * @adapter: board private structure
3594  **/
3595
3596 static boolean_t
3597 #ifdef CONFIG_E1000_NAPI
3598 e1000_clean_rx_irq_ps(struct e1000_adapter *adapter,
3599                       struct e1000_rx_ring *rx_ring,
3600                       int *work_done, int work_to_do)
3601 #else
3602 e1000_clean_rx_irq_ps(struct e1000_adapter *adapter,
3603                       struct e1000_rx_ring *rx_ring)
3604 #endif
3605 {
3606         union e1000_rx_desc_packet_split *rx_desc;
3607         struct net_device *netdev = adapter->netdev;
3608         struct pci_dev *pdev = adapter->pdev;
3609         struct e1000_buffer *buffer_info;
3610         struct e1000_ps_page *ps_page;
3611         struct e1000_ps_page_dma *ps_page_dma;
3612         struct sk_buff *skb;
3613         unsigned int i, j;
3614         uint32_t length, staterr;
3615         boolean_t cleaned = FALSE;
3616
3617         i = rx_ring->next_to_clean;
3618         rx_desc = E1000_RX_DESC_PS(*rx_ring, i);
3619         staterr = le32_to_cpu(rx_desc->wb.middle.status_error);
3620
3621         while(staterr & E1000_RXD_STAT_DD) {
3622                 buffer_info = &rx_ring->buffer_info[i];
3623                 ps_page = &rx_ring->ps_page[i];
3624                 ps_page_dma = &rx_ring->ps_page_dma[i];
3625 #ifdef CONFIG_E1000_NAPI
3626                 if(unlikely(*work_done >= work_to_do))
3627                         break;
3628                 (*work_done)++;
3629 #endif
3630                 cleaned = TRUE;
3631                 pci_unmap_single(pdev, buffer_info->dma,
3632                                  buffer_info->length,
3633                                  PCI_DMA_FROMDEVICE);
3634
3635                 skb = buffer_info->skb;
3636
3637                 if(unlikely(!(staterr & E1000_RXD_STAT_EOP))) {
3638                         E1000_DBG("%s: Packet Split buffers didn't pick up"
3639                                   " the full packet\n", netdev->name);
3640                         dev_kfree_skb_irq(skb);
3641                         goto next_desc;
3642                 }
3643
3644                 if(unlikely(staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK)) {
3645                         dev_kfree_skb_irq(skb);
3646                         goto next_desc;
3647                 }
3648
3649                 length = le16_to_cpu(rx_desc->wb.middle.length0);
3650
3651                 if(unlikely(!length)) {
3652                         E1000_DBG("%s: Last part of the packet spanning"
3653                                   " multiple descriptors\n", netdev->name);
3654                         dev_kfree_skb_irq(skb);
3655                         goto next_desc;
3656                 }
3657
3658                 /* Good Receive */
3659                 skb_put(skb, length);
3660
3661                 for(j = 0; j < adapter->rx_ps_pages; j++) {
3662                         if(!(length = le16_to_cpu(rx_desc->wb.upper.length[j])))
3663                                 break;
3664
3665                         pci_unmap_page(pdev, ps_page_dma->ps_page_dma[j],
3666                                         PAGE_SIZE, PCI_DMA_FROMDEVICE);
3667                         ps_page_dma->ps_page_dma[j] = 0;
3668                         skb_shinfo(skb)->frags[j].page =
3669                                 ps_page->ps_page[j];
3670                         ps_page->ps_page[j] = NULL;
3671                         skb_shinfo(skb)->frags[j].page_offset = 0;
3672                         skb_shinfo(skb)->frags[j].size = length;
3673                         skb_shinfo(skb)->nr_frags++;
3674                         skb->len += length;
3675                         skb->data_len += length;
3676                 }
3677
3678                 e1000_rx_checksum(adapter, staterr,
3679                                   rx_desc->wb.lower.hi_dword.csum_ip.csum, skb);
3680                 skb->protocol = eth_type_trans(skb, netdev);
3681
3682                 if(likely(rx_desc->wb.upper.header_status &
3683                           E1000_RXDPS_HDRSTAT_HDRSP)) {
3684                         adapter->rx_hdr_split++;
3685 #ifdef HAVE_RX_ZERO_COPY
3686                         skb_shinfo(skb)->zero_copy = TRUE;
3687 #endif
3688                 }
3689 #ifdef CONFIG_E1000_NAPI
3690                 if(unlikely(adapter->vlgrp && (staterr & E1000_RXD_STAT_VP))) {
3691                         vlan_hwaccel_receive_skb(skb, adapter->vlgrp,
3692                                 le16_to_cpu(rx_desc->wb.middle.vlan) &
3693                                 E1000_RXD_SPC_VLAN_MASK);
3694                 } else {
3695                         netif_receive_skb(skb);
3696                 }
3697 #else /* CONFIG_E1000_NAPI */
3698                 if(unlikely(adapter->vlgrp && (staterr & E1000_RXD_STAT_VP))) {
3699                         vlan_hwaccel_rx(skb, adapter->vlgrp,
3700                                 le16_to_cpu(rx_desc->wb.middle.vlan) &
3701                                 E1000_RXD_SPC_VLAN_MASK);
3702                 } else {
3703                         netif_rx(skb);
3704                 }
3705 #endif /* CONFIG_E1000_NAPI */
3706                 netdev->last_rx = jiffies;
3707 #ifdef CONFIG_E1000_MQ
3708                 rx_ring->rx_stats.packets++;
3709                 rx_ring->rx_stats.bytes += length;
3710 #endif
3711
3712 next_desc:
3713                 rx_desc->wb.middle.status_error &= ~0xFF;
3714                 buffer_info->skb = NULL;
3715                 if(unlikely(++i == rx_ring->count)) i = 0;
3716
3717                 rx_desc = E1000_RX_DESC_PS(*rx_ring, i);
3718                 staterr = le32_to_cpu(rx_desc->wb.middle.status_error);
3719         }
3720         rx_ring->next_to_clean = i;
3721         adapter->alloc_rx_buf(adapter, rx_ring);
3722
3723         return cleaned;
3724 }
3725
3726 /**
3727  * e1000_alloc_rx_buffers - Replace used receive buffers; legacy & extended
3728  * @adapter: address of board private structure
3729  **/
3730
3731 static void
3732 e1000_alloc_rx_buffers(struct e1000_adapter *adapter,
3733                        struct e1000_rx_ring *rx_ring)
3734 {
3735         struct net_device *netdev = adapter->netdev;
3736         struct pci_dev *pdev = adapter->pdev;
3737         struct e1000_rx_desc *rx_desc;
3738         struct e1000_buffer *buffer_info;
3739         struct sk_buff *skb;
3740         unsigned int i;
3741         unsigned int bufsz = adapter->rx_buffer_len + NET_IP_ALIGN;
3742
3743         i = rx_ring->next_to_use;
3744         buffer_info = &rx_ring->buffer_info[i];
3745
3746         while(!buffer_info->skb) {
3747                 skb = dev_alloc_skb(bufsz);
3748
3749                 if(unlikely(!skb)) {
3750                         /* Better luck next round */
3751                         break;
3752                 }
3753
3754                 /* Fix for errata 23, can't cross 64kB boundary */
3755                 if (!e1000_check_64k_bound(adapter, skb->data, bufsz)) {
3756                         struct sk_buff *oldskb = skb;
3757                         DPRINTK(RX_ERR, ERR, "skb align check failed: %u bytes "
3758                                              "at %p\n", bufsz, skb->data);
3759                         /* Try again, without freeing the previous */
3760                         skb = dev_alloc_skb(bufsz);
3761                         /* Failed allocation, critical failure */
3762                         if (!skb) {
3763                                 dev_kfree_skb(oldskb);
3764                                 break;
3765                         }
3766
3767                         if (!e1000_check_64k_bound(adapter, skb->data, bufsz)) {
3768                                 /* give up */
3769                                 dev_kfree_skb(skb);
3770                                 dev_kfree_skb(oldskb);
3771                                 break; /* while !buffer_info->skb */
3772                         } else {
3773                                 /* Use new allocation */
3774                                 dev_kfree_skb(oldskb);
3775                         }
3776                 }
3777                 /* Make buffer alignment 2 beyond a 16 byte boundary
3778                  * this will result in a 16 byte aligned IP header after
3779                  * the 14 byte MAC header is removed
3780                  */
3781                 skb_reserve(skb, NET_IP_ALIGN);
3782
3783                 skb->dev = netdev;
3784
3785                 buffer_info->skb = skb;
3786                 buffer_info->length = adapter->rx_buffer_len;
3787                 buffer_info->dma = pci_map_single(pdev,
3788                                                   skb->data,
3789                                                   adapter->rx_buffer_len,
3790                                                   PCI_DMA_FROMDEVICE);
3791
3792                 /* Fix for errata 23, can't cross 64kB boundary */
3793                 if (!e1000_check_64k_bound(adapter,
3794                                         (void *)(unsigned long)buffer_info->dma,
3795                                         adapter->rx_buffer_len)) {
3796                         DPRINTK(RX_ERR, ERR,
3797                                 "dma align check failed: %u bytes at %p\n",
3798                                 adapter->rx_buffer_len,
3799                                 (void *)(unsigned long)buffer_info->dma);
3800                         dev_kfree_skb(skb);
3801                         buffer_info->skb = NULL;
3802
3803                         pci_unmap_single(pdev, buffer_info->dma,
3804                                          adapter->rx_buffer_len,
3805                                          PCI_DMA_FROMDEVICE);
3806
3807                         break; /* while !buffer_info->skb */
3808                 }
3809                 rx_desc = E1000_RX_DESC(*rx_ring, i);
3810                 rx_desc->buffer_addr = cpu_to_le64(buffer_info->dma);
3811
3812                 if(unlikely((i & ~(E1000_RX_BUFFER_WRITE - 1)) == i)) {
3813                         /* Force memory writes to complete before letting h/w
3814                          * know there are new descriptors to fetch.  (Only
3815                          * applicable for weak-ordered memory model archs,
3816                          * such as IA-64). */
3817                         wmb();
3818                         writel(i, adapter->hw.hw_addr + rx_ring->rdt);
3819                 }
3820
3821                 if(unlikely(++i == rx_ring->count)) i = 0;
3822                 buffer_info = &rx_ring->buffer_info[i];
3823         }
3824
3825         rx_ring->next_to_use = i;
3826 }
3827
3828 /**
3829  * e1000_alloc_rx_buffers_ps - Replace used receive buffers; packet split
3830  * @adapter: address of board private structure
3831  **/
3832
3833 static void
3834 e1000_alloc_rx_buffers_ps(struct e1000_adapter *adapter,
3835                           struct e1000_rx_ring *rx_ring)
3836 {
3837         struct net_device *netdev = adapter->netdev;
3838         struct pci_dev *pdev = adapter->pdev;
3839         union e1000_rx_desc_packet_split *rx_desc;
3840         struct e1000_buffer *buffer_info;
3841         struct e1000_ps_page *ps_page;
3842         struct e1000_ps_page_dma *ps_page_dma;
3843         struct sk_buff *skb;
3844         unsigned int i, j;
3845
3846         i = rx_ring->next_to_use;
3847         buffer_info = &rx_ring->buffer_info[i];
3848         ps_page = &rx_ring->ps_page[i];
3849         ps_page_dma = &rx_ring->ps_page_dma[i];
3850
3851         while(!buffer_info->skb) {
3852                 rx_desc = E1000_RX_DESC_PS(*rx_ring, i);
3853
3854                 for(j = 0; j < PS_PAGE_BUFFERS; j++) {
3855                         if (j < adapter->rx_ps_pages) {
3856                                 if (likely(!ps_page->ps_page[j])) {
3857                                         ps_page->ps_page[j] =
3858                                                 alloc_page(GFP_ATOMIC);
3859                                         if (unlikely(!ps_page->ps_page[j]))
3860                                                 goto no_buffers;
3861                                         ps_page_dma->ps_page_dma[j] =
3862                                                 pci_map_page(pdev,
3863                                                             ps_page->ps_page[j],
3864                                                             0, PAGE_SIZE,
3865                                                             PCI_DMA_FROMDEVICE);
3866                                 }
3867                                 /* Refresh the desc even if buffer_addrs didn't
3868                                  * change because each write-back erases
3869                                  * this info.
3870                                  */
3871                                 rx_desc->read.buffer_addr[j+1] =
3872                                      cpu_to_le64(ps_page_dma->ps_page_dma[j]);
3873                         } else
3874                                 rx_desc->read.buffer_addr[j+1] = ~0;
3875                 }
3876
3877                 skb = dev_alloc_skb(adapter->rx_ps_bsize0 + NET_IP_ALIGN);
3878
3879                 if(unlikely(!skb))
3880                         break;
3881
3882                 /* Make buffer alignment 2 beyond a 16 byte boundary
3883                  * this will result in a 16 byte aligned IP header after
3884                  * the 14 byte MAC header is removed
3885                  */
3886                 skb_reserve(skb, NET_IP_ALIGN);
3887
3888                 skb->dev = netdev;
3889
3890                 buffer_info->skb = skb;
3891                 buffer_info->length = adapter->rx_ps_bsize0;
3892                 buffer_info->dma = pci_map_single(pdev, skb->data,
3893                                                   adapter->rx_ps_bsize0,
3894                                                   PCI_DMA_FROMDEVICE);
3895
3896                 rx_desc->read.buffer_addr[0] = cpu_to_le64(buffer_info->dma);
3897
3898                 if(unlikely((i & ~(E1000_RX_BUFFER_WRITE - 1)) == i)) {
3899                         /* Force memory writes to complete before letting h/w
3900                          * know there are new descriptors to fetch.  (Only
3901                          * applicable for weak-ordered memory model archs,
3902                          * such as IA-64). */
3903                         wmb();
3904                         /* Hardware increments by 16 bytes, but packet split
3905                          * descriptors are 32 bytes...so we increment tail
3906                          * twice as much.
3907                          */
3908                         writel(i<<1, adapter->hw.hw_addr + rx_ring->rdt);
3909                 }
3910
3911                 if(unlikely(++i == rx_ring->count)) i = 0;
3912                 buffer_info = &rx_ring->buffer_info[i];
3913                 ps_page = &rx_ring->ps_page[i];
3914                 ps_page_dma = &rx_ring->ps_page_dma[i];
3915         }
3916
3917 no_buffers:
3918         rx_ring->next_to_use = i;
3919 }
3920
3921 /**
3922  * e1000_smartspeed - Workaround for SmartSpeed on 82541 and 82547 controllers.
3923  * @adapter:
3924  **/
3925
3926 static void
3927 e1000_smartspeed(struct e1000_adapter *adapter)
3928 {
3929         uint16_t phy_status;
3930         uint16_t phy_ctrl;
3931
3932         if((adapter->hw.phy_type != e1000_phy_igp) || !adapter->hw.autoneg ||
3933            !(adapter->hw.autoneg_advertised & ADVERTISE_1000_FULL))
3934                 return;
3935
3936         if(adapter->smartspeed == 0) {
3937                 /* If Master/Slave config fault is asserted twice,
3938                  * we assume back-to-back */
3939                 e1000_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_status);
3940                 if(!(phy_status & SR_1000T_MS_CONFIG_FAULT)) return;
3941                 e1000_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_status);
3942                 if(!(phy_status & SR_1000T_MS_CONFIG_FAULT)) return;
3943                 e1000_read_phy_reg(&adapter->hw, PHY_1000T_CTRL, &phy_ctrl);
3944                 if(phy_ctrl & CR_1000T_MS_ENABLE) {
3945                         phy_ctrl &= ~CR_1000T_MS_ENABLE;
3946                         e1000_write_phy_reg(&adapter->hw, PHY_1000T_CTRL,
3947                                             phy_ctrl);
3948                         adapter->smartspeed++;
3949                         if(!e1000_phy_setup_autoneg(&adapter->hw) &&
3950                            !e1000_read_phy_reg(&adapter->hw, PHY_CTRL,
3951                                                &phy_ctrl)) {
3952                                 phy_ctrl |= (MII_CR_AUTO_NEG_EN |
3953                                              MII_CR_RESTART_AUTO_NEG);
3954                                 e1000_write_phy_reg(&adapter->hw, PHY_CTRL,
3955                                                     phy_ctrl);
3956                         }
3957                 }
3958                 return;
3959         } else if(adapter->smartspeed == E1000_SMARTSPEED_DOWNSHIFT) {
3960                 /* If still no link, perhaps using 2/3 pair cable */
3961                 e1000_read_phy_reg(&adapter->hw, PHY_1000T_CTRL, &phy_ctrl);
3962                 phy_ctrl |= CR_1000T_MS_ENABLE;
3963                 e1000_write_phy_reg(&adapter->hw, PHY_1000T_CTRL, phy_ctrl);
3964                 if(!e1000_phy_setup_autoneg(&adapter->hw) &&
3965                    !e1000_read_phy_reg(&adapter->hw, PHY_CTRL, &phy_ctrl)) {
3966                         phy_ctrl |= (MII_CR_AUTO_NEG_EN |
3967                                      MII_CR_RESTART_AUTO_NEG);
3968                         e1000_write_phy_reg(&adapter->hw, PHY_CTRL, phy_ctrl);
3969                 }
3970         }
3971         /* Restart process after E1000_SMARTSPEED_MAX iterations */
3972         if(adapter->smartspeed++ == E1000_SMARTSPEED_MAX)
3973                 adapter->smartspeed = 0;
3974 }
3975
3976 /**
3977  * e1000_ioctl -
3978  * @netdev:
3979  * @ifreq:
3980  * @cmd:
3981  **/
3982
3983 static int
3984 e1000_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
3985 {
3986         switch (cmd) {
3987         case SIOCGMIIPHY:
3988         case SIOCGMIIREG:
3989         case SIOCSMIIREG:
3990                 return e1000_mii_ioctl(netdev, ifr, cmd);
3991         default:
3992                 return -EOPNOTSUPP;
3993         }
3994 }
3995
3996 /**
3997  * e1000_mii_ioctl -
3998  * @netdev:
3999  * @ifreq:
4000  * @cmd:
4001  **/
4002
4003 static int
4004 e1000_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
4005 {
4006         struct e1000_adapter *adapter = netdev_priv(netdev);
4007         struct mii_ioctl_data *data = if_mii(ifr);
4008         int retval;
4009         uint16_t mii_reg;
4010         uint16_t spddplx;
4011         unsigned long flags;
4012
4013         if(adapter->hw.media_type != e1000_media_type_copper)
4014                 return -EOPNOTSUPP;
4015
4016         switch (cmd) {
4017         case SIOCGMIIPHY:
4018                 data->phy_id = adapter->hw.phy_addr;
4019                 break;
4020         case SIOCGMIIREG:
4021                 if(!capable(CAP_NET_ADMIN))
4022                         return -EPERM;
4023                 spin_lock_irqsave(&adapter->stats_lock, flags);
4024                 if(e1000_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
4025                                    &data->val_out)) {
4026                         spin_unlock_irqrestore(&adapter->stats_lock, flags);
4027                         return -EIO;
4028                 }
4029                 spin_unlock_irqrestore(&adapter->stats_lock, flags);
4030                 break;
4031         case SIOCSMIIREG:
4032                 if(!capable(CAP_NET_ADMIN))
4033                         return -EPERM;
4034                 if(data->reg_num & ~(0x1F))
4035                         return -EFAULT;
4036                 mii_reg = data->val_in;
4037                 spin_lock_irqsave(&adapter->stats_lock, flags);
4038                 if(e1000_write_phy_reg(&adapter->hw, data->reg_num,
4039                                         mii_reg)) {
4040                         spin_unlock_irqrestore(&adapter->stats_lock, flags);
4041                         return -EIO;
4042                 }
4043                 if(adapter->hw.phy_type == e1000_phy_m88) {
4044                         switch (data->reg_num) {
4045                         case PHY_CTRL:
4046                                 if(mii_reg & MII_CR_POWER_DOWN)
4047                                         break;
4048                                 if(mii_reg & MII_CR_AUTO_NEG_EN) {
4049                                         adapter->hw.autoneg = 1;
4050                                         adapter->hw.autoneg_advertised = 0x2F;
4051                                 } else {
4052                                         if (mii_reg & 0x40)
4053                                                 spddplx = SPEED_1000;
4054                                         else if (mii_reg & 0x2000)
4055                                                 spddplx = SPEED_100;
4056                                         else
4057                                                 spddplx = SPEED_10;
4058                                         spddplx += (mii_reg & 0x100)
4059                                                    ? FULL_DUPLEX :
4060                                                    HALF_DUPLEX;
4061                                         retval = e1000_set_spd_dplx(adapter,
4062                                                                     spddplx);
4063                                         if(retval) {
4064                                                 spin_unlock_irqrestore(
4065                                                         &adapter->stats_lock,
4066                                                         flags);
4067                                                 return retval;
4068                                         }
4069                                 }
4070                                 if(netif_running(adapter->netdev)) {
4071                                         e1000_down(adapter);
4072                                         e1000_up(adapter);
4073                                 } else
4074                                         e1000_reset(adapter);
4075                                 break;
4076                         case M88E1000_PHY_SPEC_CTRL:
4077                         case M88E1000_EXT_PHY_SPEC_CTRL:
4078                                 if(e1000_phy_reset(&adapter->hw)) {
4079                                         spin_unlock_irqrestore(
4080                                                 &adapter->stats_lock, flags);
4081                                         return -EIO;
4082                                 }
4083                                 break;
4084                         }
4085                 } else {
4086                         switch (data->reg_num) {
4087                         case PHY_CTRL:
4088                                 if(mii_reg & MII_CR_POWER_DOWN)
4089                                         break;
4090                                 if(netif_running(adapter->netdev)) {
4091                                         e1000_down(adapter);
4092                                         e1000_up(adapter);
4093                                 } else
4094                                         e1000_reset(adapter);
4095                                 break;
4096                         }
4097                 }
4098                 spin_unlock_irqrestore(&adapter->stats_lock, flags);
4099                 break;
4100         default:
4101                 return -EOPNOTSUPP;
4102         }
4103         return E1000_SUCCESS;
4104 }
4105
4106 void
4107 e1000_pci_set_mwi(struct e1000_hw *hw)
4108 {
4109         struct e1000_adapter *adapter = hw->back;
4110         int ret_val = pci_set_mwi(adapter->pdev);
4111
4112         if(ret_val)
4113                 DPRINTK(PROBE, ERR, "Error in setting MWI\n");
4114 }
4115
4116 void
4117 e1000_pci_clear_mwi(struct e1000_hw *hw)
4118 {
4119         struct e1000_adapter *adapter = hw->back;
4120
4121         pci_clear_mwi(adapter->pdev);
4122 }
4123
4124 void
4125 e1000_read_pci_cfg(struct e1000_hw *hw, uint32_t reg, uint16_t *value)
4126 {
4127         struct e1000_adapter *adapter = hw->back;
4128
4129         pci_read_config_word(adapter->pdev, reg, value);
4130 }
4131
4132 void
4133 e1000_write_pci_cfg(struct e1000_hw *hw, uint32_t reg, uint16_t *value)
4134 {
4135         struct e1000_adapter *adapter = hw->back;
4136
4137         pci_write_config_word(adapter->pdev, reg, *value);
4138 }
4139
4140 uint32_t
4141 e1000_io_read(struct e1000_hw *hw, unsigned long port)
4142 {
4143         return inl(port);
4144 }
4145
4146 void
4147 e1000_io_write(struct e1000_hw *hw, unsigned long port, uint32_t value)
4148 {
4149         outl(value, port);
4150 }
4151
4152 static void
4153 e1000_vlan_rx_register(struct net_device *netdev, struct vlan_group *grp)
4154 {
4155         struct e1000_adapter *adapter = netdev_priv(netdev);
4156         uint32_t ctrl, rctl;
4157
4158         e1000_irq_disable(adapter);
4159         adapter->vlgrp = grp;
4160
4161         if(grp) {
4162                 /* enable VLAN tag insert/strip */
4163                 ctrl = E1000_READ_REG(&adapter->hw, CTRL);
4164                 ctrl |= E1000_CTRL_VME;
4165                 E1000_WRITE_REG(&adapter->hw, CTRL, ctrl);
4166
4167                 /* enable VLAN receive filtering */
4168                 rctl = E1000_READ_REG(&adapter->hw, RCTL);
4169                 rctl |= E1000_RCTL_VFE;
4170                 rctl &= ~E1000_RCTL_CFIEN;
4171                 E1000_WRITE_REG(&adapter->hw, RCTL, rctl);
4172                 e1000_update_mng_vlan(adapter);
4173         } else {
4174                 /* disable VLAN tag insert/strip */
4175                 ctrl = E1000_READ_REG(&adapter->hw, CTRL);
4176                 ctrl &= ~E1000_CTRL_VME;
4177                 E1000_WRITE_REG(&adapter->hw, CTRL, ctrl);
4178
4179                 /* disable VLAN filtering */
4180                 rctl = E1000_READ_REG(&adapter->hw, RCTL);
4181                 rctl &= ~E1000_RCTL_VFE;
4182                 E1000_WRITE_REG(&adapter->hw, RCTL, rctl);
4183                 if(adapter->mng_vlan_id != (uint16_t)E1000_MNG_VLAN_NONE) {
4184                         e1000_vlan_rx_kill_vid(netdev, adapter->mng_vlan_id);
4185                         adapter->mng_vlan_id = E1000_MNG_VLAN_NONE;
4186                 }
4187         }
4188
4189         e1000_irq_enable(adapter);
4190 }
4191
4192 static void
4193 e1000_vlan_rx_add_vid(struct net_device *netdev, uint16_t vid)
4194 {
4195         struct e1000_adapter *adapter = netdev_priv(netdev);
4196         uint32_t vfta, index;
4197         if((adapter->hw.mng_cookie.status &
4198                 E1000_MNG_DHCP_COOKIE_STATUS_VLAN_SUPPORT) &&
4199                 (vid == adapter->mng_vlan_id))
4200                 return;
4201         /* add VID to filter table */
4202         index = (vid >> 5) & 0x7F;
4203         vfta = E1000_READ_REG_ARRAY(&adapter->hw, VFTA, index);
4204         vfta |= (1 << (vid & 0x1F));
4205         e1000_write_vfta(&adapter->hw, index, vfta);
4206 }
4207
4208 static void
4209 e1000_vlan_rx_kill_vid(struct net_device *netdev, uint16_t vid)
4210 {
4211         struct e1000_adapter *adapter = netdev_priv(netdev);
4212         uint32_t vfta, index;
4213
4214         e1000_irq_disable(adapter);
4215
4216         if(adapter->vlgrp)
4217                 adapter->vlgrp->vlan_devices[vid] = NULL;
4218
4219         e1000_irq_enable(adapter);
4220
4221         if((adapter->hw.mng_cookie.status &
4222                 E1000_MNG_DHCP_COOKIE_STATUS_VLAN_SUPPORT) &&
4223                 (vid == adapter->mng_vlan_id))
4224                 return;
4225         /* remove VID from filter table */
4226         index = (vid >> 5) & 0x7F;
4227         vfta = E1000_READ_REG_ARRAY(&adapter->hw, VFTA, index);
4228         vfta &= ~(1 << (vid & 0x1F));
4229         e1000_write_vfta(&adapter->hw, index, vfta);
4230 }
4231
4232 static void
4233 e1000_restore_vlan(struct e1000_adapter *adapter)
4234 {
4235         e1000_vlan_rx_register(adapter->netdev, adapter->vlgrp);
4236
4237         if(adapter->vlgrp) {
4238                 uint16_t vid;
4239                 for(vid = 0; vid < VLAN_GROUP_ARRAY_LEN; vid++) {
4240                         if(!adapter->vlgrp->vlan_devices[vid])
4241                                 continue;
4242                         e1000_vlan_rx_add_vid(adapter->netdev, vid);
4243                 }
4244         }
4245 }
4246
4247 int
4248 e1000_set_spd_dplx(struct e1000_adapter *adapter, uint16_t spddplx)
4249 {
4250         adapter->hw.autoneg = 0;
4251
4252         /* Fiber NICs only allow 1000 gbps Full duplex */
4253         if((adapter->hw.media_type == e1000_media_type_fiber) &&
4254                 spddplx != (SPEED_1000 + DUPLEX_FULL)) {
4255                 DPRINTK(PROBE, ERR, "Unsupported Speed/Duplex configuration\n");
4256                 return -EINVAL;
4257         }
4258
4259         switch(spddplx) {
4260         case SPEED_10 + DUPLEX_HALF:
4261                 adapter->hw.forced_speed_duplex = e1000_10_half;
4262                 break;
4263         case SPEED_10 + DUPLEX_FULL:
4264                 adapter->hw.forced_speed_duplex = e1000_10_full;
4265                 break;
4266         case SPEED_100 + DUPLEX_HALF:
4267                 adapter->hw.forced_speed_duplex = e1000_100_half;
4268                 break;
4269         case SPEED_100 + DUPLEX_FULL:
4270                 adapter->hw.forced_speed_duplex = e1000_100_full;
4271                 break;
4272         case SPEED_1000 + DUPLEX_FULL:
4273                 adapter->hw.autoneg = 1;
4274                 adapter->hw.autoneg_advertised = ADVERTISE_1000_FULL;
4275                 break;
4276         case SPEED_1000 + DUPLEX_HALF: /* not supported */
4277         default:
4278                 DPRINTK(PROBE, ERR, "Unsupported Speed/Duplex configuration\n");
4279                 return -EINVAL;
4280         }
4281         return 0;
4282 }
4283
4284 #ifdef CONFIG_PM
4285 static int
4286 e1000_suspend(struct pci_dev *pdev, pm_message_t state)
4287 {
4288         struct net_device *netdev = pci_get_drvdata(pdev);
4289         struct e1000_adapter *adapter = netdev_priv(netdev);
4290         uint32_t ctrl, ctrl_ext, rctl, manc, status;
4291         uint32_t wufc = adapter->wol;
4292
4293         netif_device_detach(netdev);
4294
4295         if(netif_running(netdev))
4296                 e1000_down(adapter);
4297
4298         status = E1000_READ_REG(&adapter->hw, STATUS);
4299         if(status & E1000_STATUS_LU)
4300                 wufc &= ~E1000_WUFC_LNKC;
4301
4302         if(wufc) {
4303                 e1000_setup_rctl(adapter);
4304                 e1000_set_multi(netdev);
4305
4306                 /* turn on all-multi mode if wake on multicast is enabled */
4307                 if(adapter->wol & E1000_WUFC_MC) {
4308                         rctl = E1000_READ_REG(&adapter->hw, RCTL);
4309                         rctl |= E1000_RCTL_MPE;
4310                         E1000_WRITE_REG(&adapter->hw, RCTL, rctl);
4311                 }
4312
4313                 if(adapter->hw.mac_type >= e1000_82540) {
4314                         ctrl = E1000_READ_REG(&adapter->hw, CTRL);
4315                         /* advertise wake from D3Cold */
4316                         #define E1000_CTRL_ADVD3WUC 0x00100000
4317                         /* phy power management enable */
4318                         #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
4319                         ctrl |= E1000_CTRL_ADVD3WUC |
4320                                 E1000_CTRL_EN_PHY_PWR_MGMT;
4321                         E1000_WRITE_REG(&adapter->hw, CTRL, ctrl);
4322                 }
4323
4324                 if(adapter->hw.media_type == e1000_media_type_fiber ||
4325                    adapter->hw.media_type == e1000_media_type_internal_serdes) {
4326                         /* keep the laser running in D3 */
4327                         ctrl_ext = E1000_READ_REG(&adapter->hw, CTRL_EXT);
4328                         ctrl_ext |= E1000_CTRL_EXT_SDP7_DATA;
4329                         E1000_WRITE_REG(&adapter->hw, CTRL_EXT, ctrl_ext);
4330                 }
4331
4332                 /* Allow time for pending master requests to run */
4333                 e1000_disable_pciex_master(&adapter->hw);
4334
4335                 E1000_WRITE_REG(&adapter->hw, WUC, E1000_WUC_PME_EN);
4336                 E1000_WRITE_REG(&adapter->hw, WUFC, wufc);
4337                 pci_enable_wake(pdev, 3, 1);
4338                 pci_enable_wake(pdev, 4, 1); /* 4 == D3 cold */
4339         } else {
4340                 E1000_WRITE_REG(&adapter->hw, WUC, 0);
4341                 E1000_WRITE_REG(&adapter->hw, WUFC, 0);
4342                 pci_enable_wake(pdev, 3, 0);
4343                 pci_enable_wake(pdev, 4, 0); /* 4 == D3 cold */
4344         }
4345
4346         pci_save_state(pdev);
4347
4348         if(adapter->hw.mac_type >= e1000_82540 &&
4349            adapter->hw.media_type == e1000_media_type_copper) {
4350                 manc = E1000_READ_REG(&adapter->hw, MANC);
4351                 if(manc & E1000_MANC_SMBUS_EN) {
4352                         manc |= E1000_MANC_ARP_EN;
4353                         E1000_WRITE_REG(&adapter->hw, MANC, manc);
4354                         pci_enable_wake(pdev, 3, 1);
4355                         pci_enable_wake(pdev, 4, 1); /* 4 == D3 cold */
4356                 }
4357         }
4358
4359         /* Release control of h/w to f/w.  If f/w is AMT enabled, this
4360          * would have already happened in close and is redundant. */
4361         e1000_release_hw_control(adapter);
4362
4363         pci_disable_device(pdev);
4364         pci_set_power_state(pdev, pci_choose_state(pdev, state));
4365
4366         return 0;
4367 }
4368
4369 static int
4370 e1000_resume(struct pci_dev *pdev)
4371 {
4372         struct net_device *netdev = pci_get_drvdata(pdev);
4373         struct e1000_adapter *adapter = netdev_priv(netdev);
4374         uint32_t manc, ret_val;
4375
4376         pci_set_power_state(pdev, PCI_D0);
4377         pci_restore_state(pdev);
4378         ret_val = pci_enable_device(pdev);
4379         pci_set_master(pdev);
4380
4381         pci_enable_wake(pdev, PCI_D3hot, 0);
4382         pci_enable_wake(pdev, PCI_D3cold, 0);
4383
4384         e1000_reset(adapter);
4385         E1000_WRITE_REG(&adapter->hw, WUS, ~0);
4386
4387         if(netif_running(netdev))
4388                 e1000_up(adapter);
4389
4390         netif_device_attach(netdev);
4391
4392         if(adapter->hw.mac_type >= e1000_82540 &&
4393            adapter->hw.media_type == e1000_media_type_copper) {
4394                 manc = E1000_READ_REG(&adapter->hw, MANC);
4395                 manc &= ~(E1000_MANC_ARP_EN);
4396                 E1000_WRITE_REG(&adapter->hw, MANC, manc);
4397         }
4398
4399         /* If the controller is 82573 and f/w is AMT, do not set
4400          * DRV_LOAD until the interface is up.  For all other cases,
4401          * let the f/w know that the h/w is now under the control
4402          * of the driver. */
4403         if (adapter->hw.mac_type != e1000_82573 ||
4404             !e1000_check_mng_mode(&adapter->hw))
4405                 e1000_get_hw_control(adapter);
4406
4407         return 0;
4408 }
4409 #endif
4410 #ifdef CONFIG_NET_POLL_CONTROLLER
4411 /*
4412  * Polling 'interrupt' - used by things like netconsole to send skbs
4413  * without having to re-enable interrupts. It's not called while
4414  * the interrupt routine is executing.
4415  */
4416 static void
4417 e1000_netpoll(struct net_device *netdev)
4418 {
4419         struct e1000_adapter *adapter = netdev_priv(netdev);
4420         disable_irq(adapter->pdev->irq);
4421         e1000_intr(adapter->pdev->irq, netdev, NULL);
4422         e1000_clean_tx_irq(adapter, adapter->tx_ring);
4423         enable_irq(adapter->pdev->irq);
4424 }
4425 #endif
4426
4427 /* e1000_main.c */