Pileus Git - ~andy/linux/blob - drivers/net/ethernet/sfc/nic.c

   1 /****************************************************************************
   2  * Driver for Solarflare Solarstorm network controllers and boards
   3  * Copyright 2005-2006 Fen Systems Ltd.
   4  * Copyright 2006-2011 Solarflare Communications Inc.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 as published
   8  * by the Free Software Foundation, incorporated herein by reference.
   9  */
  10
  11 #include <linux/bitops.h>
  12 #include <linux/delay.h>
  13 #include <linux/interrupt.h>
  14 #include <linux/pci.h>
  15 #include <linux/module.h>
  16 #include <linux/seq_file.h>
  17 #include "net_driver.h"
  18 #include "bitfield.h"
  19 #include "efx.h"
  20 #include "nic.h"
  21 #include "regs.h"
  22 #include "io.h"
  23 #include "workarounds.h"
  24
  25 /**************************************************************************
  26  *
  27  * Configurable values
  28  *
  29  **************************************************************************
  30  */
  31
  32 /* This is set to 16 for a good reason.  In summary, if larger than
  33  * 16, the descriptor cache holds more than a default socket
  34  * buffer's worth of packets (for UDP we can only have at most one
  35  * socket buffer's worth outstanding).  This combined with the fact
  36  * that we only get 1 TX event per descriptor cache means the NIC
  37  * goes idle.
  38  */
  39 #define TX_DC_ENTRIES 16
  40 #define TX_DC_ENTRIES_ORDER 1
  41
  42 #define RX_DC_ENTRIES 64
  43 #define RX_DC_ENTRIES_ORDER 3
  44
  45 /* If EFX_MAX_INT_ERRORS internal errors occur within
  46  * EFX_INT_ERROR_EXPIRE seconds, we consider the NIC broken and
  47  * disable it.
  48  */
  49 #define EFX_INT_ERROR_EXPIRE 3600
  50 #define EFX_MAX_INT_ERRORS 5
  51
  52 /* Depth of RX flush request fifo */
  53 #define EFX_RX_FLUSH_COUNT 4
  54
  55 /* Driver generated events */
  56 #define _EFX_CHANNEL_MAGIC_TEST         0x000101
  57 #define _EFX_CHANNEL_MAGIC_FILL         0x000102
  58 #define _EFX_CHANNEL_MAGIC_RX_DRAIN     0x000103
  59 #define _EFX_CHANNEL_MAGIC_TX_DRAIN     0x000104
  60
  61 #define _EFX_CHANNEL_MAGIC(_code, _data)        ((_code) << 8 | (_data))
  62 #define _EFX_CHANNEL_MAGIC_CODE(_magic)         ((_magic) >> 8)
  63
  64 #define EFX_CHANNEL_MAGIC_TEST(_channel)                                \
  65         _EFX_CHANNEL_MAGIC(_EFX_CHANNEL_MAGIC_TEST, (_channel)->channel)
  66 #define EFX_CHANNEL_MAGIC_FILL(_rx_queue)                               \
  67         _EFX_CHANNEL_MAGIC(_EFX_CHANNEL_MAGIC_FILL,                     \
  68                            efx_rx_queue_index(_rx_queue))
  69 #define EFX_CHANNEL_MAGIC_RX_DRAIN(_rx_queue)                           \
  70         _EFX_CHANNEL_MAGIC(_EFX_CHANNEL_MAGIC_RX_DRAIN,                 \
  71                            efx_rx_queue_index(_rx_queue))
  72 #define EFX_CHANNEL_MAGIC_TX_DRAIN(_tx_queue)                           \
  73         _EFX_CHANNEL_MAGIC(_EFX_CHANNEL_MAGIC_TX_DRAIN,                 \
  74                            (_tx_queue)->queue)
  75
  76 /**************************************************************************
  77  *
  78  * Solarstorm hardware access
  79  *
  80  **************************************************************************/
  81
  82 static inline void efx_write_buf_tbl(struct efx_nic *efx, efx_qword_t *value,
  83                                      unsigned int index)
  84 {
  85         efx_sram_writeq(efx, efx->membase + efx->type->buf_tbl_base,
  86                         value, index);
  87 }
  88
  89 /* Read the current event from the event queue */
  90 static inline efx_qword_t *efx_event(struct efx_channel *channel,
  91                                      unsigned int index)
  92 {
  93         return ((efx_qword_t *) (channel->eventq.addr)) +
  94                 (index & channel->eventq_mask);
  95 }
  96
  97 /* See if an event is present
  98  *
  99  * We check both the high and low dword of the event for all ones.  We
 100  * wrote all ones when we cleared the event, and no valid event can
 101  * have all ones in either its high or low dwords.  This approach is
 102  * robust against reordering.
 103  *
 104  * Note that using a single 64-bit comparison is incorrect; even
 105  * though the CPU read will be atomic, the DMA write may not be.
 106  */
 107 static inline int efx_event_present(efx_qword_t *event)
 108 {
 109         return !(EFX_DWORD_IS_ALL_ONES(event->dword[0]) |
 110                   EFX_DWORD_IS_ALL_ONES(event->dword[1]));
 111 }
 112
 113 static bool efx_masked_compare_oword(const efx_oword_t *a, const efx_oword_t *b,
 114                                      const efx_oword_t *mask)
 115 {
 116         return ((a->u64[0] ^ b->u64[0]) & mask->u64[0]) ||
 117                 ((a->u64[1] ^ b->u64[1]) & mask->u64[1]);
 118 }
 119
 120 int efx_nic_test_registers(struct efx_nic *efx,
 121                            const struct efx_nic_register_test *regs,
 122                            size_t n_regs)
 123 {
 124         unsigned address = 0, i, j;
 125         efx_oword_t mask, imask, original, reg, buf;
 126
 127         /* Falcon should be in loopback to isolate the XMAC from the PHY */
 128         WARN_ON(!LOOPBACK_INTERNAL(efx));
 129
 130         for (i = 0; i < n_regs; ++i) {
 131                 address = regs[i].address;
 132                 mask = imask = regs[i].mask;
 133                 EFX_INVERT_OWORD(imask);
 134
 135                 efx_reado(efx, &original, address);
 136
 137                 /* bit sweep on and off */
 138                 for (j = 0; j < 128; j++) {
 139                         if (!EFX_EXTRACT_OWORD32(mask, j, j))
 140                                 continue;
 141
 142                         /* Test this testable bit can be set in isolation */
 143                         EFX_AND_OWORD(reg, original, mask);
 144                         EFX_SET_OWORD32(reg, j, j, 1);
 145
 146                         efx_writeo(efx, &reg, address);
 147                         efx_reado(efx, &buf, address);
 148
 149                         if (efx_masked_compare_oword(&reg, &buf, &mask))
 150                                 goto fail;
 151
 152                         /* Test this testable bit can be cleared in isolation */
 153                         EFX_OR_OWORD(reg, original, mask);
 154                         EFX_SET_OWORD32(reg, j, j, 0);
 155
 156                         efx_writeo(efx, &reg, address);
 157                         efx_reado(efx, &buf, address);
 158
 159                         if (efx_masked_compare_oword(&reg, &buf, &mask))
 160                                 goto fail;
 161                 }
 162
 163                 efx_writeo(efx, &original, address);
 164         }
 165
 166         return 0;
 167
 168 fail:
 169         netif_err(efx, hw, efx->net_dev,
 170                   "wrote "EFX_OWORD_FMT" read "EFX_OWORD_FMT
 171                   " at address 0x%x mask "EFX_OWORD_FMT"\n", EFX_OWORD_VAL(reg),
 172                   EFX_OWORD_VAL(buf), address, EFX_OWORD_VAL(mask));
 173         return -EIO;
 174 }
 175
 176 /**************************************************************************
 177  *
 178  * Special buffer handling
 179  * Special buffers are used for event queues and the TX and RX
 180  * descriptor rings.
 181  *
 182  *************************************************************************/
 183
 184 /*
 185  * Initialise a special buffer
 186  *
 187  * This will define a buffer (previously allocated via
 188  * efx_alloc_special_buffer()) in the buffer table, allowing
 189  * it to be used for event queues, descriptor rings etc.
 190  */
 191 static void
 192 efx_init_special_buffer(struct efx_nic *efx, struct efx_special_buffer *buffer)
 193 {
 194         efx_qword_t buf_desc;
 195         unsigned int index;
 196         dma_addr_t dma_addr;
 197         int i;
 198
 199         EFX_BUG_ON_PARANOID(!buffer->addr);
 200
 201         /* Write buffer descriptors to NIC */
 202         for (i = 0; i < buffer->entries; i++) {
 203                 index = buffer->index + i;
 204                 dma_addr = buffer->dma_addr + (i * EFX_BUF_SIZE);
 205                 netif_dbg(efx, probe, efx->net_dev,
 206                           "mapping special buffer %d at %llx\n",
 207                           index, (unsigned long long)dma_addr);
 208                 EFX_POPULATE_QWORD_3(buf_desc,
 209                                      FRF_AZ_BUF_ADR_REGION, 0,
 210                                      FRF_AZ_BUF_ADR_FBUF, dma_addr >> 12,
 211                                      FRF_AZ_BUF_OWNER_ID_FBUF, 0);
 212                 efx_write_buf_tbl(efx, &buf_desc, index);
 213         }
 214 }
 215
 216 /* Unmaps a buffer and clears the buffer table entries */
 217 static void
 218 efx_fini_special_buffer(struct efx_nic *efx, struct efx_special_buffer *buffer)
 219 {
 220         efx_oword_t buf_tbl_upd;
 221         unsigned int start = buffer->index;
 222         unsigned int end = (buffer->index + buffer->entries - 1);
 223
 224         if (!buffer->entries)
 225                 return;
 226
 227         netif_dbg(efx, hw, efx->net_dev, "unmapping special buffers %d-%d\n",
 228                   buffer->index, buffer->index + buffer->entries - 1);
 229
 230         EFX_POPULATE_OWORD_4(buf_tbl_upd,
 231                              FRF_AZ_BUF_UPD_CMD, 0,
 232                              FRF_AZ_BUF_CLR_CMD, 1,
 233                              FRF_AZ_BUF_CLR_END_ID, end,
 234                              FRF_AZ_BUF_CLR_START_ID, start);
 235         efx_writeo(efx, &buf_tbl_upd, FR_AZ_BUF_TBL_UPD);
 236 }
 237
 238 /*
 239  * Allocate a new special buffer
 240  *
 241  * This allocates memory for a new buffer, clears it and allocates a
 242  * new buffer ID range.  It does not write into the buffer table.
 243  *
 244  * This call will allocate 4KB buffers, since 8KB buffers can't be
 245  * used for event queues and descriptor rings.
 246  */
 247 static int efx_alloc_special_buffer(struct efx_nic *efx,
 248                                     struct efx_special_buffer *buffer,
 249                                     unsigned int len)
 250 {
 251         len = ALIGN(len, EFX_BUF_SIZE);
 252
 253         buffer->addr = dma_alloc_coherent(&efx->pci_dev->dev, len,
 254                                           &buffer->dma_addr, GFP_KERNEL);
 255         if (!buffer->addr)
 256                 return -ENOMEM;
 257         buffer->len = len;
 258         buffer->entries = len / EFX_BUF_SIZE;
 259         BUG_ON(buffer->dma_addr & (EFX_BUF_SIZE - 1));
 260
 261         /* All zeros is a potentially valid event so memset to 0xff */
 262         memset(buffer->addr, 0xff, len);
 263
 264         /* Select new buffer ID */
 265         buffer->index = efx->next_buffer_table;
 266         efx->next_buffer_table += buffer->entries;
 267 #ifdef CONFIG_SFC_SRIOV
 268         BUG_ON(efx_sriov_enabled(efx) &&
 269                efx->vf_buftbl_base < efx->next_buffer_table);
 270 #endif
 271
 272         netif_dbg(efx, probe, efx->net_dev,
 273                   "allocating special buffers %d-%d at %llx+%x "
 274                   "(virt %p phys %llx)\n", buffer->index,
 275                   buffer->index + buffer->entries - 1,
 276                   (u64)buffer->dma_addr, len,
 277                   buffer->addr, (u64)virt_to_phys(buffer->addr));
 278
 279         return 0;
 280 }
 281
 282 static void
 283 efx_free_special_buffer(struct efx_nic *efx, struct efx_special_buffer *buffer)
 284 {
 285         if (!buffer->addr)
 286                 return;
 287
 288         netif_dbg(efx, hw, efx->net_dev,
 289                   "deallocating special buffers %d-%d at %llx+%x "
 290                   "(virt %p phys %llx)\n", buffer->index,
 291                   buffer->index + buffer->entries - 1,
 292                   (u64)buffer->dma_addr, buffer->len,
 293                   buffer->addr, (u64)virt_to_phys(buffer->addr));
 294
 295         dma_free_coherent(&efx->pci_dev->dev, buffer->len, buffer->addr,
 296                           buffer->dma_addr);
 297         buffer->addr = NULL;
 298         buffer->entries = 0;
 299 }
 300
 301 /**************************************************************************
 302  *
 303  * Generic buffer handling
 304  * These buffers are used for interrupt status and MAC stats
 305  *
 306  **************************************************************************/
 307
 308 int efx_nic_alloc_buffer(struct efx_nic *efx, struct efx_buffer *buffer,
 309                          unsigned int len)
 310 {
 311         buffer->addr = pci_alloc_consistent(efx->pci_dev, len,
 312                                             &buffer->dma_addr);
 313         if (!buffer->addr)
 314                 return -ENOMEM;
 315         buffer->len = len;
 316         memset(buffer->addr, 0, len);
 317         return 0;
 318 }
 319
 320 void efx_nic_free_buffer(struct efx_nic *efx, struct efx_buffer *buffer)
 321 {
 322         if (buffer->addr) {
 323                 pci_free_consistent(efx->pci_dev, buffer->len,
 324                                     buffer->addr, buffer->dma_addr);
 325                 buffer->addr = NULL;
 326         }
 327 }
 328
 329 /**************************************************************************
 330  *
 331  * TX path
 332  *
 333  **************************************************************************/
 334
 335 /* Returns a pointer to the specified transmit descriptor in the TX
 336  * descriptor queue belonging to the specified channel.
 337  */
 338 static inline efx_qword_t *
 339 efx_tx_desc(struct efx_tx_queue *tx_queue, unsigned int index)
 340 {
 341         return ((efx_qword_t *) (tx_queue->txd.addr)) + index;
 342 }
 343
 344 /* This writes to the TX_DESC_WPTR; write pointer for TX descriptor ring */
 345 static inline void efx_notify_tx_desc(struct efx_tx_queue *tx_queue)
 346 {
 347         unsigned write_ptr;
 348         efx_dword_t reg;
 349
 350         write_ptr = tx_queue->write_count & tx_queue->ptr_mask;
 351         EFX_POPULATE_DWORD_1(reg, FRF_AZ_TX_DESC_WPTR_DWORD, write_ptr);
 352         efx_writed_page(tx_queue->efx, &reg,
 353                         FR_AZ_TX_DESC_UPD_DWORD_P0, tx_queue->queue);
 354 }
 355
 356 /* Write pointer and first descriptor for TX descriptor ring */
 357 static inline void efx_push_tx_desc(struct efx_tx_queue *tx_queue,
 358                                     const efx_qword_t *txd)
 359 {
 360         unsigned write_ptr;
 361         efx_oword_t reg;
 362
 363         BUILD_BUG_ON(FRF_AZ_TX_DESC_LBN != 0);
 364         BUILD_BUG_ON(FR_AA_TX_DESC_UPD_KER != FR_BZ_TX_DESC_UPD_P0);
 365
 366         write_ptr = tx_queue->write_count & tx_queue->ptr_mask;
 367         EFX_POPULATE_OWORD_2(reg, FRF_AZ_TX_DESC_PUSH_CMD, true,
 368                              FRF_AZ_TX_DESC_WPTR, write_ptr);
 369         reg.qword[0] = *txd;
 370         efx_writeo_page(tx_queue->efx, &reg,
 371                         FR_BZ_TX_DESC_UPD_P0, tx_queue->queue);
 372 }
 373
 374 static inline bool
 375 efx_may_push_tx_desc(struct efx_tx_queue *tx_queue, unsigned int write_count)
 376 {
 377         unsigned empty_read_count = ACCESS_ONCE(tx_queue->empty_read_count);
 378
 379         if (empty_read_count == 0)
 380                 return false;
 381
 382         tx_queue->empty_read_count = 0;
 383         return ((empty_read_count ^ write_count) & ~EFX_EMPTY_COUNT_VALID) == 0;
 384 }
 385
 386 /* For each entry inserted into the software descriptor ring, create a
 387  * descriptor in the hardware TX descriptor ring (in host memory), and
 388  * write a doorbell.
 389  */
 390 void efx_nic_push_buffers(struct efx_tx_queue *tx_queue)
 391 {
 392
 393         struct efx_tx_buffer *buffer;
 394         efx_qword_t *txd;
 395         unsigned write_ptr;
 396         unsigned old_write_count = tx_queue->write_count;
 397
 398         BUG_ON(tx_queue->write_count == tx_queue->insert_count);
 399
 400         do {
 401                 write_ptr = tx_queue->write_count & tx_queue->ptr_mask;
 402                 buffer = &tx_queue->buffer[write_ptr];
 403                 txd = efx_tx_desc(tx_queue, write_ptr);
 404                 ++tx_queue->write_count;
 405
 406                 /* Create TX descriptor ring entry */
 407                 EFX_POPULATE_QWORD_4(*txd,
 408                                      FSF_AZ_TX_KER_CONT, buffer->continuation,
 409                                      FSF_AZ_TX_KER_BYTE_COUNT, buffer->len,
 410                                      FSF_AZ_TX_KER_BUF_REGION, 0,
 411                                      FSF_AZ_TX_KER_BUF_ADDR, buffer->dma_addr);
 412         } while (tx_queue->write_count != tx_queue->insert_count);
 413
 414         wmb(); /* Ensure descriptors are written before they are fetched */
 415
 416         if (efx_may_push_tx_desc(tx_queue, old_write_count)) {
 417                 txd = efx_tx_desc(tx_queue,
 418                                   old_write_count & tx_queue->ptr_mask);
 419                 efx_push_tx_desc(tx_queue, txd);
 420                 ++tx_queue->pushes;
 421         } else {
 422                 efx_notify_tx_desc(tx_queue);
 423         }
 424 }
 425
 426 /* Allocate hardware resources for a TX queue */
 427 int efx_nic_probe_tx(struct efx_tx_queue *tx_queue)
 428 {
 429         struct efx_nic *efx = tx_queue->efx;
 430         unsigned entries;
 431
 432         entries = tx_queue->ptr_mask + 1;
 433         return efx_alloc_special_buffer(efx, &tx_queue->txd,
 434                                         entries * sizeof(efx_qword_t));
 435 }
 436
 437 void efx_nic_init_tx(struct efx_tx_queue *tx_queue)
 438 {
 439         struct efx_nic *efx = tx_queue->efx;
 440         efx_oword_t reg;
 441
 442         /* Pin TX descriptor ring */
 443         efx_init_special_buffer(efx, &tx_queue->txd);
 444
 445         /* Push TX descriptor ring to card */
 446         EFX_POPULATE_OWORD_10(reg,
 447                               FRF_AZ_TX_DESCQ_EN, 1,
 448                               FRF_AZ_TX_ISCSI_DDIG_EN, 0,
 449                               FRF_AZ_TX_ISCSI_HDIG_EN, 0,
 450                               FRF_AZ_TX_DESCQ_BUF_BASE_ID, tx_queue->txd.index,
 451                               FRF_AZ_TX_DESCQ_EVQ_ID,
 452                               tx_queue->channel->channel,
 453                               FRF_AZ_TX_DESCQ_OWNER_ID, 0,
 454                               FRF_AZ_TX_DESCQ_LABEL, tx_queue->queue,
 455                               FRF_AZ_TX_DESCQ_SIZE,
 456                               __ffs(tx_queue->txd.entries),
 457                               FRF_AZ_TX_DESCQ_TYPE, 0,
 458                               FRF_BZ_TX_NON_IP_DROP_DIS, 1);
 459
 460         if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) {
 461                 int csum = tx_queue->queue & EFX_TXQ_TYPE_OFFLOAD;
 462                 EFX_SET_OWORD_FIELD(reg, FRF_BZ_TX_IP_CHKSM_DIS, !csum);
 463                 EFX_SET_OWORD_FIELD(reg, FRF_BZ_TX_TCP_CHKSM_DIS,
 464                                     !csum);
 465         }
 466
 467         efx_writeo_table(efx, &reg, efx->type->txd_ptr_tbl_base,
 468                          tx_queue->queue);
 469
 470         if (efx_nic_rev(efx) < EFX_REV_FALCON_B0) {
 471                 /* Only 128 bits in this register */
 472                 BUILD_BUG_ON(EFX_MAX_TX_QUEUES > 128);
 473
 474                 efx_reado(efx, &reg, FR_AA_TX_CHKSM_CFG);
 475                 if (tx_queue->queue & EFX_TXQ_TYPE_OFFLOAD)
 476                         clear_bit_le(tx_queue->queue, (void *)&reg);
 477                 else
 478                         set_bit_le(tx_queue->queue, (void *)&reg);
 479                 efx_writeo(efx, &reg, FR_AA_TX_CHKSM_CFG);
 480         }
 481
 482         if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) {
 483                 EFX_POPULATE_OWORD_1(reg,
 484                                      FRF_BZ_TX_PACE,
 485                                      (tx_queue->queue & EFX_TXQ_TYPE_HIGHPRI) ?
 486                                      FFE_BZ_TX_PACE_OFF :
 487                                      FFE_BZ_TX_PACE_RESERVED);
 488                 efx_writeo_table(efx, &reg, FR_BZ_TX_PACE_TBL,
 489                                  tx_queue->queue);
 490         }
 491 }
 492
 493 static void efx_flush_tx_queue(struct efx_tx_queue *tx_queue)
 494 {
 495         struct efx_nic *efx = tx_queue->efx;
 496         efx_oword_t tx_flush_descq;
 497
 498         EFX_POPULATE_OWORD_2(tx_flush_descq,
 499                              FRF_AZ_TX_FLUSH_DESCQ_CMD, 1,
 500                              FRF_AZ_TX_FLUSH_DESCQ, tx_queue->queue);
 501         efx_writeo(efx, &tx_flush_descq, FR_AZ_TX_FLUSH_DESCQ);
 502 }
 503
 504 void efx_nic_fini_tx(struct efx_tx_queue *tx_queue)
 505 {
 506         struct efx_nic *efx = tx_queue->efx;
 507         efx_oword_t tx_desc_ptr;
 508
 509         /* Remove TX descriptor ring from card */
 510         EFX_ZERO_OWORD(tx_desc_ptr);
 511         efx_writeo_table(efx, &tx_desc_ptr, efx->type->txd_ptr_tbl_base,
 512                          tx_queue->queue);
 513
 514         /* Unpin TX descriptor ring */
 515         efx_fini_special_buffer(efx, &tx_queue->txd);
 516 }
 517
 518 /* Free buffers backing TX queue */
 519 void efx_nic_remove_tx(struct efx_tx_queue *tx_queue)
 520 {
 521         efx_free_special_buffer(tx_queue->efx, &tx_queue->txd);
 522 }
 523
 524 /**************************************************************************
 525  *
 526  * RX path
 527  *
 528  **************************************************************************/
 529
 530 /* Returns a pointer to the specified descriptor in the RX descriptor queue */
 531 static inline efx_qword_t *
 532 efx_rx_desc(struct efx_rx_queue *rx_queue, unsigned int index)
 533 {
 534         return ((efx_qword_t *) (rx_queue->rxd.addr)) + index;
 535 }
 536
 537 /* This creates an entry in the RX descriptor queue */
 538 static inline void
 539 efx_build_rx_desc(struct efx_rx_queue *rx_queue, unsigned index)
 540 {
 541         struct efx_rx_buffer *rx_buf;
 542         efx_qword_t *rxd;
 543
 544         rxd = efx_rx_desc(rx_queue, index);
 545         rx_buf = efx_rx_buffer(rx_queue, index);
 546         EFX_POPULATE_QWORD_3(*rxd,
 547                              FSF_AZ_RX_KER_BUF_SIZE,
 548                              rx_buf->len -
 549                              rx_queue->efx->type->rx_buffer_padding,
 550                              FSF_AZ_RX_KER_BUF_REGION, 0,
 551                              FSF_AZ_RX_KER_BUF_ADDR, rx_buf->dma_addr);
 552 }
 553
 554 /* This writes to the RX_DESC_WPTR register for the specified receive
 555  * descriptor ring.
 556  */
 557 void efx_nic_notify_rx_desc(struct efx_rx_queue *rx_queue)
 558 {
 559         struct efx_nic *efx = rx_queue->efx;
 560         efx_dword_t reg;
 561         unsigned write_ptr;
 562
 563         while (rx_queue->notified_count != rx_queue->added_count) {
 564                 efx_build_rx_desc(
 565                         rx_queue,
 566                         rx_queue->notified_count & rx_queue->ptr_mask);
 567                 ++rx_queue->notified_count;
 568         }
 569
 570         wmb();
 571         write_ptr = rx_queue->added_count & rx_queue->ptr_mask;
 572         EFX_POPULATE_DWORD_1(reg, FRF_AZ_RX_DESC_WPTR_DWORD, write_ptr);
 573         efx_writed_page(efx, &reg, FR_AZ_RX_DESC_UPD_DWORD_P0,
 574                         efx_rx_queue_index(rx_queue));
 575 }
 576
 577 int efx_nic_probe_rx(struct efx_rx_queue *rx_queue)
 578 {
 579         struct efx_nic *efx = rx_queue->efx;
 580         unsigned entries;
 581
 582         entries = rx_queue->ptr_mask + 1;
 583         return efx_alloc_special_buffer(efx, &rx_queue->rxd,
 584                                         entries * sizeof(efx_qword_t));
 585 }
 586
 587 void efx_nic_init_rx(struct efx_rx_queue *rx_queue)
 588 {
 589         efx_oword_t rx_desc_ptr;
 590         struct efx_nic *efx = rx_queue->efx;
 591         bool is_b0 = efx_nic_rev(efx) >= EFX_REV_FALCON_B0;
 592         bool iscsi_digest_en = is_b0;
 593
 594         netif_dbg(efx, hw, efx->net_dev,
 595                   "RX queue %d ring in special buffers %d-%d\n",
 596                   efx_rx_queue_index(rx_queue), rx_queue->rxd.index,
 597                   rx_queue->rxd.index + rx_queue->rxd.entries - 1);
 598
 599         /* Pin RX descriptor ring */
 600         efx_init_special_buffer(efx, &rx_queue->rxd);
 601
 602         /* Push RX descriptor ring to card */
 603         EFX_POPULATE_OWORD_10(rx_desc_ptr,
 604                               FRF_AZ_RX_ISCSI_DDIG_EN, iscsi_digest_en,
 605                               FRF_AZ_RX_ISCSI_HDIG_EN, iscsi_digest_en,
 606                               FRF_AZ_RX_DESCQ_BUF_BASE_ID, rx_queue->rxd.index,
 607                               FRF_AZ_RX_DESCQ_EVQ_ID,
 608                               efx_rx_queue_channel(rx_queue)->channel,
 609                               FRF_AZ_RX_DESCQ_OWNER_ID, 0,
 610                               FRF_AZ_RX_DESCQ_LABEL,
 611                               efx_rx_queue_index(rx_queue),
 612                               FRF_AZ_RX_DESCQ_SIZE,
 613                               __ffs(rx_queue->rxd.entries),
 614                               FRF_AZ_RX_DESCQ_TYPE, 0 /* kernel queue */ ,
 615                               /* For >=B0 this is scatter so disable */
 616                               FRF_AZ_RX_DESCQ_JUMBO, !is_b0,
 617                               FRF_AZ_RX_DESCQ_EN, 1);
 618         efx_writeo_table(efx, &rx_desc_ptr, efx->type->rxd_ptr_tbl_base,
 619                          efx_rx_queue_index(rx_queue));
 620 }
 621
 622 static void efx_flush_rx_queue(struct efx_rx_queue *rx_queue)
 623 {
 624         struct efx_nic *efx = rx_queue->efx;
 625         efx_oword_t rx_flush_descq;
 626
 627         EFX_POPULATE_OWORD_2(rx_flush_descq,
 628                              FRF_AZ_RX_FLUSH_DESCQ_CMD, 1,
 629                              FRF_AZ_RX_FLUSH_DESCQ,
 630                              efx_rx_queue_index(rx_queue));
 631         efx_writeo(efx, &rx_flush_descq, FR_AZ_RX_FLUSH_DESCQ);
 632 }
 633
 634 void efx_nic_fini_rx(struct efx_rx_queue *rx_queue)
 635 {
 636         efx_oword_t rx_desc_ptr;
 637         struct efx_nic *efx = rx_queue->efx;
 638
 639         /* Remove RX descriptor ring from card */
 640         EFX_ZERO_OWORD(rx_desc_ptr);
 641         efx_writeo_table(efx, &rx_desc_ptr, efx->type->rxd_ptr_tbl_base,
 642                          efx_rx_queue_index(rx_queue));
 643
 644         /* Unpin RX descriptor ring */
 645         efx_fini_special_buffer(efx, &rx_queue->rxd);
 646 }
 647
 648 /* Free buffers backing RX queue */
 649 void efx_nic_remove_rx(struct efx_rx_queue *rx_queue)
 650 {
 651         efx_free_special_buffer(rx_queue->efx, &rx_queue->rxd);
 652 }
 653
 654 /**************************************************************************
 655  *
 656  * Flush handling
 657  *
 658  **************************************************************************/
 659
 660 /* efx_nic_flush_queues() must be woken up when all flushes are completed,
 661  * or more RX flushes can be kicked off.
 662  */
 663 static bool efx_flush_wake(struct efx_nic *efx)
 664 {
 665         /* Ensure that all updates are visible to efx_nic_flush_queues() */
 666         smp_mb();
 667
 668         return (atomic_read(&efx->drain_pending) == 0 ||
 669                 (atomic_read(&efx->rxq_flush_outstanding) < EFX_RX_FLUSH_COUNT
 670                  && atomic_read(&efx->rxq_flush_pending) > 0));
 671 }
 672
 673 /* Flush all the transmit queues, and continue flushing receive queues until
 674  * they're all flushed. Wait for the DRAIN events to be recieved so that there
 675  * are no more RX and TX events left on any channel. */
 676 int efx_nic_flush_queues(struct efx_nic *efx)
 677 {
 678         unsigned timeout = msecs_to_jiffies(5000); /* 5s for all flushes and drains */
 679         struct efx_channel *channel;
 680         struct efx_rx_queue *rx_queue;
 681         struct efx_tx_queue *tx_queue;
 682         int rc = 0;
 683
 684         efx->fc_disable++;
 685         efx->type->prepare_flush(efx);
 686
 687         efx_for_each_channel(channel, efx) {
 688                 efx_for_each_channel_tx_queue(tx_queue, channel) {
 689                         atomic_inc(&efx->drain_pending);
 690                         efx_flush_tx_queue(tx_queue);
 691                 }
 692                 efx_for_each_channel_rx_queue(rx_queue, channel) {
 693                         atomic_inc(&efx->drain_pending);
 694                         rx_queue->flush_pending = true;
 695                         atomic_inc(&efx->rxq_flush_pending);
 696                 }
 697         }
 698
 699         while (timeout && atomic_read(&efx->drain_pending) > 0) {
 700                 /* If SRIOV is enabled, then offload receive queue flushing to
 701                  * the firmware (though we will still have to poll for
 702                  * completion). If that fails, fall back to the old scheme.
 703                  */
 704                 if (efx_sriov_enabled(efx)) {
 705                         rc = efx_mcdi_flush_rxqs(efx);
 706                         if (!rc)
 707                                 goto wait;
 708                 }
 709
 710                 /* The hardware supports four concurrent rx flushes, each of
 711                  * which may need to be retried if there is an outstanding
 712                  * descriptor fetch
 713                  */
 714                 efx_for_each_channel(channel, efx) {
 715                         efx_for_each_channel_rx_queue(rx_queue, channel) {
 716                                 if (atomic_read(&efx->rxq_flush_outstanding) >=
 717                                     EFX_RX_FLUSH_COUNT)
 718                                         break;
 719
 720                                 if (rx_queue->flush_pending) {
 721                                         rx_queue->flush_pending = false;
 722                                         atomic_dec(&efx->rxq_flush_pending);
 723                                         atomic_inc(&efx->rxq_flush_outstanding);
 724                                         efx_flush_rx_queue(rx_queue);
 725                                 }
 726                         }
 727                 }
 728
 729         wait:
 730                 timeout = wait_event_timeout(efx->flush_wq, efx_flush_wake(efx),
 731                                              timeout);
 732         }
 733
 734         if (atomic_read(&efx->drain_pending)) {
 735                 netif_err(efx, hw, efx->net_dev, "failed to flush %d queues "
 736                           "(rx %d+%d)\n", atomic_read(&efx->drain_pending),
 737                           atomic_read(&efx->rxq_flush_outstanding),
 738                           atomic_read(&efx->rxq_flush_pending));
 739                 rc = -ETIMEDOUT;
 740
 741                 atomic_set(&efx->drain_pending, 0);
 742                 atomic_set(&efx->rxq_flush_pending, 0);
 743                 atomic_set(&efx->rxq_flush_outstanding, 0);
 744         }
 745
 746         efx->fc_disable--;
 747
 748         return rc;
 749 }
 750
 751 /**************************************************************************
 752  *
 753  * Event queue processing
 754  * Event queues are processed by per-channel tasklets.
 755  *
 756  **************************************************************************/
 757
 758 /* Update a channel's event queue's read pointer (RPTR) register
 759  *
 760  * This writes the EVQ_RPTR_REG register for the specified channel's
 761  * event queue.
 762  */
 763 void efx_nic_eventq_read_ack(struct efx_channel *channel)
 764 {
 765         efx_dword_t reg;
 766         struct efx_nic *efx = channel->efx;
 767
 768         EFX_POPULATE_DWORD_1(reg, FRF_AZ_EVQ_RPTR,
 769                              channel->eventq_read_ptr & channel->eventq_mask);
 770         efx_writed_table(efx, &reg, efx->type->evq_rptr_tbl_base,
 771                          channel->channel);
 772 }
 773
 774 /* Use HW to insert a SW defined event */
 775 void efx_generate_event(struct efx_nic *efx, unsigned int evq,
 776                         efx_qword_t *event)
 777 {
 778         efx_oword_t drv_ev_reg;
 779
 780         BUILD_BUG_ON(FRF_AZ_DRV_EV_DATA_LBN != 0 ||
 781                      FRF_AZ_DRV_EV_DATA_WIDTH != 64);
 782         drv_ev_reg.u32[0] = event->u32[0];
 783         drv_ev_reg.u32[1] = event->u32[1];
 784         drv_ev_reg.u32[2] = 0;
 785         drv_ev_reg.u32[3] = 0;
 786         EFX_SET_OWORD_FIELD(drv_ev_reg, FRF_AZ_DRV_EV_QID, evq);
 787         efx_writeo(efx, &drv_ev_reg, FR_AZ_DRV_EV);
 788 }
 789
 790 static void efx_magic_event(struct efx_channel *channel, u32 magic)
 791 {
 792         efx_qword_t event;
 793
 794         EFX_POPULATE_QWORD_2(event, FSF_AZ_EV_CODE,
 795                              FSE_AZ_EV_CODE_DRV_GEN_EV,
 796                              FSF_AZ_DRV_GEN_EV_MAGIC, magic);
 797         efx_generate_event(channel->efx, channel->channel, &event);
 798 }
 799
 800 /* Handle a transmit completion event
 801  *
 802  * The NIC batches TX completion events; the message we receive is of
 803  * the form "complete all TX events up to this index".
 804  */
 805 static int
 806 efx_handle_tx_event(struct efx_channel *channel, efx_qword_t *event)
 807 {
 808         unsigned int tx_ev_desc_ptr;
 809         unsigned int tx_ev_q_label;
 810         struct efx_tx_queue *tx_queue;
 811         struct efx_nic *efx = channel->efx;
 812         int tx_packets = 0;
 813
 814         if (unlikely(ACCESS_ONCE(efx->reset_pending)))
 815                 return 0;
 816
 817         if (likely(EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_COMP))) {
 818                 /* Transmit completion */
 819                 tx_ev_desc_ptr = EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_DESC_PTR);
 820                 tx_ev_q_label = EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_Q_LABEL);
 821                 tx_queue = efx_channel_get_tx_queue(
 822                         channel, tx_ev_q_label % EFX_TXQ_TYPES);
 823                 tx_packets = ((tx_ev_desc_ptr - tx_queue->read_count) &
 824                               tx_queue->ptr_mask);
 825                 efx_xmit_done(tx_queue, tx_ev_desc_ptr);
 826         } else if (EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_WQ_FF_FULL)) {
 827                 /* Rewrite the FIFO write pointer */
 828                 tx_ev_q_label = EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_Q_LABEL);
 829                 tx_queue = efx_channel_get_tx_queue(
 830                         channel, tx_ev_q_label % EFX_TXQ_TYPES);
 831
 832                 netif_tx_lock(efx->net_dev);
 833                 efx_notify_tx_desc(tx_queue);
 834                 netif_tx_unlock(efx->net_dev);
 835         } else if (EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_PKT_ERR) &&
 836                    EFX_WORKAROUND_10727(efx)) {
 837                 efx_schedule_reset(efx, RESET_TYPE_TX_DESC_FETCH);
 838         } else {
 839                 netif_err(efx, tx_err, efx->net_dev,
 840                           "channel %d unexpected TX event "
 841                           EFX_QWORD_FMT"\n", channel->channel,
 842                           EFX_QWORD_VAL(*event));
 843         }
 844
 845         return tx_packets;
 846 }
 847
 848 /* Detect errors included in the rx_evt_pkt_ok bit. */
 849 static u16 efx_handle_rx_not_ok(struct efx_rx_queue *rx_queue,
 850                                 const efx_qword_t *event)
 851 {
 852         struct efx_channel *channel = efx_rx_queue_channel(rx_queue);
 853         struct efx_nic *efx = rx_queue->efx;
 854         bool rx_ev_buf_owner_id_err, rx_ev_ip_hdr_chksum_err;
 855         bool rx_ev_tcp_udp_chksum_err, rx_ev_eth_crc_err;
 856         bool rx_ev_frm_trunc, rx_ev_drib_nib, rx_ev_tobe_disc;
 857         bool rx_ev_other_err, rx_ev_pause_frm;
 858         bool rx_ev_hdr_type, rx_ev_mcast_pkt;
 859         unsigned rx_ev_pkt_type;
 860
 861         rx_ev_hdr_type = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_HDR_TYPE);
 862         rx_ev_mcast_pkt = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_MCAST_PKT);
 863         rx_ev_tobe_disc = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_TOBE_DISC);
 864         rx_ev_pkt_type = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_PKT_TYPE);
 865         rx_ev_buf_owner_id_err = EFX_QWORD_FIELD(*event,
 866                                                  FSF_AZ_RX_EV_BUF_OWNER_ID_ERR);
 867         rx_ev_ip_hdr_chksum_err = EFX_QWORD_FIELD(*event,
 868                                                   FSF_AZ_RX_EV_IP_HDR_CHKSUM_ERR);
 869         rx_ev_tcp_udp_chksum_err = EFX_QWORD_FIELD(*event,
 870                                                    FSF_AZ_RX_EV_TCP_UDP_CHKSUM_ERR);
 871         rx_ev_eth_crc_err = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_ETH_CRC_ERR);
 872         rx_ev_frm_trunc = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_FRM_TRUNC);
 873         rx_ev_drib_nib = ((efx_nic_rev(efx) >= EFX_REV_FALCON_B0) ?
 874                           0 : EFX_QWORD_FIELD(*event, FSF_AA_RX_EV_DRIB_NIB));
 875         rx_ev_pause_frm = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_PAUSE_FRM_ERR);
 876
 877         /* Every error apart from tobe_disc and pause_frm */
 878         rx_ev_other_err = (rx_ev_drib_nib | rx_ev_tcp_udp_chksum_err |
 879                            rx_ev_buf_owner_id_err | rx_ev_eth_crc_err |
 880                            rx_ev_frm_trunc | rx_ev_ip_hdr_chksum_err);
 881
 882         /* Count errors that are not in MAC stats.  Ignore expected
 883          * checksum errors during self-test. */
 884         if (rx_ev_frm_trunc)
 885                 ++channel->n_rx_frm_trunc;
 886         else if (rx_ev_tobe_disc)
 887                 ++channel->n_rx_tobe_disc;
 888         else if (!efx->loopback_selftest) {
 889                 if (rx_ev_ip_hdr_chksum_err)
 890                         ++channel->n_rx_ip_hdr_chksum_err;
 891                 else if (rx_ev_tcp_udp_chksum_err)
 892                         ++channel->n_rx_tcp_udp_chksum_err;
 893         }
 894
 895         /* TOBE_DISC is expected on unicast mismatches; don't print out an
 896          * error message.  FRM_TRUNC indicates RXDP dropped the packet due
 897          * to a FIFO overflow.
 898          */
 899 #ifdef DEBUG
 900         if (rx_ev_other_err && net_ratelimit()) {
 901                 netif_dbg(efx, rx_err, efx->net_dev,
 902                           " RX queue %d unexpected RX event "
 903                           EFX_QWORD_FMT "%s%s%s%s%s%s%s%s\n",
 904                           efx_rx_queue_index(rx_queue), EFX_QWORD_VAL(*event),
 905                           rx_ev_buf_owner_id_err ? " [OWNER_ID_ERR]" : "",
 906                           rx_ev_ip_hdr_chksum_err ?
 907                           " [IP_HDR_CHKSUM_ERR]" : "",
 908                           rx_ev_tcp_udp_chksum_err ?
 909                           " [TCP_UDP_CHKSUM_ERR]" : "",
 910                           rx_ev_eth_crc_err ? " [ETH_CRC_ERR]" : "",
 911                           rx_ev_frm_trunc ? " [FRM_TRUNC]" : "",
 912                           rx_ev_drib_nib ? " [DRIB_NIB]" : "",
 913                           rx_ev_tobe_disc ? " [TOBE_DISC]" : "",
 914                           rx_ev_pause_frm ? " [PAUSE]" : "");
 915         }
 916 #endif
 917
 918         /* The frame must be discarded if any of these are true. */
 919         return (rx_ev_eth_crc_err | rx_ev_frm_trunc | rx_ev_drib_nib |
 920                 rx_ev_tobe_disc | rx_ev_pause_frm) ?
 921                 EFX_RX_PKT_DISCARD : 0;
 922 }
 923
 924 /* Handle receive events that are not in-order. */
 925 static void
 926 efx_handle_rx_bad_index(struct efx_rx_queue *rx_queue, unsigned index)
 927 {
 928         struct efx_nic *efx = rx_queue->efx;
 929         unsigned expected, dropped;
 930
 931         expected = rx_queue->removed_count & rx_queue->ptr_mask;
 932         dropped = (index - expected) & rx_queue->ptr_mask;
 933         netif_info(efx, rx_err, efx->net_dev,
 934                    "dropped %d events (index=%d expected=%d)\n",
 935                    dropped, index, expected);
 936
 937         efx_schedule_reset(efx, EFX_WORKAROUND_5676(efx) ?
 938                            RESET_TYPE_RX_RECOVERY : RESET_TYPE_DISABLE);
 939 }
 940
 941 /* Handle a packet received event
 942  *
 943  * The NIC gives a "discard" flag if it's a unicast packet with the
 944  * wrong destination address
 945  * Also "is multicast" and "matches multicast filter" flags can be used to
 946  * discard non-matching multicast packets.
 947  */
 948 static void
 949 efx_handle_rx_event(struct efx_channel *channel, const efx_qword_t *event)
 950 {
 951         unsigned int rx_ev_desc_ptr, rx_ev_byte_cnt;
 952         unsigned int rx_ev_hdr_type, rx_ev_mcast_pkt;
 953         unsigned expected_ptr;
 954         bool rx_ev_pkt_ok;
 955         u16 flags;
 956         struct efx_rx_queue *rx_queue;
 957         struct efx_nic *efx = channel->efx;
 958
 959         if (unlikely(ACCESS_ONCE(efx->reset_pending)))
 960                 return;
 961
 962         /* Basic packet information */
 963         rx_ev_byte_cnt = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_BYTE_CNT);
 964         rx_ev_pkt_ok = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_PKT_OK);
 965         rx_ev_hdr_type = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_HDR_TYPE);
 966         WARN_ON(EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_JUMBO_CONT));
 967         WARN_ON(EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_SOP) != 1);
 968         WARN_ON(EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_Q_LABEL) !=
 969                 channel->channel);
 970
 971         rx_queue = efx_channel_get_rx_queue(channel);
 972
 973         rx_ev_desc_ptr = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_DESC_PTR);
 974         expected_ptr = rx_queue->removed_count & rx_queue->ptr_mask;
 975         if (unlikely(rx_ev_desc_ptr != expected_ptr))
 976                 efx_handle_rx_bad_index(rx_queue, rx_ev_desc_ptr);
 977
 978         if (likely(rx_ev_pkt_ok)) {
 979                 /* If packet is marked as OK and packet type is TCP/IP or
 980                  * UDP/IP, then we can rely on the hardware checksum.
 981                  */
 982                 flags = (rx_ev_hdr_type == FSE_CZ_RX_EV_HDR_TYPE_IPV4V6_TCP ||
 983                          rx_ev_hdr_type == FSE_CZ_RX_EV_HDR_TYPE_IPV4V6_UDP) ?
 984                         EFX_RX_PKT_CSUMMED : 0;
 985         } else {
 986                 flags = efx_handle_rx_not_ok(rx_queue, event);
 987         }
 988
 989         /* Detect multicast packets that didn't match the filter */
 990         rx_ev_mcast_pkt = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_MCAST_PKT);
 991         if (rx_ev_mcast_pkt) {
 992                 unsigned int rx_ev_mcast_hash_match =
 993                         EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_MCAST_HASH_MATCH);
 994
 995                 if (unlikely(!rx_ev_mcast_hash_match)) {
 996                         ++channel->n_rx_mcast_mismatch;
 997                         flags |= EFX_RX_PKT_DISCARD;
 998                 }
 999         }
1000
1001         channel->irq_mod_score += 2;
1002
1003         /* Handle received packet */
1004         efx_rx_packet(rx_queue, rx_ev_desc_ptr, rx_ev_byte_cnt, flags);
1005 }
1006
1007 /* If this flush done event corresponds to a &struct efx_tx_queue, then
1008  * send an %EFX_CHANNEL_MAGIC_TX_DRAIN event to drain the event queue
1009  * of all transmit completions.
1010  */
1011 static void
1012 efx_handle_tx_flush_done(struct efx_nic *efx, efx_qword_t *event)
1013 {
1014         struct efx_tx_queue *tx_queue;
1015         int qid;
1016
1017         qid = EFX_QWORD_FIELD(*event, FSF_AZ_DRIVER_EV_SUBDATA);
1018         if (qid < EFX_TXQ_TYPES * efx->n_tx_channels) {
1019                 tx_queue = efx_get_tx_queue(efx, qid / EFX_TXQ_TYPES,
1020                                             qid % EFX_TXQ_TYPES);
1021
1022                 efx_magic_event(tx_queue->channel,
1023                                 EFX_CHANNEL_MAGIC_TX_DRAIN(tx_queue));
1024         }
1025 }
1026
1027 /* If this flush done event corresponds to a &struct efx_rx_queue: If the flush
1028  * was succesful then send an %EFX_CHANNEL_MAGIC_RX_DRAIN, otherwise add
1029  * the RX queue back to the mask of RX queues in need of flushing.
1030  */
1031 static void
1032 efx_handle_rx_flush_done(struct efx_nic *efx, efx_qword_t *event)
1033 {
1034         struct efx_channel *channel;
1035         struct efx_rx_queue *rx_queue;
1036         int qid;
1037         bool failed;
1038
1039         qid = EFX_QWORD_FIELD(*event, FSF_AZ_DRIVER_EV_RX_DESCQ_ID);
1040         failed = EFX_QWORD_FIELD(*event, FSF_AZ_DRIVER_EV_RX_FLUSH_FAIL);
1041         if (qid >= efx->n_channels)
1042                 return;
1043         channel = efx_get_channel(efx, qid);
1044         if (!efx_channel_has_rx_queue(channel))
1045                 return;
1046         rx_queue = efx_channel_get_rx_queue(channel);
1047
1048         if (failed) {
1049                 netif_info(efx, hw, efx->net_dev,
1050                            "RXQ %d flush retry\n", qid);
1051                 rx_queue->flush_pending = true;
1052                 atomic_inc(&efx->rxq_flush_pending);
1053         } else {
1054                 efx_magic_event(efx_rx_queue_channel(rx_queue),
1055                                 EFX_CHANNEL_MAGIC_RX_DRAIN(rx_queue));
1056         }
1057         atomic_dec(&efx->rxq_flush_outstanding);
1058         if (efx_flush_wake(efx))
1059                 wake_up(&efx->flush_wq);
1060 }
1061
1062 static void
1063 efx_handle_drain_event(struct efx_channel *channel)
1064 {
1065         struct efx_nic *efx = channel->efx;
1066
1067         WARN_ON(atomic_read(&efx->drain_pending) == 0);
1068         atomic_dec(&efx->drain_pending);
1069         if (efx_flush_wake(efx))
1070                 wake_up(&efx->flush_wq);
1071 }
1072
1073 static void
1074 efx_handle_generated_event(struct efx_channel *channel, efx_qword_t *event)
1075 {
1076         struct efx_nic *efx = channel->efx;
1077         struct efx_rx_queue *rx_queue =
1078                 efx_channel_has_rx_queue(channel) ?
1079                 efx_channel_get_rx_queue(channel) : NULL;
1080         unsigned magic, code;
1081
1082         magic = EFX_QWORD_FIELD(*event, FSF_AZ_DRV_GEN_EV_MAGIC);
1083         code = _EFX_CHANNEL_MAGIC_CODE(magic);
1084
1085         if (magic == EFX_CHANNEL_MAGIC_TEST(channel)) {
1086                 channel->event_test_cpu = raw_smp_processor_id();
1087         } else if (rx_queue && magic == EFX_CHANNEL_MAGIC_FILL(rx_queue)) {
1088                 /* The queue must be empty, so we won't receive any rx
1089                  * events, so efx_process_channel() won't refill the
1090                  * queue. Refill it here */
1091                 efx_fast_push_rx_descriptors(rx_queue);
1092         } else if (rx_queue && magic == EFX_CHANNEL_MAGIC_RX_DRAIN(rx_queue)) {
1093                 rx_queue->enabled = false;
1094                 efx_handle_drain_event(channel);
1095         } else if (code == _EFX_CHANNEL_MAGIC_TX_DRAIN) {
1096                 efx_handle_drain_event(channel);
1097         } else {
1098                 netif_dbg(efx, hw, efx->net_dev, "channel %d received "
1099                           "generated event "EFX_QWORD_FMT"\n",
1100                           channel->channel, EFX_QWORD_VAL(*event));
1101         }
1102 }
1103
1104 static void
1105 efx_handle_driver_event(struct efx_channel *channel, efx_qword_t *event)
1106 {
1107         struct efx_nic *efx = channel->efx;
1108         unsigned int ev_sub_code;
1109         unsigned int ev_sub_data;
1110
1111         ev_sub_code = EFX_QWORD_FIELD(*event, FSF_AZ_DRIVER_EV_SUBCODE);
1112         ev_sub_data = EFX_QWORD_FIELD(*event, FSF_AZ_DRIVER_EV_SUBDATA);
1113
1114         switch (ev_sub_code) {
1115         case FSE_AZ_TX_DESCQ_FLS_DONE_EV:
1116                 netif_vdbg(efx, hw, efx->net_dev, "channel %d TXQ %d flushed\n",
1117                            channel->channel, ev_sub_data);
1118                 efx_handle_tx_flush_done(efx, event);
1119                 efx_sriov_tx_flush_done(efx, event);
1120                 break;
1121         case FSE_AZ_RX_DESCQ_FLS_DONE_EV:
1122                 netif_vdbg(efx, hw, efx->net_dev, "channel %d RXQ %d flushed\n",
1123                            channel->channel, ev_sub_data);
1124                 efx_handle_rx_flush_done(efx, event);
1125                 efx_sriov_rx_flush_done(efx, event);
1126                 break;
1127         case FSE_AZ_EVQ_INIT_DONE_EV:
1128                 netif_dbg(efx, hw, efx->net_dev,
1129                           "channel %d EVQ %d initialised\n",
1130                           channel->channel, ev_sub_data);
1131                 break;
1132         case FSE_AZ_SRM_UPD_DONE_EV:
1133                 netif_vdbg(efx, hw, efx->net_dev,
1134                            "channel %d SRAM update done\n", channel->channel);
1135                 break;
1136         case FSE_AZ_WAKE_UP_EV:
1137                 netif_vdbg(efx, hw, efx->net_dev,
1138                            "channel %d RXQ %d wakeup event\n",
1139                            channel->channel, ev_sub_data);
1140                 break;
1141         case FSE_AZ_TIMER_EV:
1142                 netif_vdbg(efx, hw, efx->net_dev,
1143                            "channel %d RX queue %d timer expired\n",
1144                            channel->channel, ev_sub_data);
1145                 break;
1146         case FSE_AA_RX_RECOVER_EV:
1147                 netif_err(efx, rx_err, efx->net_dev,
1148                           "channel %d seen DRIVER RX_RESET event. "
1149                         "Resetting.\n", channel->channel);
1150                 atomic_inc(&efx->rx_reset);
1151                 efx_schedule_reset(efx,
1152                                    EFX_WORKAROUND_6555(efx) ?
1153                                    RESET_TYPE_RX_RECOVERY :
1154                                    RESET_TYPE_DISABLE);
1155                 break;
1156         case FSE_BZ_RX_DSC_ERROR_EV:
1157                 if (ev_sub_data < EFX_VI_BASE) {
1158                         netif_err(efx, rx_err, efx->net_dev,
1159                                   "RX DMA Q %d reports descriptor fetch error."
1160                                   " RX Q %d is disabled.\n", ev_sub_data,
1161                                   ev_sub_data);
1162                         efx_schedule_reset(efx, RESET_TYPE_RX_DESC_FETCH);
1163                 } else
1164                         efx_sriov_desc_fetch_err(efx, ev_sub_data);
1165                 break;
1166         case FSE_BZ_TX_DSC_ERROR_EV:
1167                 if (ev_sub_data < EFX_VI_BASE) {
1168                         netif_err(efx, tx_err, efx->net_dev,
1169                                   "TX DMA Q %d reports descriptor fetch error."
1170                                   " TX Q %d is disabled.\n", ev_sub_data,
1171                                   ev_sub_data);
1172                         efx_schedule_reset(efx, RESET_TYPE_TX_DESC_FETCH);
1173                 } else
1174                         efx_sriov_desc_fetch_err(efx, ev_sub_data);
1175                 break;
1176         default:
1177                 netif_vdbg(efx, hw, efx->net_dev,
1178                            "channel %d unknown driver event code %d "
1179                            "data %04x\n", channel->channel, ev_sub_code,
1180                            ev_sub_data);
1181                 break;
1182         }
1183 }
1184
1185 int efx_nic_process_eventq(struct efx_channel *channel, int budget)
1186 {
1187         struct efx_nic *efx = channel->efx;
1188         unsigned int read_ptr;
1189         efx_qword_t event, *p_event;
1190         int ev_code;
1191         int tx_packets = 0;
1192         int spent = 0;
1193
1194         read_ptr = channel->eventq_read_ptr;
1195
1196         for (;;) {
1197                 p_event = efx_event(channel, read_ptr);
1198                 event = *p_event;
1199
1200                 if (!efx_event_present(&event))
1201                         /* End of events */
1202                         break;
1203
1204                 netif_vdbg(channel->efx, intr, channel->efx->net_dev,
1205                            "channel %d event is "EFX_QWORD_FMT"\n",
1206                            channel->channel, EFX_QWORD_VAL(event));
1207
1208                 /* Clear this event by marking it all ones */
1209                 EFX_SET_QWORD(*p_event);
1210
1211                 ++read_ptr;
1212
1213                 ev_code = EFX_QWORD_FIELD(event, FSF_AZ_EV_CODE);
1214
1215                 switch (ev_code) {
1216                 case FSE_AZ_EV_CODE_RX_EV:
1217                         efx_handle_rx_event(channel, &event);
1218                         if (++spent == budget)
1219                                 goto out;
1220                         break;
1221                 case FSE_AZ_EV_CODE_TX_EV:
1222                         tx_packets += efx_handle_tx_event(channel, &event);
1223                         if (tx_packets > efx->txq_entries) {
1224                                 spent = budget;
1225                                 goto out;
1226                         }
1227                         break;
1228                 case FSE_AZ_EV_CODE_DRV_GEN_EV:
1229                         efx_handle_generated_event(channel, &event);
1230                         break;
1231                 case FSE_AZ_EV_CODE_DRIVER_EV:
1232                         efx_handle_driver_event(channel, &event);
1233                         break;
1234                 case FSE_CZ_EV_CODE_USER_EV:
1235                         efx_sriov_event(channel, &event);
1236                         break;
1237                 case FSE_CZ_EV_CODE_MCDI_EV:
1238                         efx_mcdi_process_event(channel, &event);
1239                         break;
1240                 case FSE_AZ_EV_CODE_GLOBAL_EV:
1241                         if (efx->type->handle_global_event &&
1242                             efx->type->handle_global_event(channel, &event))
1243                                 break;
1244                         /* else fall through */
1245                 default:
1246                         netif_err(channel->efx, hw, channel->efx->net_dev,
1247                                   "channel %d unknown event type %d (data "
1248                                   EFX_QWORD_FMT ")\n", channel->channel,
1249                                   ev_code, EFX_QWORD_VAL(event));
1250                 }
1251         }
1252
1253 out:
1254         channel->eventq_read_ptr = read_ptr;
1255         return spent;
1256 }
1257
1258 /* Check whether an event is present in the eventq at the current
1259  * read pointer.  Only useful for self-test.
1260  */
1261 bool efx_nic_event_present(struct efx_channel *channel)
1262 {
1263         return efx_event_present(efx_event(channel, channel->eventq_read_ptr));
1264 }
1265
1266 /* Allocate buffer table entries for event queue */
1267 int efx_nic_probe_eventq(struct efx_channel *channel)
1268 {
1269         struct efx_nic *efx = channel->efx;
1270         unsigned entries;
1271
1272         entries = channel->eventq_mask + 1;
1273         return efx_alloc_special_buffer(efx, &channel->eventq,
1274                                         entries * sizeof(efx_qword_t));
1275 }
1276
1277 void efx_nic_init_eventq(struct efx_channel *channel)
1278 {
1279         efx_oword_t reg;
1280         struct efx_nic *efx = channel->efx;
1281
1282         netif_dbg(efx, hw, efx->net_dev,
1283                   "channel %d event queue in special buffers %d-%d\n",
1284                   channel->channel, channel->eventq.index,
1285                   channel->eventq.index + channel->eventq.entries - 1);
1286
1287         if (efx_nic_rev(efx) >= EFX_REV_SIENA_A0) {
1288                 EFX_POPULATE_OWORD_3(reg,
1289                                      FRF_CZ_TIMER_Q_EN, 1,
1290                                      FRF_CZ_HOST_NOTIFY_MODE, 0,
1291                                      FRF_CZ_TIMER_MODE, FFE_CZ_TIMER_MODE_DIS);
1292                 efx_writeo_table(efx, &reg, FR_BZ_TIMER_TBL, channel->channel);
1293         }
1294
1295         /* Pin event queue buffer */
1296         efx_init_special_buffer(efx, &channel->eventq);
1297
1298         /* Fill event queue with all ones (i.e. empty events) */
1299         memset(channel->eventq.addr, 0xff, channel->eventq.len);
1300
1301         /* Push event queue to card */
1302         EFX_POPULATE_OWORD_3(reg,
1303                              FRF_AZ_EVQ_EN, 1,
1304                              FRF_AZ_EVQ_SIZE, __ffs(channel->eventq.entries),
1305                              FRF_AZ_EVQ_BUF_BASE_ID, channel->eventq.index);
1306         efx_writeo_table(efx, &reg, efx->type->evq_ptr_tbl_base,
1307                          channel->channel);
1308
1309         efx->type->push_irq_moderation(channel);
1310 }
1311
1312 void efx_nic_fini_eventq(struct efx_channel *channel)
1313 {
1314         efx_oword_t reg;
1315         struct efx_nic *efx = channel->efx;
1316
1317         /* Remove event queue from card */
1318         EFX_ZERO_OWORD(reg);
1319         efx_writeo_table(efx, &reg, efx->type->evq_ptr_tbl_base,
1320                          channel->channel);
1321         if (efx_nic_rev(efx) >= EFX_REV_SIENA_A0)
1322                 efx_writeo_table(efx, &reg, FR_BZ_TIMER_TBL, channel->channel);
1323
1324         /* Unpin event queue */
1325         efx_fini_special_buffer(efx, &channel->eventq);
1326 }
1327
1328 /* Free buffers backing event queue */
1329 void efx_nic_remove_eventq(struct efx_channel *channel)
1330 {
1331         efx_free_special_buffer(channel->efx, &channel->eventq);
1332 }
1333
1334
1335 void efx_nic_event_test_start(struct efx_channel *channel)
1336 {
1337         channel->event_test_cpu = -1;
1338         smp_wmb();
1339         efx_magic_event(channel, EFX_CHANNEL_MAGIC_TEST(channel));
1340 }
1341
1342 void efx_nic_generate_fill_event(struct efx_rx_queue *rx_queue)
1343 {
1344         efx_magic_event(efx_rx_queue_channel(rx_queue),
1345                         EFX_CHANNEL_MAGIC_FILL(rx_queue));
1346 }
1347
1348 /**************************************************************************
1349  *
1350  * Hardware interrupts
1351  * The hardware interrupt handler does very little work; all the event
1352  * queue processing is carried out by per-channel tasklets.
1353  *
1354  **************************************************************************/
1355
1356 /* Enable/disable/generate interrupts */
1357 static inline void efx_nic_interrupts(struct efx_nic *efx,
1358                                       bool enabled, bool force)
1359 {
1360         efx_oword_t int_en_reg_ker;
1361
1362         EFX_POPULATE_OWORD_3(int_en_reg_ker,
1363                              FRF_AZ_KER_INT_LEVE_SEL, efx->irq_level,
1364                              FRF_AZ_KER_INT_KER, force,
1365                              FRF_AZ_DRV_INT_EN_KER, enabled);
1366         efx_writeo(efx, &int_en_reg_ker, FR_AZ_INT_EN_KER);
1367 }
1368
1369 void efx_nic_enable_interrupts(struct efx_nic *efx)
1370 {
1371         EFX_ZERO_OWORD(*((efx_oword_t *) efx->irq_status.addr));
1372         wmb(); /* Ensure interrupt vector is clear before interrupts enabled */
1373
1374         efx_nic_interrupts(efx, true, false);
1375 }
1376
1377 void efx_nic_disable_interrupts(struct efx_nic *efx)
1378 {
1379         /* Disable interrupts */
1380         efx_nic_interrupts(efx, false, false);
1381 }
1382
1383 /* Generate a test interrupt
1384  * Interrupt must already have been enabled, otherwise nasty things
1385  * may happen.
1386  */
1387 void efx_nic_irq_test_start(struct efx_nic *efx)
1388 {
1389         efx->last_irq_cpu = -1;
1390         smp_wmb();
1391         efx_nic_interrupts(efx, true, true);
1392 }
1393
1394 /* Process a fatal interrupt
1395  * Disable bus mastering ASAP and schedule a reset
1396  */
1397 irqreturn_t efx_nic_fatal_interrupt(struct efx_nic *efx)
1398 {
1399         struct falcon_nic_data *nic_data = efx->nic_data;
1400         efx_oword_t *int_ker = efx->irq_status.addr;
1401         efx_oword_t fatal_intr;
1402         int error, mem_perr;
1403
1404         efx_reado(efx, &fatal_intr, FR_AZ_FATAL_INTR_KER);
1405         error = EFX_OWORD_FIELD(fatal_intr, FRF_AZ_FATAL_INTR);
1406
1407         netif_err(efx, hw, efx->net_dev, "SYSTEM ERROR "EFX_OWORD_FMT" status "
1408                   EFX_OWORD_FMT ": %s\n", EFX_OWORD_VAL(*int_ker),
1409                   EFX_OWORD_VAL(fatal_intr),
1410                   error ? "disabling bus mastering" : "no recognised error");
1411
1412         /* If this is a memory parity error dump which blocks are offending */
1413         mem_perr = (EFX_OWORD_FIELD(fatal_intr, FRF_AZ_MEM_PERR_INT_KER) ||
1414                     EFX_OWORD_FIELD(fatal_intr, FRF_AZ_SRM_PERR_INT_KER));
1415         if (mem_perr) {
1416                 efx_oword_t reg;
1417                 efx_reado(efx, &reg, FR_AZ_MEM_STAT);
1418                 netif_err(efx, hw, efx->net_dev,
1419                           "SYSTEM ERROR: memory parity error "EFX_OWORD_FMT"\n",
1420                           EFX_OWORD_VAL(reg));
1421         }
1422
1423         /* Disable both devices */
1424         pci_clear_master(efx->pci_dev);
1425         if (efx_nic_is_dual_func(efx))
1426                 pci_clear_master(nic_data->pci_dev2);
1427         efx_nic_disable_interrupts(efx);
1428
1429         /* Count errors and reset or disable the NIC accordingly */
1430         if (efx->int_error_count == 0 ||
1431             time_after(jiffies, efx->int_error_expire)) {
1432                 efx->int_error_count = 0;
1433                 efx->int_error_expire =
1434                         jiffies + EFX_INT_ERROR_EXPIRE * HZ;
1435         }
1436         if (++efx->int_error_count < EFX_MAX_INT_ERRORS) {
1437                 netif_err(efx, hw, efx->net_dev,
1438                           "SYSTEM ERROR - reset scheduled\n");
1439                 efx_schedule_reset(efx, RESET_TYPE_INT_ERROR);
1440         } else {
1441                 netif_err(efx, hw, efx->net_dev,
1442                           "SYSTEM ERROR - max number of errors seen."
1443                           "NIC will be disabled\n");
1444                 efx_schedule_reset(efx, RESET_TYPE_DISABLE);
1445         }
1446
1447         return IRQ_HANDLED;
1448 }
1449
1450 /* Handle a legacy interrupt
1451  * Acknowledges the interrupt and schedule event queue processing.
1452  */
1453 static irqreturn_t efx_legacy_interrupt(int irq, void *dev_id)
1454 {
1455         struct efx_nic *efx = dev_id;
1456         efx_oword_t *int_ker = efx->irq_status.addr;
1457         irqreturn_t result = IRQ_NONE;
1458         struct efx_channel *channel;
1459         efx_dword_t reg;
1460         u32 queues;
1461         int syserr;
1462
1463         /* Could this be ours?  If interrupts are disabled then the
1464          * channel state may not be valid.
1465          */
1466         if (!efx->legacy_irq_enabled)
1467                 return result;
1468
1469         /* Read the ISR which also ACKs the interrupts */
1470         efx_readd(efx, &reg, FR_BZ_INT_ISR0);
1471         queues = EFX_EXTRACT_DWORD(reg, 0, 31);
1472
1473         /* Handle non-event-queue sources */
1474         if (queues & (1U << efx->irq_level)) {
1475                 syserr = EFX_OWORD_FIELD(*int_ker, FSF_AZ_NET_IVEC_FATAL_INT);
1476                 if (unlikely(syserr))
1477                         return efx_nic_fatal_interrupt(efx);
1478                 efx->last_irq_cpu = raw_smp_processor_id();
1479         }
1480
1481         if (queues != 0) {
1482                 if (EFX_WORKAROUND_15783(efx))
1483                         efx->irq_zero_count = 0;
1484
1485                 /* Schedule processing of any interrupting queues */
1486                 efx_for_each_channel(channel, efx) {
1487                         if (queues & 1)
1488                                 efx_schedule_channel_irq(channel);
1489                         queues >>= 1;
1490                 }
1491                 result = IRQ_HANDLED;
1492
1493         } else if (EFX_WORKAROUND_15783(efx)) {
1494                 efx_qword_t *event;
1495
1496                 /* We can't return IRQ_HANDLED more than once on seeing ISR=0
1497                  * because this might be a shared interrupt. */
1498                 if (efx->irq_zero_count++ == 0)
1499                         result = IRQ_HANDLED;
1500
1501                 /* Ensure we schedule or rearm all event queues */
1502                 efx_for_each_channel(channel, efx) {
1503                         event = efx_event(channel, channel->eventq_read_ptr);
1504                         if (efx_event_present(event))
1505                                 efx_schedule_channel_irq(channel);
1506                         else
1507                                 efx_nic_eventq_read_ack(channel);
1508                 }
1509         }
1510
1511         if (result == IRQ_HANDLED)
1512                 netif_vdbg(efx, intr, efx->net_dev,
1513                            "IRQ %d on CPU %d status " EFX_DWORD_FMT "\n",
1514                            irq, raw_smp_processor_id(), EFX_DWORD_VAL(reg));
1515
1516         return result;
1517 }
1518
1519 /* Handle an MSI interrupt
1520  *
1521  * Handle an MSI hardware interrupt.  This routine schedules event
1522  * queue processing.  No interrupt acknowledgement cycle is necessary.
1523  * Also, we never need to check that the interrupt is for us, since
1524  * MSI interrupts cannot be shared.
1525  */
1526 static irqreturn_t efx_msi_interrupt(int irq, void *dev_id)
1527 {
1528         struct efx_channel *channel = *(struct efx_channel **)dev_id;
1529         struct efx_nic *efx = channel->efx;
1530         efx_oword_t *int_ker = efx->irq_status.addr;
1531         int syserr;
1532
1533         netif_vdbg(efx, intr, efx->net_dev,
1534                    "IRQ %d on CPU %d status " EFX_OWORD_FMT "\n",
1535                    irq, raw_smp_processor_id(), EFX_OWORD_VAL(*int_ker));
1536
1537         /* Handle non-event-queue sources */
1538         if (channel->channel == efx->irq_level) {
1539                 syserr = EFX_OWORD_FIELD(*int_ker, FSF_AZ_NET_IVEC_FATAL_INT);
1540                 if (unlikely(syserr))
1541                         return efx_nic_fatal_interrupt(efx);
1542                 efx->last_irq_cpu = raw_smp_processor_id();
1543         }
1544
1545         /* Schedule processing of the channel */
1546         efx_schedule_channel_irq(channel);
1547
1548         return IRQ_HANDLED;
1549 }
1550
1551
1552 /* Setup RSS indirection table.
1553  * This maps from the hash value of the packet to RXQ
1554  */
1555 void efx_nic_push_rx_indir_table(struct efx_nic *efx)
1556 {
1557         size_t i = 0;
1558         efx_dword_t dword;
1559
1560         if (efx_nic_rev(efx) < EFX_REV_FALCON_B0)
1561                 return;
1562
1563         BUILD_BUG_ON(ARRAY_SIZE(efx->rx_indir_table) !=
1564                      FR_BZ_RX_INDIRECTION_TBL_ROWS);
1565
1566         for (i = 0; i < FR_BZ_RX_INDIRECTION_TBL_ROWS; i++) {
1567                 EFX_POPULATE_DWORD_1(dword, FRF_BZ_IT_QUEUE,
1568                                      efx->rx_indir_table[i]);
1569                 efx_writed_table(efx, &dword, FR_BZ_RX_INDIRECTION_TBL, i);
1570         }
1571 }
1572
1573 /* Hook interrupt handler(s)
1574  * Try MSI and then legacy interrupts.
1575  */
1576 int efx_nic_init_interrupt(struct efx_nic *efx)
1577 {
1578         struct efx_channel *channel;
1579         int rc;
1580
1581         if (!EFX_INT_MODE_USE_MSI(efx)) {
1582                 irq_handler_t handler;
1583                 if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0)
1584                         handler = efx_legacy_interrupt;
1585                 else
1586                         handler = falcon_legacy_interrupt_a1;
1587
1588                 rc = request_irq(efx->legacy_irq, handler, IRQF_SHARED,
1589                                  efx->name, efx);
1590                 if (rc) {
1591                         netif_err(efx, drv, efx->net_dev,
1592                                   "failed to hook legacy IRQ %d\n",
1593                                   efx->pci_dev->irq);
1594                         goto fail1;
1595                 }
1596                 return 0;
1597         }
1598
1599         /* Hook MSI or MSI-X interrupt */
1600         efx_for_each_channel(channel, efx) {
1601                 rc = request_irq(channel->irq, efx_msi_interrupt,
1602                                  IRQF_PROBE_SHARED, /* Not shared */
1603                                  efx->channel_name[channel->channel],
1604                                  &efx->channel[channel->channel]);
1605                 if (rc) {
1606                         netif_err(efx, drv, efx->net_dev,
1607                                   "failed to hook IRQ %d\n", channel->irq);
1608                         goto fail2;
1609                 }
1610         }
1611
1612         return 0;
1613
1614  fail2:
1615         efx_for_each_channel(channel, efx)
1616                 free_irq(channel->irq, &efx->channel[channel->channel]);
1617  fail1:
1618         return rc;
1619 }
1620
1621 void efx_nic_fini_interrupt(struct efx_nic *efx)
1622 {
1623         struct efx_channel *channel;
1624         efx_oword_t reg;
1625
1626         /* Disable MSI/MSI-X interrupts */
1627         efx_for_each_channel(channel, efx) {
1628                 if (channel->irq)
1629                         free_irq(channel->irq, &efx->channel[channel->channel]);
1630         }
1631
1632         /* ACK legacy interrupt */
1633         if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0)
1634                 efx_reado(efx, &reg, FR_BZ_INT_ISR0);
1635         else
1636                 falcon_irq_ack_a1(efx);
1637
1638         /* Disable legacy interrupt */
1639         if (efx->legacy_irq)
1640                 free_irq(efx->legacy_irq, efx);
1641 }
1642
1643 /* Looks at available SRAM resources and works out how many queues we
1644  * can support, and where things like descriptor caches should live.
1645  *
1646  * SRAM is split up as follows:
1647  * 0                          buftbl entries for channels
1648  * efx->vf_buftbl_base        buftbl entries for SR-IOV
1649  * efx->rx_dc_base            RX descriptor caches
1650  * efx->tx_dc_base            TX descriptor caches
1651  */
1652 void efx_nic_dimension_resources(struct efx_nic *efx, unsigned sram_lim_qw)
1653 {
1654         unsigned vi_count, buftbl_min;
1655
1656         /* Account for the buffer table entries backing the datapath channels
1657          * and the descriptor caches for those channels.
1658          */
1659         buftbl_min = ((efx->n_rx_channels * EFX_MAX_DMAQ_SIZE +
1660                        efx->n_tx_channels * EFX_TXQ_TYPES * EFX_MAX_DMAQ_SIZE +
1661                        efx->n_channels * EFX_MAX_EVQ_SIZE)
1662                       * sizeof(efx_qword_t) / EFX_BUF_SIZE);
1663         vi_count = max(efx->n_channels, efx->n_tx_channels * EFX_TXQ_TYPES);
1664
1665 #ifdef CONFIG_SFC_SRIOV
1666         if (efx_sriov_wanted(efx)) {
1667                 unsigned vi_dc_entries, buftbl_free, entries_per_vf, vf_limit;
1668
1669                 efx->vf_buftbl_base = buftbl_min;
1670
1671                 vi_dc_entries = RX_DC_ENTRIES + TX_DC_ENTRIES;
1672                 vi_count = max(vi_count, EFX_VI_BASE);
1673                 buftbl_free = (sram_lim_qw - buftbl_min -
1674                                vi_count * vi_dc_entries);
1675
1676                 entries_per_vf = ((vi_dc_entries + EFX_VF_BUFTBL_PER_VI) *
1677                                   efx_vf_size(efx));
1678                 vf_limit = min(buftbl_free / entries_per_vf,
1679                                (1024U - EFX_VI_BASE) >> efx->vi_scale);
1680
1681                 if (efx->vf_count > vf_limit) {
1682                         netif_err(efx, probe, efx->net_dev,
1683                                   "Reducing VF count from from %d to %d\n",
1684                                   efx->vf_count, vf_limit);
1685                         efx->vf_count = vf_limit;
1686                 }
1687                 vi_count += efx->vf_count * efx_vf_size(efx);
1688         }
1689 #endif
1690
1691         efx->tx_dc_base = sram_lim_qw - vi_count * TX_DC_ENTRIES;
1692         efx->rx_dc_base = efx->tx_dc_base - vi_count * RX_DC_ENTRIES;
1693 }
1694
1695 u32 efx_nic_fpga_ver(struct efx_nic *efx)
1696 {
1697         efx_oword_t altera_build;
1698         efx_reado(efx, &altera_build, FR_AZ_ALTERA_BUILD);
1699         return EFX_OWORD_FIELD(altera_build, FRF_AZ_ALTERA_BUILD_VER);
1700 }
1701
1702 void efx_nic_init_common(struct efx_nic *efx)
1703 {
1704         efx_oword_t temp;
1705
1706         /* Set positions of descriptor caches in SRAM. */
1707         EFX_POPULATE_OWORD_1(temp, FRF_AZ_SRM_TX_DC_BASE_ADR, efx->tx_dc_base);
1708         efx_writeo(efx, &temp, FR_AZ_SRM_TX_DC_CFG);
1709         EFX_POPULATE_OWORD_1(temp, FRF_AZ_SRM_RX_DC_BASE_ADR, efx->rx_dc_base);
1710         efx_writeo(efx, &temp, FR_AZ_SRM_RX_DC_CFG);
1711
1712         /* Set TX descriptor cache size. */
1713         BUILD_BUG_ON(TX_DC_ENTRIES != (8 << TX_DC_ENTRIES_ORDER));
1714         EFX_POPULATE_OWORD_1(temp, FRF_AZ_TX_DC_SIZE, TX_DC_ENTRIES_ORDER);
1715         efx_writeo(efx, &temp, FR_AZ_TX_DC_CFG);
1716
1717         /* Set RX descriptor cache size.  Set low watermark to size-8, as
1718          * this allows most efficient prefetching.
1719          */
1720         BUILD_BUG_ON(RX_DC_ENTRIES != (8 << RX_DC_ENTRIES_ORDER));
1721         EFX_POPULATE_OWORD_1(temp, FRF_AZ_RX_DC_SIZE, RX_DC_ENTRIES_ORDER);
1722         efx_writeo(efx, &temp, FR_AZ_RX_DC_CFG);
1723         EFX_POPULATE_OWORD_1(temp, FRF_AZ_RX_DC_PF_LWM, RX_DC_ENTRIES - 8);
1724         efx_writeo(efx, &temp, FR_AZ_RX_DC_PF_WM);
1725
1726         /* Program INT_KER address */
1727         EFX_POPULATE_OWORD_2(temp,
1728                              FRF_AZ_NORM_INT_VEC_DIS_KER,
1729                              EFX_INT_MODE_USE_MSI(efx),
1730                              FRF_AZ_INT_ADR_KER, efx->irq_status.dma_addr);
1731         efx_writeo(efx, &temp, FR_AZ_INT_ADR_KER);
1732
1733         if (EFX_WORKAROUND_17213(efx) && !EFX_INT_MODE_USE_MSI(efx))
1734                 /* Use an interrupt level unused by event queues */
1735                 efx->irq_level = 0x1f;
1736         else
1737                 /* Use a valid MSI-X vector */
1738                 efx->irq_level = 0;
1739
1740         /* Enable all the genuinely fatal interrupts.  (They are still
1741          * masked by the overall interrupt mask, controlled by
1742          * falcon_interrupts()).
1743          *
1744          * Note: All other fatal interrupts are enabled
1745          */
1746         EFX_POPULATE_OWORD_3(temp,
1747                              FRF_AZ_ILL_ADR_INT_KER_EN, 1,
1748                              FRF_AZ_RBUF_OWN_INT_KER_EN, 1,
1749                              FRF_AZ_TBUF_OWN_INT_KER_EN, 1);
1750         if (efx_nic_rev(efx) >= EFX_REV_SIENA_A0)
1751                 EFX_SET_OWORD_FIELD(temp, FRF_CZ_SRAM_PERR_INT_P_KER_EN, 1);
1752         EFX_INVERT_OWORD(temp);
1753         efx_writeo(efx, &temp, FR_AZ_FATAL_INTR_KER);
1754
1755         efx_nic_push_rx_indir_table(efx);
1756
1757         /* Disable the ugly timer-based TX DMA backoff and allow TX DMA to be
1758          * controlled by the RX FIFO fill level. Set arbitration to one pkt/Q.
1759          */
1760         efx_reado(efx, &temp, FR_AZ_TX_RESERVED);
1761         EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_RX_SPACER, 0xfe);
1762         EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_RX_SPACER_EN, 1);
1763         EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_ONE_PKT_PER_Q, 1);
1764         EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_PUSH_EN, 1);
1765         EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_DIS_NON_IP_EV, 1);
1766         /* Enable SW_EV to inherit in char driver - assume harmless here */
1767         EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_SOFT_EVT_EN, 1);
1768         /* Prefetch threshold 2 => fetch when descriptor cache half empty */
1769         EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_PREF_THRESHOLD, 2);
1770         /* Disable hardware watchdog which can misfire */
1771         EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff);
1772         /* Squash TX of packets of 16 bytes or less */
1773         if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0)
1774                 EFX_SET_OWORD_FIELD(temp, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1);
1775         efx_writeo(efx, &temp, FR_AZ_TX_RESERVED);
1776
1777         if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) {
1778                 EFX_POPULATE_OWORD_4(temp,
1779                                      /* Default values */
1780                                      FRF_BZ_TX_PACE_SB_NOT_AF, 0x15,
1781                                      FRF_BZ_TX_PACE_SB_AF, 0xb,
1782                                      FRF_BZ_TX_PACE_FB_BASE, 0,
1783                                      /* Allow large pace values in the
1784                                       * fast bin. */
1785                                      FRF_BZ_TX_PACE_BIN_TH,
1786                                      FFE_BZ_TX_PACE_RESERVED);
1787                 efx_writeo(efx, &temp, FR_BZ_TX_PACE);
1788         }
1789 }
1790
1791 /* Register dump */
1792
1793 #define REGISTER_REVISION_A     1
1794 #define REGISTER_REVISION_B     2
1795 #define REGISTER_REVISION_C     3
1796 #define REGISTER_REVISION_Z     3       /* latest revision */
1797
1798 struct efx_nic_reg {
1799         u32 offset:24;
1800         u32 min_revision:2, max_revision:2;
1801 };
1802
1803 #define REGISTER(name, min_rev, max_rev) {                              \
1804         FR_ ## min_rev ## max_rev ## _ ## name,                         \
1805         REGISTER_REVISION_ ## min_rev, REGISTER_REVISION_ ## max_rev    \
1806 }
1807 #define REGISTER_AA(name) REGISTER(name, A, A)
1808 #define REGISTER_AB(name) REGISTER(name, A, B)
1809 #define REGISTER_AZ(name) REGISTER(name, A, Z)
1810 #define REGISTER_BB(name) REGISTER(name, B, B)
1811 #define REGISTER_BZ(name) REGISTER(name, B, Z)
1812 #define REGISTER_CZ(name) REGISTER(name, C, Z)
1813
1814 static const struct efx_nic_reg efx_nic_regs[] = {
1815         REGISTER_AZ(ADR_REGION),
1816         REGISTER_AZ(INT_EN_KER),
1817         REGISTER_BZ(INT_EN_CHAR),
1818         REGISTER_AZ(INT_ADR_KER),
1819         REGISTER_BZ(INT_ADR_CHAR),
1820         /* INT_ACK_KER is WO */
1821         /* INT_ISR0 is RC */
1822         REGISTER_AZ(HW_INIT),
1823         REGISTER_CZ(USR_EV_CFG),
1824         REGISTER_AB(EE_SPI_HCMD),
1825         REGISTER_AB(EE_SPI_HADR),
1826         REGISTER_AB(EE_SPI_HDATA),
1827         REGISTER_AB(EE_BASE_PAGE),
1828         REGISTER_AB(EE_VPD_CFG0),
1829         /* EE_VPD_SW_CNTL and EE_VPD_SW_DATA are not used */
1830         /* PMBX_DBG_IADDR and PBMX_DBG_IDATA are indirect */
1831         /* PCIE_CORE_INDIRECT is indirect */
1832         REGISTER_AB(NIC_STAT),
1833         REGISTER_AB(GPIO_CTL),
1834         REGISTER_AB(GLB_CTL),
1835         /* FATAL_INTR_KER and FATAL_INTR_CHAR are partly RC */
1836         REGISTER_BZ(DP_CTRL),
1837         REGISTER_AZ(MEM_STAT),
1838         REGISTER_AZ(CS_DEBUG),
1839         REGISTER_AZ(ALTERA_BUILD),
1840         REGISTER_AZ(CSR_SPARE),
1841         REGISTER_AB(PCIE_SD_CTL0123),
1842         REGISTER_AB(PCIE_SD_CTL45),
1843         REGISTER_AB(PCIE_PCS_CTL_STAT),
1844         /* DEBUG_DATA_OUT is not used */
1845         /* DRV_EV is WO */
1846         REGISTER_AZ(EVQ_CTL),
1847         REGISTER_AZ(EVQ_CNT1),
1848         REGISTER_AZ(EVQ_CNT2),
1849         REGISTER_AZ(BUF_TBL_CFG),
1850         REGISTER_AZ(SRM_RX_DC_CFG),
1851         REGISTER_AZ(SRM_TX_DC_CFG),
1852         REGISTER_AZ(SRM_CFG),
1853         /* BUF_TBL_UPD is WO */
1854         REGISTER_AZ(SRM_UPD_EVQ),
1855         REGISTER_AZ(SRAM_PARITY),
1856         REGISTER_AZ(RX_CFG),
1857         REGISTER_BZ(RX_FILTER_CTL),
1858         /* RX_FLUSH_DESCQ is WO */
1859         REGISTER_AZ(RX_DC_CFG),
1860         REGISTER_AZ(RX_DC_PF_WM),
1861         REGISTER_BZ(RX_RSS_TKEY),
1862         /* RX_NODESC_DROP is RC */
1863         REGISTER_AA(RX_SELF_RST),
1864         /* RX_DEBUG, RX_PUSH_DROP are not used */
1865         REGISTER_CZ(RX_RSS_IPV6_REG1),
1866         REGISTER_CZ(RX_RSS_IPV6_REG2),
1867         REGISTER_CZ(RX_RSS_IPV6_REG3),
1868         /* TX_FLUSH_DESCQ is WO */
1869         REGISTER_AZ(TX_DC_CFG),
1870         REGISTER_AA(TX_CHKSM_CFG),
1871         REGISTER_AZ(TX_CFG),
1872         /* TX_PUSH_DROP is not used */
1873         REGISTER_AZ(TX_RESERVED),
1874         REGISTER_BZ(TX_PACE),
1875         /* TX_PACE_DROP_QID is RC */
1876         REGISTER_BB(TX_VLAN),
1877         REGISTER_BZ(TX_IPFIL_PORTEN),
1878         REGISTER_AB(MD_TXD),
1879         REGISTER_AB(MD_RXD),
1880         REGISTER_AB(MD_CS),
1881         REGISTER_AB(MD_PHY_ADR),
1882         REGISTER_AB(MD_ID),
1883         /* MD_STAT is RC */
1884         REGISTER_AB(MAC_STAT_DMA),
1885         REGISTER_AB(MAC_CTRL),
1886         REGISTER_BB(GEN_MODE),
1887         REGISTER_AB(MAC_MC_HASH_REG0),
1888         REGISTER_AB(MAC_MC_HASH_REG1),
1889         REGISTER_AB(GM_CFG1),
1890         REGISTER_AB(GM_CFG2),
1891         /* GM_IPG and GM_HD are not used */
1892         REGISTER_AB(GM_MAX_FLEN),
1893         /* GM_TEST is not used */
1894         REGISTER_AB(GM_ADR1),
1895         REGISTER_AB(GM_ADR2),
1896         REGISTER_AB(GMF_CFG0),
1897         REGISTER_AB(GMF_CFG1),
1898         REGISTER_AB(GMF_CFG2),
1899         REGISTER_AB(GMF_CFG3),
1900         REGISTER_AB(GMF_CFG4),
1901         REGISTER_AB(GMF_CFG5),
1902         REGISTER_BB(TX_SRC_MAC_CTL),
1903         REGISTER_AB(XM_ADR_LO),
1904         REGISTER_AB(XM_ADR_HI),
1905         REGISTER_AB(XM_GLB_CFG),
1906         REGISTER_AB(XM_TX_CFG),
1907         REGISTER_AB(XM_RX_CFG),
1908         REGISTER_AB(XM_MGT_INT_MASK),
1909         REGISTER_AB(XM_FC),
1910         REGISTER_AB(XM_PAUSE_TIME),
1911         REGISTER_AB(XM_TX_PARAM),
1912         REGISTER_AB(XM_RX_PARAM),
1913         /* XM_MGT_INT_MSK (note no 'A') is RC */
1914         REGISTER_AB(XX_PWR_RST),
1915         REGISTER_AB(XX_SD_CTL),
1916         REGISTER_AB(XX_TXDRV_CTL),
1917         /* XX_PRBS_CTL, XX_PRBS_CHK and XX_PRBS_ERR are not used */
1918         /* XX_CORE_STAT is partly RC */
1919 };
1920
1921 struct efx_nic_reg_table {
1922         u32 offset:24;
1923         u32 min_revision:2, max_revision:2;
1924         u32 step:6, rows:21;
1925 };
1926
1927 #define REGISTER_TABLE_DIMENSIONS(_, offset, min_rev, max_rev, step, rows) { \
1928         offset,                                                         \
1929         REGISTER_REVISION_ ## min_rev, REGISTER_REVISION_ ## max_rev,   \
1930         step, rows                                                      \
1931 }
1932 #define REGISTER_TABLE(name, min_rev, max_rev)                          \
1933         REGISTER_TABLE_DIMENSIONS(                                      \
1934                 name, FR_ ## min_rev ## max_rev ## _ ## name,           \
1935                 min_rev, max_rev,                                       \
1936                 FR_ ## min_rev ## max_rev ## _ ## name ## _STEP,        \
1937                 FR_ ## min_rev ## max_rev ## _ ## name ## _ROWS)
1938 #define REGISTER_TABLE_AA(name) REGISTER_TABLE(name, A, A)
1939 #define REGISTER_TABLE_AZ(name) REGISTER_TABLE(name, A, Z)
1940 #define REGISTER_TABLE_BB(name) REGISTER_TABLE(name, B, B)
1941 #define REGISTER_TABLE_BZ(name) REGISTER_TABLE(name, B, Z)
1942 #define REGISTER_TABLE_BB_CZ(name)                                      \
1943         REGISTER_TABLE_DIMENSIONS(name, FR_BZ_ ## name, B, B,           \
1944                                   FR_BZ_ ## name ## _STEP,              \
1945                                   FR_BB_ ## name ## _ROWS),             \
1946         REGISTER_TABLE_DIMENSIONS(name, FR_BZ_ ## name, C, Z,           \
1947                                   FR_BZ_ ## name ## _STEP,              \
1948                                   FR_CZ_ ## name ## _ROWS)
1949 #define REGISTER_TABLE_CZ(name) REGISTER_TABLE(name, C, Z)
1950
1951 static const struct efx_nic_reg_table efx_nic_reg_tables[] = {
1952         /* DRIVER is not used */
1953         /* EVQ_RPTR, TIMER_COMMAND, USR_EV and {RX,TX}_DESC_UPD are WO */
1954         REGISTER_TABLE_BB(TX_IPFIL_TBL),
1955         REGISTER_TABLE_BB(TX_SRC_MAC_TBL),
1956         REGISTER_TABLE_AA(RX_DESC_PTR_TBL_KER),
1957         REGISTER_TABLE_BB_CZ(RX_DESC_PTR_TBL),
1958         REGISTER_TABLE_AA(TX_DESC_PTR_TBL_KER),
1959         REGISTER_TABLE_BB_CZ(TX_DESC_PTR_TBL),
1960         REGISTER_TABLE_AA(EVQ_PTR_TBL_KER),
1961         REGISTER_TABLE_BB_CZ(EVQ_PTR_TBL),
1962         /* We can't reasonably read all of the buffer table (up to 8MB!).
1963          * However this driver will only use a few entries.  Reading
1964          * 1K entries allows for some expansion of queue count and
1965          * size before we need to change the version. */
1966         REGISTER_TABLE_DIMENSIONS(BUF_FULL_TBL_KER, FR_AA_BUF_FULL_TBL_KER,
1967                                   A, A, 8, 1024),
1968         REGISTER_TABLE_DIMENSIONS(BUF_FULL_TBL, FR_BZ_BUF_FULL_TBL,
1969                                   B, Z, 8, 1024),
1970         REGISTER_TABLE_CZ(RX_MAC_FILTER_TBL0),
1971         REGISTER_TABLE_BB_CZ(TIMER_TBL),
1972         REGISTER_TABLE_BB_CZ(TX_PACE_TBL),
1973         REGISTER_TABLE_BZ(RX_INDIRECTION_TBL),
1974         /* TX_FILTER_TBL0 is huge and not used by this driver */
1975         REGISTER_TABLE_CZ(TX_MAC_FILTER_TBL0),
1976         REGISTER_TABLE_CZ(MC_TREG_SMEM),
1977         /* MSIX_PBA_TABLE is not mapped */
1978         /* SRM_DBG is not mapped (and is redundant with BUF_FLL_TBL) */
1979         REGISTER_TABLE_BZ(RX_FILTER_TBL0),
1980 };
1981
1982 size_t efx_nic_get_regs_len(struct efx_nic *efx)
1983 {
1984         const struct efx_nic_reg *reg;
1985         const struct efx_nic_reg_table *table;
1986         size_t len = 0;
1987
1988         for (reg = efx_nic_regs;
1989              reg < efx_nic_regs + ARRAY_SIZE(efx_nic_regs);
1990              reg++)
1991                 if (efx->type->revision >= reg->min_revision &&
1992                     efx->type->revision <= reg->max_revision)
1993                         len += sizeof(efx_oword_t);
1994
1995         for (table = efx_nic_reg_tables;
1996              table < efx_nic_reg_tables + ARRAY_SIZE(efx_nic_reg_tables);
1997              table++)
1998                 if (efx->type->revision >= table->min_revision &&
1999                     efx->type->revision <= table->max_revision)
2000                         len += table->rows * min_t(size_t, table->step, 16);
2001
2002         return len;
2003 }
2004
2005 void efx_nic_get_regs(struct efx_nic *efx, void *buf)
2006 {
2007         const struct efx_nic_reg *reg;
2008         const struct efx_nic_reg_table *table;
2009
2010         for (reg = efx_nic_regs;
2011              reg < efx_nic_regs + ARRAY_SIZE(efx_nic_regs);
2012              reg++) {
2013                 if (efx->type->revision >= reg->min_revision &&
2014                     efx->type->revision <= reg->max_revision) {
2015                         efx_reado(efx, (efx_oword_t *)buf, reg->offset);
2016                         buf += sizeof(efx_oword_t);
2017                 }
2018         }
2019
2020         for (table = efx_nic_reg_tables;
2021              table < efx_nic_reg_tables + ARRAY_SIZE(efx_nic_reg_tables);
2022              table++) {
2023                 size_t size, i;
2024
2025                 if (!(efx->type->revision >= table->min_revision &&
2026                       efx->type->revision <= table->max_revision))
2027                         continue;
2028
2029                 size = min_t(size_t, table->step, 16);
2030
2031                 for (i = 0; i < table->rows; i++) {
2032                         switch (table->step) {
2033                         case 4: /* 32-bit register or SRAM */
2034                                 efx_readd_table(efx, buf, table->offset, i);
2035                                 break;
2036                         case 8: /* 64-bit SRAM */
2037                                 efx_sram_readq(efx,
2038                                                efx->membase + table->offset,
2039                                                buf, i);
2040                                 break;
2041                         case 16: /* 128-bit register */
2042                                 efx_reado_table(efx, buf, table->offset, i);
2043                                 break;
2044                         case 32: /* 128-bit register, interleaved */
2045                                 efx_reado_table(efx, buf, table->offset, 2 * i);
2046                                 break;
2047                         default:
2048                                 WARN_ON(1);
2049                                 return;
2050                         }
2051                         buf += size;
2052                 }
2053         }
2054 }