Pileus Git - ~andy/linux/blob - drivers/block/rsxx/dma.c

   1 /*
   2 * Filename: dma.c
   3 *
   4 *
   5 * Authors: Joshua Morris <josh.h.morris@us.ibm.com>
   6 *       Philip Kelleher <pjk1939@linux.vnet.ibm.com>
   7 *
   8 * (C) Copyright 2013 IBM Corporation
   9 *
  10 * This program is free software; you can redistribute it and/or
  11 * modify it under the terms of the GNU General Public License as
  12 * published by the Free Software Foundation; either version 2 of the
  13 * License, or (at your option) any later version.
  14 *
  15 * This program is distributed in the hope that it will be useful, but
  16 * WITHOUT ANY WARRANTY; without even the implied warranty of
  17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  18 * General Public License for more details.
  19 *
  20 * You should have received a copy of the GNU General Public License
  21 * along with this program; if not, write to the Free Software Foundation,
  22 * Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  23 */
  24
  25 #include <linux/slab.h>
  26 #include "rsxx_priv.h"
  27
  28 struct rsxx_dma {
  29         struct list_head         list;
  30         u8                       cmd;
  31         unsigned int             laddr;     /* Logical address */
  32         struct {
  33                 u32              off;
  34                 u32              cnt;
  35         } sub_page;
  36         dma_addr_t               dma_addr;
  37         struct page              *page;
  38         unsigned int             pg_off;    /* Page Offset */
  39         rsxx_dma_cb              cb;
  40         void                     *cb_data;
  41 };
  42
  43 /* This timeout is used to detect a stalled DMA channel */
  44 #define DMA_ACTIVITY_TIMEOUT    msecs_to_jiffies(10000)
  45
  46 struct hw_status {
  47         u8      status;
  48         u8      tag;
  49         __le16  count;
  50         __le32  _rsvd2;
  51         __le64  _rsvd3;
  52 } __packed;
  53
  54 enum rsxx_dma_status {
  55         DMA_SW_ERR    = 0x1,
  56         DMA_HW_FAULT  = 0x2,
  57         DMA_CANCELLED = 0x4,
  58 };
  59
  60 struct hw_cmd {
  61         u8      command;
  62         u8      tag;
  63         u8      _rsvd;
  64         u8      sub_page; /* Bit[0:2]: 512byte offset */
  65                           /* Bit[4:6]: 512byte count */
  66         __le32  device_addr;
  67         __le64  host_addr;
  68 } __packed;
  69
  70 enum rsxx_hw_cmd {
  71         HW_CMD_BLK_DISCARD      = 0x70,
  72         HW_CMD_BLK_WRITE        = 0x80,
  73         HW_CMD_BLK_READ         = 0xC0,
  74         HW_CMD_BLK_RECON_READ   = 0xE0,
  75 };
  76
  77 enum rsxx_hw_status {
  78         HW_STATUS_CRC           = 0x01,
  79         HW_STATUS_HARD_ERR      = 0x02,
  80         HW_STATUS_SOFT_ERR      = 0x04,
  81         HW_STATUS_FAULT         = 0x08,
  82 };
  83
  84 static struct kmem_cache *rsxx_dma_pool;
  85
  86 struct dma_tracker {
  87         int                     next_tag;
  88         struct rsxx_dma *dma;
  89 };
  90
  91 #define DMA_TRACKER_LIST_SIZE8 (sizeof(struct dma_tracker_list) + \
  92                 (sizeof(struct dma_tracker) * RSXX_MAX_OUTSTANDING_CMDS))
  93
  94 struct dma_tracker_list {
  95         spinlock_t              lock;
  96         int                     head;
  97         struct dma_tracker      list[0];
  98 };
  99
 100
 101 /*----------------- Misc Utility Functions -------------------*/
 102 static unsigned int rsxx_addr8_to_laddr(u64 addr8, struct rsxx_cardinfo *card)
 103 {
 104         unsigned long long tgt_addr8;
 105
 106         tgt_addr8 = ((addr8 >> card->_stripe.upper_shift) &
 107                       card->_stripe.upper_mask) |
 108                     ((addr8) & card->_stripe.lower_mask);
 109         do_div(tgt_addr8, RSXX_HW_BLK_SIZE);
 110         return tgt_addr8;
 111 }
 112
 113 static unsigned int rsxx_get_dma_tgt(struct rsxx_cardinfo *card, u64 addr8)
 114 {
 115         unsigned int tgt;
 116
 117         tgt = (addr8 >> card->_stripe.target_shift) & card->_stripe.target_mask;
 118
 119         return tgt;
 120 }
 121
 122 void rsxx_dma_queue_reset(struct rsxx_cardinfo *card)
 123 {
 124         /* Reset all DMA Command/Status Queues */
 125         iowrite32(DMA_QUEUE_RESET, card->regmap + RESET);
 126 }
 127
 128 static unsigned int get_dma_size(struct rsxx_dma *dma)
 129 {
 130         if (dma->sub_page.cnt)
 131                 return dma->sub_page.cnt << 9;
 132         else
 133                 return RSXX_HW_BLK_SIZE;
 134 }
 135
 136
 137 /*----------------- DMA Tracker -------------------*/
 138 static void set_tracker_dma(struct dma_tracker_list *trackers,
 139                             int tag,
 140                             struct rsxx_dma *dma)
 141 {
 142         trackers->list[tag].dma = dma;
 143 }
 144
 145 static struct rsxx_dma *get_tracker_dma(struct dma_tracker_list *trackers,
 146                                             int tag)
 147 {
 148         return trackers->list[tag].dma;
 149 }
 150
 151 static int pop_tracker(struct dma_tracker_list *trackers)
 152 {
 153         int tag;
 154
 155         spin_lock(&trackers->lock);
 156         tag = trackers->head;
 157         if (tag != -1) {
 158                 trackers->head = trackers->list[tag].next_tag;
 159                 trackers->list[tag].next_tag = -1;
 160         }
 161         spin_unlock(&trackers->lock);
 162
 163         return tag;
 164 }
 165
 166 static void push_tracker(struct dma_tracker_list *trackers, int tag)
 167 {
 168         spin_lock(&trackers->lock);
 169         trackers->list[tag].next_tag = trackers->head;
 170         trackers->head = tag;
 171         trackers->list[tag].dma = NULL;
 172         spin_unlock(&trackers->lock);
 173 }
 174
 175
 176 /*----------------- Interrupt Coalescing -------------*/
 177 /*
 178  * Interrupt Coalescing Register Format:
 179  * Interrupt Timer (64ns units) [15:0]
 180  * Interrupt Count [24:16]
 181  * Reserved [31:25]
 182 */
 183 #define INTR_COAL_LATENCY_MASK       (0x0000ffff)
 184
 185 #define INTR_COAL_COUNT_SHIFT        16
 186 #define INTR_COAL_COUNT_BITS         9
 187 #define INTR_COAL_COUNT_MASK         (((1 << INTR_COAL_COUNT_BITS) - 1) << \
 188                                         INTR_COAL_COUNT_SHIFT)
 189 #define INTR_COAL_LATENCY_UNITS_NS   64
 190
 191
 192 static u32 dma_intr_coal_val(u32 mode, u32 count, u32 latency)
 193 {
 194         u32 latency_units = latency / INTR_COAL_LATENCY_UNITS_NS;
 195
 196         if (mode == RSXX_INTR_COAL_DISABLED)
 197                 return 0;
 198
 199         return ((count << INTR_COAL_COUNT_SHIFT) & INTR_COAL_COUNT_MASK) |
 200                         (latency_units & INTR_COAL_LATENCY_MASK);
 201
 202 }
 203
 204 static void dma_intr_coal_auto_tune(struct rsxx_cardinfo *card)
 205 {
 206         int i;
 207         u32 q_depth = 0;
 208         u32 intr_coal;
 209
 210         if (card->config.data.intr_coal.mode != RSXX_INTR_COAL_AUTO_TUNE ||
 211             unlikely(card->eeh_state))
 212                 return;
 213
 214         for (i = 0; i < card->n_targets; i++)
 215                 q_depth += atomic_read(&card->ctrl[i].stats.hw_q_depth);
 216
 217         intr_coal = dma_intr_coal_val(card->config.data.intr_coal.mode,
 218                                       q_depth / 2,
 219                                       card->config.data.intr_coal.latency);
 220         iowrite32(intr_coal, card->regmap + INTR_COAL);
 221 }
 222
 223 /*----------------- RSXX DMA Handling -------------------*/
 224 static void rsxx_complete_dma(struct rsxx_dma_ctrl *ctrl,
 225                                   struct rsxx_dma *dma,
 226                                   unsigned int status)
 227 {
 228         if (status & DMA_SW_ERR)
 229                 ctrl->stats.dma_sw_err++;
 230         if (status & DMA_HW_FAULT)
 231                 ctrl->stats.dma_hw_fault++;
 232         if (status & DMA_CANCELLED)
 233                 ctrl->stats.dma_cancelled++;
 234
 235         if (dma->dma_addr)
 236                 pci_unmap_page(ctrl->card->dev, dma->dma_addr,
 237                                get_dma_size(dma),
 238                                dma->cmd == HW_CMD_BLK_WRITE ?
 239                                            PCI_DMA_TODEVICE :
 240                                            PCI_DMA_FROMDEVICE);
 241
 242         if (dma->cb)
 243                 dma->cb(ctrl->card, dma->cb_data, status ? 1 : 0);
 244
 245         kmem_cache_free(rsxx_dma_pool, dma);
 246 }
 247
 248 static void rsxx_requeue_dma(struct rsxx_dma_ctrl *ctrl,
 249                                  struct rsxx_dma *dma)
 250 {
 251         /*
 252          * Requeued DMAs go to the front of the queue so they are issued
 253          * first.
 254          */
 255         spin_lock(&ctrl->queue_lock);
 256         list_add(&dma->list, &ctrl->queue);
 257         spin_unlock(&ctrl->queue_lock);
 258 }
 259
 260 static void rsxx_handle_dma_error(struct rsxx_dma_ctrl *ctrl,
 261                                       struct rsxx_dma *dma,
 262                                       u8 hw_st)
 263 {
 264         unsigned int status = 0;
 265         int requeue_cmd = 0;
 266
 267         dev_dbg(CARD_TO_DEV(ctrl->card),
 268                 "Handling DMA error(cmd x%02x, laddr x%08x st:x%02x)\n",
 269                 dma->cmd, dma->laddr, hw_st);
 270
 271         if (hw_st & HW_STATUS_CRC)
 272                 ctrl->stats.crc_errors++;
 273         if (hw_st & HW_STATUS_HARD_ERR)
 274                 ctrl->stats.hard_errors++;
 275         if (hw_st & HW_STATUS_SOFT_ERR)
 276                 ctrl->stats.soft_errors++;
 277
 278         switch (dma->cmd) {
 279         case HW_CMD_BLK_READ:
 280                 if (hw_st & (HW_STATUS_CRC | HW_STATUS_HARD_ERR)) {
 281                         if (ctrl->card->scrub_hard) {
 282                                 dma->cmd = HW_CMD_BLK_RECON_READ;
 283                                 requeue_cmd = 1;
 284                                 ctrl->stats.reads_retried++;
 285                         } else {
 286                                 status |= DMA_HW_FAULT;
 287                                 ctrl->stats.reads_failed++;
 288                         }
 289                 } else if (hw_st & HW_STATUS_FAULT) {
 290                         status |= DMA_HW_FAULT;
 291                         ctrl->stats.reads_failed++;
 292                 }
 293
 294                 break;
 295         case HW_CMD_BLK_RECON_READ:
 296                 if (hw_st & (HW_STATUS_CRC | HW_STATUS_HARD_ERR)) {
 297                         /* Data could not be reconstructed. */
 298                         status |= DMA_HW_FAULT;
 299                         ctrl->stats.reads_failed++;
 300                 }
 301
 302                 break;
 303         case HW_CMD_BLK_WRITE:
 304                 status |= DMA_HW_FAULT;
 305                 ctrl->stats.writes_failed++;
 306
 307                 break;
 308         case HW_CMD_BLK_DISCARD:
 309                 status |= DMA_HW_FAULT;
 310                 ctrl->stats.discards_failed++;
 311
 312                 break;
 313         default:
 314                 dev_err(CARD_TO_DEV(ctrl->card),
 315                         "Unknown command in DMA!(cmd: x%02x "
 316                            "laddr x%08x st: x%02x\n",
 317                            dma->cmd, dma->laddr, hw_st);
 318                 status |= DMA_SW_ERR;
 319
 320                 break;
 321         }
 322
 323         if (requeue_cmd)
 324                 rsxx_requeue_dma(ctrl, dma);
 325         else
 326                 rsxx_complete_dma(ctrl, dma, status);
 327 }
 328
 329 static void dma_engine_stalled(unsigned long data)
 330 {
 331         struct rsxx_dma_ctrl *ctrl = (struct rsxx_dma_ctrl *)data;
 332
 333         if (atomic_read(&ctrl->stats.hw_q_depth) == 0 ||
 334             unlikely(ctrl->card->eeh_state))
 335                 return;
 336
 337         if (ctrl->cmd.idx != ioread32(ctrl->regmap + SW_CMD_IDX)) {
 338                 /*
 339                  * The dma engine was stalled because the SW_CMD_IDX write
 340                  * was lost. Issue it again to recover.
 341                  */
 342                 dev_warn(CARD_TO_DEV(ctrl->card),
 343                         "SW_CMD_IDX write was lost, re-writing...\n");
 344                 iowrite32(ctrl->cmd.idx, ctrl->regmap + SW_CMD_IDX);
 345                 mod_timer(&ctrl->activity_timer,
 346                           jiffies + DMA_ACTIVITY_TIMEOUT);
 347         } else {
 348                 dev_warn(CARD_TO_DEV(ctrl->card),
 349                         "DMA channel %d has stalled, faulting interface.\n",
 350                         ctrl->id);
 351                 ctrl->card->dma_fault = 1;
 352         }
 353 }
 354
 355 static void rsxx_issue_dmas(struct work_struct *work)
 356 {
 357         struct rsxx_dma_ctrl *ctrl;
 358         struct rsxx_dma *dma;
 359         int tag;
 360         int cmds_pending = 0;
 361         struct hw_cmd *hw_cmd_buf;
 362
 363         ctrl = container_of(work, struct rsxx_dma_ctrl, issue_dma_work);
 364         hw_cmd_buf = ctrl->cmd.buf;
 365
 366         if (unlikely(ctrl->card->halt) ||
 367             unlikely(ctrl->card->eeh_state))
 368                 return;
 369
 370         while (1) {
 371                 spin_lock(&ctrl->queue_lock);
 372                 if (list_empty(&ctrl->queue)) {
 373                         spin_unlock(&ctrl->queue_lock);
 374                         break;
 375                 }
 376                 spin_unlock(&ctrl->queue_lock);
 377
 378                 tag = pop_tracker(ctrl->trackers);
 379                 if (tag == -1)
 380                         break;
 381
 382                 spin_lock(&ctrl->queue_lock);
 383                 dma = list_entry(ctrl->queue.next, struct rsxx_dma, list);
 384                 list_del(&dma->list);
 385                 ctrl->stats.sw_q_depth--;
 386                 spin_unlock(&ctrl->queue_lock);
 387
 388                 /*
 389                  * This will catch any DMAs that slipped in right before the
 390                  * fault, but was queued after all the other DMAs were
 391                  * cancelled.
 392                  */
 393                 if (unlikely(ctrl->card->dma_fault)) {
 394                         push_tracker(ctrl->trackers, tag);
 395                         rsxx_complete_dma(ctrl, dma, DMA_CANCELLED);
 396                         continue;
 397                 }
 398
 399                 set_tracker_dma(ctrl->trackers, tag, dma);
 400                 hw_cmd_buf[ctrl->cmd.idx].command  = dma->cmd;
 401                 hw_cmd_buf[ctrl->cmd.idx].tag      = tag;
 402                 hw_cmd_buf[ctrl->cmd.idx]._rsvd    = 0;
 403                 hw_cmd_buf[ctrl->cmd.idx].sub_page =
 404                                         ((dma->sub_page.cnt & 0x7) << 4) |
 405                                          (dma->sub_page.off & 0x7);
 406
 407                 hw_cmd_buf[ctrl->cmd.idx].device_addr =
 408                                         cpu_to_le32(dma->laddr);
 409
 410                 hw_cmd_buf[ctrl->cmd.idx].host_addr =
 411                                         cpu_to_le64(dma->dma_addr);
 412
 413                 dev_dbg(CARD_TO_DEV(ctrl->card),
 414                         "Issue DMA%d(laddr %d tag %d) to idx %d\n",
 415                         ctrl->id, dma->laddr, tag, ctrl->cmd.idx);
 416
 417                 ctrl->cmd.idx = (ctrl->cmd.idx + 1) & RSXX_CS_IDX_MASK;
 418                 cmds_pending++;
 419
 420                 if (dma->cmd == HW_CMD_BLK_WRITE)
 421                         ctrl->stats.writes_issued++;
 422                 else if (dma->cmd == HW_CMD_BLK_DISCARD)
 423                         ctrl->stats.discards_issued++;
 424                 else
 425                         ctrl->stats.reads_issued++;
 426         }
 427
 428         /* Let HW know we've queued commands. */
 429         if (cmds_pending) {
 430                 atomic_add(cmds_pending, &ctrl->stats.hw_q_depth);
 431                 mod_timer(&ctrl->activity_timer,
 432                           jiffies + DMA_ACTIVITY_TIMEOUT);
 433
 434                 if (unlikely(ctrl->card->eeh_state)) {
 435                         del_timer_sync(&ctrl->activity_timer);
 436                         return;
 437                 }
 438
 439                 iowrite32(ctrl->cmd.idx, ctrl->regmap + SW_CMD_IDX);
 440         }
 441 }
 442
 443 static void rsxx_dma_done(struct work_struct *work)
 444 {
 445         struct rsxx_dma_ctrl *ctrl;
 446         struct rsxx_dma *dma;
 447         unsigned long flags;
 448         u16 count;
 449         u8 status;
 450         u8 tag;
 451         struct hw_status *hw_st_buf;
 452
 453         ctrl = container_of(work, struct rsxx_dma_ctrl, dma_done_work);
 454         hw_st_buf = ctrl->status.buf;
 455
 456         if (unlikely(ctrl->card->halt) ||
 457             unlikely(ctrl->card->dma_fault) ||
 458             unlikely(ctrl->card->eeh_state))
 459                 return;
 460
 461         count = le16_to_cpu(hw_st_buf[ctrl->status.idx].count);
 462
 463         while (count == ctrl->e_cnt) {
 464                 /*
 465                  * The read memory-barrier is necessary to keep aggressive
 466                  * processors/optimizers (such as the PPC Apple G5) from
 467                  * reordering the following status-buffer tag & status read
 468                  * *before* the count read on subsequent iterations of the
 469                  * loop!
 470                  */
 471                 rmb();
 472
 473                 status = hw_st_buf[ctrl->status.idx].status;
 474                 tag    = hw_st_buf[ctrl->status.idx].tag;
 475
 476                 dma = get_tracker_dma(ctrl->trackers, tag);
 477                 if (dma == NULL) {
 478                         spin_lock_irqsave(&ctrl->card->irq_lock, flags);
 479                         rsxx_disable_ier(ctrl->card, CR_INTR_DMA_ALL);
 480                         spin_unlock_irqrestore(&ctrl->card->irq_lock, flags);
 481
 482                         dev_err(CARD_TO_DEV(ctrl->card),
 483                                 "No tracker for tag %d "
 484                                 "(idx %d id %d)\n",
 485                                 tag, ctrl->status.idx, ctrl->id);
 486                         return;
 487                 }
 488
 489                 dev_dbg(CARD_TO_DEV(ctrl->card),
 490                         "Completing DMA%d"
 491                         "(laddr x%x tag %d st: x%x cnt: x%04x) from idx %d.\n",
 492                         ctrl->id, dma->laddr, tag, status, count,
 493                         ctrl->status.idx);
 494
 495                 atomic_dec(&ctrl->stats.hw_q_depth);
 496
 497                 mod_timer(&ctrl->activity_timer,
 498                           jiffies + DMA_ACTIVITY_TIMEOUT);
 499
 500                 if (status)
 501                         rsxx_handle_dma_error(ctrl, dma, status);
 502                 else
 503                         rsxx_complete_dma(ctrl, dma, 0);
 504
 505                 push_tracker(ctrl->trackers, tag);
 506
 507                 ctrl->status.idx = (ctrl->status.idx + 1) &
 508                                    RSXX_CS_IDX_MASK;
 509                 ctrl->e_cnt++;
 510
 511                 count = le16_to_cpu(hw_st_buf[ctrl->status.idx].count);
 512         }
 513
 514         dma_intr_coal_auto_tune(ctrl->card);
 515
 516         if (atomic_read(&ctrl->stats.hw_q_depth) == 0)
 517                 del_timer_sync(&ctrl->activity_timer);
 518
 519         spin_lock_irqsave(&ctrl->card->irq_lock, flags);
 520         rsxx_enable_ier(ctrl->card, CR_INTR_DMA(ctrl->id));
 521         spin_unlock_irqrestore(&ctrl->card->irq_lock, flags);
 522
 523         spin_lock(&ctrl->queue_lock);
 524         if (ctrl->stats.sw_q_depth)
 525                 queue_work(ctrl->issue_wq, &ctrl->issue_dma_work);
 526         spin_unlock(&ctrl->queue_lock);
 527 }
 528
 529 static int rsxx_cleanup_dma_queue(struct rsxx_cardinfo *card,
 530                                       struct list_head *q)
 531 {
 532         struct rsxx_dma *dma;
 533         struct rsxx_dma *tmp;
 534         int cnt = 0;
 535
 536         list_for_each_entry_safe(dma, tmp, q, list) {
 537                 list_del(&dma->list);
 538
 539                 if (dma->dma_addr)
 540                         pci_unmap_page(card->dev, dma->dma_addr,
 541                                        get_dma_size(dma),
 542                                        (dma->cmd == HW_CMD_BLK_WRITE) ?
 543                                        PCI_DMA_TODEVICE :
 544                                        PCI_DMA_FROMDEVICE);
 545                 kmem_cache_free(rsxx_dma_pool, dma);
 546                 cnt++;
 547         }
 548
 549         return cnt;
 550 }
 551
 552 static int rsxx_queue_discard(struct rsxx_cardinfo *card,
 553                                   struct list_head *q,
 554                                   unsigned int laddr,
 555                                   rsxx_dma_cb cb,
 556                                   void *cb_data)
 557 {
 558         struct rsxx_dma *dma;
 559
 560         dma = kmem_cache_alloc(rsxx_dma_pool, GFP_KERNEL);
 561         if (!dma)
 562                 return -ENOMEM;
 563
 564         dma->cmd          = HW_CMD_BLK_DISCARD;
 565         dma->laddr        = laddr;
 566         dma->dma_addr     = 0;
 567         dma->sub_page.off = 0;
 568         dma->sub_page.cnt = 0;
 569         dma->page         = NULL;
 570         dma->pg_off       = 0;
 571         dma->cb           = cb;
 572         dma->cb_data      = cb_data;
 573
 574         dev_dbg(CARD_TO_DEV(card), "Queuing[D] laddr %x\n", dma->laddr);
 575
 576         list_add_tail(&dma->list, q);
 577
 578         return 0;
 579 }
 580
 581 static int rsxx_queue_dma(struct rsxx_cardinfo *card,
 582                               struct list_head *q,
 583                               int dir,
 584                               unsigned int dma_off,
 585                               unsigned int dma_len,
 586                               unsigned int laddr,
 587                               struct page *page,
 588                               unsigned int pg_off,
 589                               rsxx_dma_cb cb,
 590                               void *cb_data)
 591 {
 592         struct rsxx_dma *dma;
 593
 594         dma = kmem_cache_alloc(rsxx_dma_pool, GFP_KERNEL);
 595         if (!dma)
 596                 return -ENOMEM;
 597
 598         dma->dma_addr = pci_map_page(card->dev, page, pg_off, dma_len,
 599                                      dir ? PCI_DMA_TODEVICE :
 600                                      PCI_DMA_FROMDEVICE);
 601         if (!dma->dma_addr) {
 602                 kmem_cache_free(rsxx_dma_pool, dma);
 603                 return -ENOMEM;
 604         }
 605
 606         dma->cmd          = dir ? HW_CMD_BLK_WRITE : HW_CMD_BLK_READ;
 607         dma->laddr        = laddr;
 608         dma->sub_page.off = (dma_off >> 9);
 609         dma->sub_page.cnt = (dma_len >> 9);
 610         dma->page         = page;
 611         dma->pg_off       = pg_off;
 612         dma->cb           = cb;
 613         dma->cb_data      = cb_data;
 614
 615         dev_dbg(CARD_TO_DEV(card),
 616                 "Queuing[%c] laddr %x off %d cnt %d page %p pg_off %d\n",
 617                 dir ? 'W' : 'R', dma->laddr, dma->sub_page.off,
 618                 dma->sub_page.cnt, dma->page, dma->pg_off);
 619
 620         /* Queue the DMA */
 621         list_add_tail(&dma->list, q);
 622
 623         return 0;
 624 }
 625
 626 int rsxx_dma_queue_bio(struct rsxx_cardinfo *card,
 627                            struct bio *bio,
 628                            atomic_t *n_dmas,
 629                            rsxx_dma_cb cb,
 630                            void *cb_data)
 631 {
 632         struct list_head dma_list[RSXX_MAX_TARGETS];
 633         struct bio_vec *bvec;
 634         unsigned long long addr8;
 635         unsigned int laddr;
 636         unsigned int bv_len;
 637         unsigned int bv_off;
 638         unsigned int dma_off;
 639         unsigned int dma_len;
 640         int dma_cnt[RSXX_MAX_TARGETS];
 641         int tgt;
 642         int st;
 643         int i;
 644
 645         addr8 = bio->bi_sector << 9; /* sectors are 512 bytes */
 646         atomic_set(n_dmas, 0);
 647
 648         for (i = 0; i < card->n_targets; i++) {
 649                 INIT_LIST_HEAD(&dma_list[i]);
 650                 dma_cnt[i] = 0;
 651         }
 652
 653         if (bio->bi_rw & REQ_DISCARD) {
 654                 bv_len = bio->bi_size;
 655
 656                 while (bv_len > 0) {
 657                         tgt   = rsxx_get_dma_tgt(card, addr8);
 658                         laddr = rsxx_addr8_to_laddr(addr8, card);
 659
 660                         st = rsxx_queue_discard(card, &dma_list[tgt], laddr,
 661                                                     cb, cb_data);
 662                         if (st)
 663                                 goto bvec_err;
 664
 665                         dma_cnt[tgt]++;
 666                         atomic_inc(n_dmas);
 667                         addr8  += RSXX_HW_BLK_SIZE;
 668                         bv_len -= RSXX_HW_BLK_SIZE;
 669                 }
 670         } else {
 671                 bio_for_each_segment(bvec, bio, i) {
 672                         bv_len = bvec->bv_len;
 673                         bv_off = bvec->bv_offset;
 674
 675                         while (bv_len > 0) {
 676                                 tgt   = rsxx_get_dma_tgt(card, addr8);
 677                                 laddr = rsxx_addr8_to_laddr(addr8, card);
 678                                 dma_off = addr8 & RSXX_HW_BLK_MASK;
 679                                 dma_len = min(bv_len,
 680                                               RSXX_HW_BLK_SIZE - dma_off);
 681
 682                                 st = rsxx_queue_dma(card, &dma_list[tgt],
 683                                                         bio_data_dir(bio),
 684                                                         dma_off, dma_len,
 685                                                         laddr, bvec->bv_page,
 686                                                         bv_off, cb, cb_data);
 687                                 if (st)
 688                                         goto bvec_err;
 689
 690                                 dma_cnt[tgt]++;
 691                                 atomic_inc(n_dmas);
 692                                 addr8  += dma_len;
 693                                 bv_off += dma_len;
 694                                 bv_len -= dma_len;
 695                         }
 696                 }
 697         }
 698
 699         for (i = 0; i < card->n_targets; i++) {
 700                 if (!list_empty(&dma_list[i])) {
 701                         spin_lock(&card->ctrl[i].queue_lock);
 702                         card->ctrl[i].stats.sw_q_depth += dma_cnt[i];
 703                         list_splice_tail(&dma_list[i], &card->ctrl[i].queue);
 704                         spin_unlock(&card->ctrl[i].queue_lock);
 705
 706                         queue_work(card->ctrl[i].issue_wq,
 707                                    &card->ctrl[i].issue_dma_work);
 708                 }
 709         }
 710
 711         return 0;
 712
 713 bvec_err:
 714         for (i = 0; i < card->n_targets; i++)
 715                 rsxx_cleanup_dma_queue(card, &dma_list[i]);
 716
 717         return st;
 718 }
 719
 720
 721 /*----------------- DMA Engine Initialization & Setup -------------------*/
 722 int rsxx_hw_buffers_init(struct pci_dev *dev, struct rsxx_dma_ctrl *ctrl)
 723 {
 724         ctrl->status.buf = pci_alloc_consistent(dev, STATUS_BUFFER_SIZE8,
 725                                 &ctrl->status.dma_addr);
 726         ctrl->cmd.buf = pci_alloc_consistent(dev, COMMAND_BUFFER_SIZE8,
 727                                 &ctrl->cmd.dma_addr);
 728         if (ctrl->status.buf == NULL || ctrl->cmd.buf == NULL)
 729                 return -ENOMEM;
 730
 731         memset(ctrl->status.buf, 0xac, STATUS_BUFFER_SIZE8);
 732         iowrite32(lower_32_bits(ctrl->status.dma_addr),
 733                 ctrl->regmap + SB_ADD_LO);
 734         iowrite32(upper_32_bits(ctrl->status.dma_addr),
 735                 ctrl->regmap + SB_ADD_HI);
 736
 737         memset(ctrl->cmd.buf, 0x83, COMMAND_BUFFER_SIZE8);
 738         iowrite32(lower_32_bits(ctrl->cmd.dma_addr), ctrl->regmap + CB_ADD_LO);
 739         iowrite32(upper_32_bits(ctrl->cmd.dma_addr), ctrl->regmap + CB_ADD_HI);
 740
 741         ctrl->status.idx = ioread32(ctrl->regmap + HW_STATUS_CNT);
 742         if (ctrl->status.idx > RSXX_MAX_OUTSTANDING_CMDS) {
 743                 dev_crit(&dev->dev, "Failed reading status cnt x%x\n",
 744                         ctrl->status.idx);
 745                 return -EINVAL;
 746         }
 747         iowrite32(ctrl->status.idx, ctrl->regmap + HW_STATUS_CNT);
 748         iowrite32(ctrl->status.idx, ctrl->regmap + SW_STATUS_CNT);
 749
 750         ctrl->cmd.idx = ioread32(ctrl->regmap + HW_CMD_IDX);
 751         if (ctrl->cmd.idx > RSXX_MAX_OUTSTANDING_CMDS) {
 752                 dev_crit(&dev->dev, "Failed reading cmd cnt x%x\n",
 753                         ctrl->status.idx);
 754                 return -EINVAL;
 755         }
 756         iowrite32(ctrl->cmd.idx, ctrl->regmap + HW_CMD_IDX);
 757         iowrite32(ctrl->cmd.idx, ctrl->regmap + SW_CMD_IDX);
 758
 759         return 0;
 760 }
 761
 762 static int rsxx_dma_ctrl_init(struct pci_dev *dev,
 763                                   struct rsxx_dma_ctrl *ctrl)
 764 {
 765         int i;
 766         int st;
 767
 768         memset(&ctrl->stats, 0, sizeof(ctrl->stats));
 769
 770         ctrl->trackers = vmalloc(DMA_TRACKER_LIST_SIZE8);
 771         if (!ctrl->trackers)
 772                 return -ENOMEM;
 773
 774         ctrl->trackers->head = 0;
 775         for (i = 0; i < RSXX_MAX_OUTSTANDING_CMDS; i++) {
 776                 ctrl->trackers->list[i].next_tag = i + 1;
 777                 ctrl->trackers->list[i].dma = NULL;
 778         }
 779         ctrl->trackers->list[RSXX_MAX_OUTSTANDING_CMDS-1].next_tag = -1;
 780         spin_lock_init(&ctrl->trackers->lock);
 781
 782         spin_lock_init(&ctrl->queue_lock);
 783         INIT_LIST_HEAD(&ctrl->queue);
 784
 785         setup_timer(&ctrl->activity_timer, dma_engine_stalled,
 786                                         (unsigned long)ctrl);
 787
 788         ctrl->issue_wq = alloc_ordered_workqueue(DRIVER_NAME"_issue", 0);
 789         if (!ctrl->issue_wq)
 790                 return -ENOMEM;
 791
 792         ctrl->done_wq = alloc_ordered_workqueue(DRIVER_NAME"_done", 0);
 793         if (!ctrl->done_wq)
 794                 return -ENOMEM;
 795
 796         INIT_WORK(&ctrl->issue_dma_work, rsxx_issue_dmas);
 797         INIT_WORK(&ctrl->dma_done_work, rsxx_dma_done);
 798
 799         st = rsxx_hw_buffers_init(dev, ctrl);
 800         if (st)
 801                 return st;
 802
 803         return 0;
 804 }
 805
 806 static int rsxx_dma_stripe_setup(struct rsxx_cardinfo *card,
 807                               unsigned int stripe_size8)
 808 {
 809         if (!is_power_of_2(stripe_size8)) {
 810                 dev_err(CARD_TO_DEV(card),
 811                         "stripe_size is NOT a power of 2!\n");
 812                 return -EINVAL;
 813         }
 814
 815         card->_stripe.lower_mask = stripe_size8 - 1;
 816
 817         card->_stripe.upper_mask  = ~(card->_stripe.lower_mask);
 818         card->_stripe.upper_shift = ffs(card->n_targets) - 1;
 819
 820         card->_stripe.target_mask = card->n_targets - 1;
 821         card->_stripe.target_shift = ffs(stripe_size8) - 1;
 822
 823         dev_dbg(CARD_TO_DEV(card), "_stripe.lower_mask   = x%016llx\n",
 824                 card->_stripe.lower_mask);
 825         dev_dbg(CARD_TO_DEV(card), "_stripe.upper_shift  = x%016llx\n",
 826                 card->_stripe.upper_shift);
 827         dev_dbg(CARD_TO_DEV(card), "_stripe.upper_mask   = x%016llx\n",
 828                 card->_stripe.upper_mask);
 829         dev_dbg(CARD_TO_DEV(card), "_stripe.target_mask  = x%016llx\n",
 830                 card->_stripe.target_mask);
 831         dev_dbg(CARD_TO_DEV(card), "_stripe.target_shift = x%016llx\n",
 832                 card->_stripe.target_shift);
 833
 834         return 0;
 835 }
 836
 837 int rsxx_dma_configure(struct rsxx_cardinfo *card)
 838 {
 839         u32 intr_coal;
 840
 841         intr_coal = dma_intr_coal_val(card->config.data.intr_coal.mode,
 842                                       card->config.data.intr_coal.count,
 843                                       card->config.data.intr_coal.latency);
 844         iowrite32(intr_coal, card->regmap + INTR_COAL);
 845
 846         return rsxx_dma_stripe_setup(card, card->config.data.stripe_size);
 847 }
 848
 849 int rsxx_dma_setup(struct rsxx_cardinfo *card)
 850 {
 851         unsigned long flags;
 852         int st;
 853         int i;
 854
 855         dev_info(CARD_TO_DEV(card),
 856                 "Initializing %d DMA targets\n",
 857                 card->n_targets);
 858
 859         /* Regmap is divided up into 4K chunks. One for each DMA channel */
 860         for (i = 0; i < card->n_targets; i++)
 861                 card->ctrl[i].regmap = card->regmap + (i * 4096);
 862
 863         card->dma_fault = 0;
 864
 865         /* Reset the DMA queues */
 866         rsxx_dma_queue_reset(card);
 867
 868         /************* Setup DMA Control *************/
 869         for (i = 0; i < card->n_targets; i++) {
 870                 st = rsxx_dma_ctrl_init(card->dev, &card->ctrl[i]);
 871                 if (st)
 872                         goto failed_dma_setup;
 873
 874                 card->ctrl[i].card = card;
 875                 card->ctrl[i].id = i;
 876         }
 877
 878         card->scrub_hard = 1;
 879
 880         if (card->config_valid)
 881                 rsxx_dma_configure(card);
 882
 883         /* Enable the interrupts after all setup has completed. */
 884         for (i = 0; i < card->n_targets; i++) {
 885                 spin_lock_irqsave(&card->irq_lock, flags);
 886                 rsxx_enable_ier_and_isr(card, CR_INTR_DMA(i));
 887                 spin_unlock_irqrestore(&card->irq_lock, flags);
 888         }
 889
 890         return 0;
 891
 892 failed_dma_setup:
 893         for (i = 0; i < card->n_targets; i++) {
 894                 struct rsxx_dma_ctrl *ctrl = &card->ctrl[i];
 895
 896                 if (ctrl->issue_wq) {
 897                         destroy_workqueue(ctrl->issue_wq);
 898                         ctrl->issue_wq = NULL;
 899                 }
 900
 901                 if (ctrl->done_wq) {
 902                         destroy_workqueue(ctrl->done_wq);
 903                         ctrl->done_wq = NULL;
 904                 }
 905
 906                 if (ctrl->trackers)
 907                         vfree(ctrl->trackers);
 908
 909                 if (ctrl->status.buf)
 910                         pci_free_consistent(card->dev, STATUS_BUFFER_SIZE8,
 911                                             ctrl->status.buf,
 912                                             ctrl->status.dma_addr);
 913                 if (ctrl->cmd.buf)
 914                         pci_free_consistent(card->dev, COMMAND_BUFFER_SIZE8,
 915                                             ctrl->cmd.buf, ctrl->cmd.dma_addr);
 916         }
 917
 918         return st;
 919 }
 920
 921
 922 void rsxx_dma_destroy(struct rsxx_cardinfo *card)
 923 {
 924         struct rsxx_dma_ctrl *ctrl;
 925         struct rsxx_dma *dma;
 926         int i, j;
 927         int cnt = 0;
 928
 929         for (i = 0; i < card->n_targets; i++) {
 930                 ctrl = &card->ctrl[i];
 931
 932                 if (ctrl->issue_wq) {
 933                         destroy_workqueue(ctrl->issue_wq);
 934                         ctrl->issue_wq = NULL;
 935                 }
 936
 937                 if (ctrl->done_wq) {
 938                         destroy_workqueue(ctrl->done_wq);
 939                         ctrl->done_wq = NULL;
 940                 }
 941
 942                 if (timer_pending(&ctrl->activity_timer))
 943                         del_timer_sync(&ctrl->activity_timer);
 944
 945                 /* Clean up the DMA queue */
 946                 spin_lock(&ctrl->queue_lock);
 947                 cnt = rsxx_cleanup_dma_queue(card, &ctrl->queue);
 948                 spin_unlock(&ctrl->queue_lock);
 949
 950                 if (cnt)
 951                         dev_info(CARD_TO_DEV(card),
 952                                 "Freed %d queued DMAs on channel %d\n",
 953                                 cnt, i);
 954
 955                 /* Clean up issued DMAs */
 956                 for (j = 0; j < RSXX_MAX_OUTSTANDING_CMDS; j++) {
 957                         dma = get_tracker_dma(ctrl->trackers, j);
 958                         if (dma) {
 959                                 pci_unmap_page(card->dev, dma->dma_addr,
 960                                                get_dma_size(dma),
 961                                                (dma->cmd == HW_CMD_BLK_WRITE) ?
 962                                                PCI_DMA_TODEVICE :
 963                                                PCI_DMA_FROMDEVICE);
 964                                 kmem_cache_free(rsxx_dma_pool, dma);
 965                                 cnt++;
 966                         }
 967                 }
 968
 969                 if (cnt)
 970                         dev_info(CARD_TO_DEV(card),
 971                                 "Freed %d pending DMAs on channel %d\n",
 972                                 cnt, i);
 973
 974                 vfree(ctrl->trackers);
 975
 976                 pci_free_consistent(card->dev, STATUS_BUFFER_SIZE8,
 977                                     ctrl->status.buf, ctrl->status.dma_addr);
 978                 pci_free_consistent(card->dev, COMMAND_BUFFER_SIZE8,
 979                                     ctrl->cmd.buf, ctrl->cmd.dma_addr);
 980         }
 981 }
 982
 983 void rsxx_eeh_save_issued_dmas(struct rsxx_cardinfo *card)
 984 {
 985         int i;
 986         int j;
 987         int cnt;
 988         struct rsxx_dma *dma;
 989         struct list_head issued_dmas[card->n_targets];
 990
 991         for (i = 0; i < card->n_targets; i++) {
 992                 INIT_LIST_HEAD(&issued_dmas[i]);
 993                 cnt = 0;
 994                 for (j = 0; j < RSXX_MAX_OUTSTANDING_CMDS; j++) {
 995                         dma = get_tracker_dma(card->ctrl[i].trackers, j);
 996                         if (dma == NULL)
 997                                 continue;
 998
 999                         if (dma->cmd == HW_CMD_BLK_WRITE)
1000                                 card->ctrl[i].stats.writes_issued--;
1001                         else if (dma->cmd == HW_CMD_BLK_DISCARD)
1002                                 card->ctrl[i].stats.discards_issued--;
1003                         else
1004                                 card->ctrl[i].stats.reads_issued--;
1005
1006                         list_add_tail(&dma->list, &issued_dmas[i]);
1007                         push_tracker(card->ctrl[i].trackers, j);
1008                         cnt++;
1009                 }
1010
1011                 spin_lock(&card->ctrl[i].queue_lock);
1012                 list_splice(&issued_dmas[i], &card->ctrl[i].queue);
1013
1014                 atomic_sub(cnt, &card->ctrl[i].stats.hw_q_depth);
1015                 card->ctrl[i].stats.sw_q_depth += cnt;
1016                 card->ctrl[i].e_cnt = 0;
1017
1018                 list_for_each_entry(dma, &card->ctrl[i].queue, list) {
1019                         if (dma->dma_addr)
1020                                 pci_unmap_page(card->dev, dma->dma_addr,
1021                                                get_dma_size(dma),
1022                                                dma->cmd == HW_CMD_BLK_WRITE ?
1023                                                PCI_DMA_TODEVICE :
1024                                                PCI_DMA_FROMDEVICE);
1025                 }
1026                 spin_unlock(&card->ctrl[i].queue_lock);
1027         }
1028 }
1029
1030 void rsxx_eeh_cancel_dmas(struct rsxx_cardinfo *card)
1031 {
1032         struct rsxx_dma *dma;
1033         struct rsxx_dma *tmp;
1034         int i;
1035
1036         for (i = 0; i < card->n_targets; i++) {
1037                 spin_lock(&card->ctrl[i].queue_lock);
1038                 list_for_each_entry_safe(dma, tmp, &card->ctrl[i].queue, list) {
1039                         list_del(&dma->list);
1040
1041                         rsxx_complete_dma(&card->ctrl[i], dma, DMA_CANCELLED);
1042                 }
1043                 spin_unlock(&card->ctrl[i].queue_lock);
1044         }
1045 }
1046
1047 int rsxx_eeh_remap_dmas(struct rsxx_cardinfo *card)
1048 {
1049         struct rsxx_dma *dma;
1050         struct rsxx_dma *tmp;
1051         int i;
1052
1053         for (i = 0; i < card->n_targets; i++) {
1054                 spin_lock(&card->ctrl[i].queue_lock);
1055                 list_for_each_entry(dma, &card->ctrl[i].queue, list) {
1056                         dma->dma_addr = pci_map_page(card->dev, dma->page,
1057                                         dma->pg_off, get_dma_size(dma),
1058                                         dma->cmd == HW_CMD_BLK_WRITE ?
1059                                         PCI_DMA_TODEVICE :
1060                                         PCI_DMA_FROMDEVICE);
1061                         if (!dma->dma_addr) {
1062                                 kmem_cache_free(rsxx_dma_pool, dma);
1063                                 return -ENOMEM;
1064                         }
1065                 }
1066                 spin_unlock(&card->ctrl[i].queue_lock);
1067         }
1068
1069         return 0;
1070 }
1071
1072 int rsxx_dma_init(void)
1073 {
1074         rsxx_dma_pool = KMEM_CACHE(rsxx_dma, SLAB_HWCACHE_ALIGN);
1075         if (!rsxx_dma_pool)
1076                 return -ENOMEM;
1077
1078         return 0;
1079 }
1080
1081
1082 void rsxx_dma_cleanup(void)
1083 {
1084         kmem_cache_destroy(rsxx_dma_pool);
1085 }
1086