Pileus Git - ~andy/linux/blob - drivers/scsi/be2iscsi/be_main.c

   1 /**
   2  * Copyright (C) 2005 - 2012 Emulex
   3  * All rights reserved.
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License version 2
   7  * as published by the Free Software Foundation.  The full GNU General
   8  * Public License is included in this distribution in the file called COPYING.
   9  *
  10  * Written by: Jayamohan Kallickal (jayamohan.kallickal@emulex.com)
  11  *
  12  * Contact Information:
  13  * linux-drivers@emulex.com
  14  *
  15  * Emulex
  16  * 3333 Susan Street
  17  * Costa Mesa, CA 92626
  18  */
  19
  20 #include <linux/reboot.h>
  21 #include <linux/delay.h>
  22 #include <linux/slab.h>
  23 #include <linux/interrupt.h>
  24 #include <linux/blkdev.h>
  25 #include <linux/pci.h>
  26 #include <linux/string.h>
  27 #include <linux/kernel.h>
  28 #include <linux/semaphore.h>
  29 #include <linux/iscsi_boot_sysfs.h>
  30 #include <linux/module.h>
  31 #include <linux/bsg-lib.h>
  32
  33 #include <scsi/libiscsi.h>
  34 #include <scsi/scsi_bsg_iscsi.h>
  35 #include <scsi/scsi_netlink.h>
  36 #include <scsi/scsi_transport_iscsi.h>
  37 #include <scsi/scsi_transport.h>
  38 #include <scsi/scsi_cmnd.h>
  39 #include <scsi/scsi_device.h>
  40 #include <scsi/scsi_host.h>
  41 #include <scsi/scsi.h>
  42 #include "be_main.h"
  43 #include "be_iscsi.h"
  44 #include "be_mgmt.h"
  45 #include "be_cmds.h"
  46
  47 static unsigned int be_iopoll_budget = 10;
  48 static unsigned int be_max_phys_size = 64;
  49 static unsigned int enable_msix = 1;
  50
  51 MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
  52 MODULE_DESCRIPTION(DRV_DESC " " BUILD_STR);
  53 MODULE_VERSION(BUILD_STR);
  54 MODULE_AUTHOR("Emulex Corporation");
  55 MODULE_LICENSE("GPL");
  56 module_param(be_iopoll_budget, int, 0);
  57 module_param(enable_msix, int, 0);
  58 module_param(be_max_phys_size, uint, S_IRUGO);
  59 MODULE_PARM_DESC(be_max_phys_size,
  60                 "Maximum Size (In Kilobytes) of physically contiguous "
  61                 "memory that can be allocated. Range is 16 - 128");
  62
  63 #define beiscsi_disp_param(_name)\
  64 ssize_t \
  65 beiscsi_##_name##_disp(struct device *dev,\
  66                         struct device_attribute *attrib, char *buf)     \
  67 {       \
  68         struct Scsi_Host *shost = class_to_shost(dev);\
  69         struct beiscsi_hba *phba = iscsi_host_priv(shost); \
  70         uint32_t param_val = 0; \
  71         param_val = phba->attr_##_name;\
  72         return snprintf(buf, PAGE_SIZE, "%d\n",\
  73                         phba->attr_##_name);\
  74 }
  75
  76 #define beiscsi_change_param(_name, _minval, _maxval, _defaval)\
  77 int \
  78 beiscsi_##_name##_change(struct beiscsi_hba *phba, uint32_t val)\
  79 {\
  80         if (val >= _minval && val <= _maxval) {\
  81                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,\
  82                             "BA_%d : beiscsi_"#_name" updated "\
  83                             "from 0x%x ==> 0x%x\n",\
  84                             phba->attr_##_name, val); \
  85                 phba->attr_##_name = val;\
  86                 return 0;\
  87         } \
  88         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, \
  89                     "BA_%d beiscsi_"#_name" attribute "\
  90                     "cannot be updated to 0x%x, "\
  91                     "range allowed is ["#_minval" - "#_maxval"]\n", val);\
  92                 return -EINVAL;\
  93 }
  94
  95 #define beiscsi_store_param(_name)  \
  96 ssize_t \
  97 beiscsi_##_name##_store(struct device *dev,\
  98                          struct device_attribute *attr, const char *buf,\
  99                          size_t count) \
 100 { \
 101         struct Scsi_Host  *shost = class_to_shost(dev);\
 102         struct beiscsi_hba *phba = iscsi_host_priv(shost);\
 103         uint32_t param_val = 0;\
 104         if (!isdigit(buf[0]))\
 105                 return -EINVAL;\
 106         if (sscanf(buf, "%i", &param_val) != 1)\
 107                 return -EINVAL;\
 108         if (beiscsi_##_name##_change(phba, param_val) == 0) \
 109                 return strlen(buf);\
 110         else \
 111                 return -EINVAL;\
 112 }
 113
 114 #define beiscsi_init_param(_name, _minval, _maxval, _defval) \
 115 int \
 116 beiscsi_##_name##_init(struct beiscsi_hba *phba, uint32_t val) \
 117 { \
 118         if (val >= _minval && val <= _maxval) {\
 119                 phba->attr_##_name = val;\
 120                 return 0;\
 121         } \
 122         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,\
 123                     "BA_%d beiscsi_"#_name" attribute " \
 124                     "cannot be updated to 0x%x, "\
 125                     "range allowed is ["#_minval" - "#_maxval"]\n", val);\
 126         phba->attr_##_name = _defval;\
 127         return -EINVAL;\
 128 }
 129
 130 #define BEISCSI_RW_ATTR(_name, _minval, _maxval, _defval, _descp) \
 131 static uint beiscsi_##_name = _defval;\
 132 module_param(beiscsi_##_name, uint, S_IRUGO);\
 133 MODULE_PARM_DESC(beiscsi_##_name, _descp);\
 134 beiscsi_disp_param(_name)\
 135 beiscsi_change_param(_name, _minval, _maxval, _defval)\
 136 beiscsi_store_param(_name)\
 137 beiscsi_init_param(_name, _minval, _maxval, _defval)\
 138 DEVICE_ATTR(beiscsi_##_name, S_IRUGO | S_IWUSR,\
 139               beiscsi_##_name##_disp, beiscsi_##_name##_store)
 140
 141 /*
 142  * When new log level added update the
 143  * the MAX allowed value for log_enable
 144  */
 145 BEISCSI_RW_ATTR(log_enable, 0x00,
 146                 0xFF, 0x00, "Enable logging Bit Mask\n"
 147                 "\t\t\t\tInitialization Events  : 0x01\n"
 148                 "\t\t\t\tMailbox Events         : 0x02\n"
 149                 "\t\t\t\tMiscellaneous Events   : 0x04\n"
 150                 "\t\t\t\tError Handling         : 0x08\n"
 151                 "\t\t\t\tIO Path Events         : 0x10\n"
 152                 "\t\t\t\tConfiguration Path     : 0x20\n");
 153
 154 DEVICE_ATTR(beiscsi_drvr_ver, S_IRUGO, beiscsi_drvr_ver_disp, NULL);
 155 DEVICE_ATTR(beiscsi_adapter_family, S_IRUGO, beiscsi_adap_family_disp, NULL);
 156 struct device_attribute *beiscsi_attrs[] = {
 157         &dev_attr_beiscsi_log_enable,
 158         &dev_attr_beiscsi_drvr_ver,
 159         &dev_attr_beiscsi_adapter_family,
 160         NULL,
 161 };
 162
 163 static char const *cqe_desc[] = {
 164         "RESERVED_DESC",
 165         "SOL_CMD_COMPLETE",
 166         "SOL_CMD_KILLED_DATA_DIGEST_ERR",
 167         "CXN_KILLED_PDU_SIZE_EXCEEDS_DSL",
 168         "CXN_KILLED_BURST_LEN_MISMATCH",
 169         "CXN_KILLED_AHS_RCVD",
 170         "CXN_KILLED_HDR_DIGEST_ERR",
 171         "CXN_KILLED_UNKNOWN_HDR",
 172         "CXN_KILLED_STALE_ITT_TTT_RCVD",
 173         "CXN_KILLED_INVALID_ITT_TTT_RCVD",
 174         "CXN_KILLED_RST_RCVD",
 175         "CXN_KILLED_TIMED_OUT",
 176         "CXN_KILLED_RST_SENT",
 177         "CXN_KILLED_FIN_RCVD",
 178         "CXN_KILLED_BAD_UNSOL_PDU_RCVD",
 179         "CXN_KILLED_BAD_WRB_INDEX_ERROR",
 180         "CXN_KILLED_OVER_RUN_RESIDUAL",
 181         "CXN_KILLED_UNDER_RUN_RESIDUAL",
 182         "CMD_KILLED_INVALID_STATSN_RCVD",
 183         "CMD_KILLED_INVALID_R2T_RCVD",
 184         "CMD_CXN_KILLED_LUN_INVALID",
 185         "CMD_CXN_KILLED_ICD_INVALID",
 186         "CMD_CXN_KILLED_ITT_INVALID",
 187         "CMD_CXN_KILLED_SEQ_OUTOFORDER",
 188         "CMD_CXN_KILLED_INVALID_DATASN_RCVD",
 189         "CXN_INVALIDATE_NOTIFY",
 190         "CXN_INVALIDATE_INDEX_NOTIFY",
 191         "CMD_INVALIDATED_NOTIFY",
 192         "UNSOL_HDR_NOTIFY",
 193         "UNSOL_DATA_NOTIFY",
 194         "UNSOL_DATA_DIGEST_ERROR_NOTIFY",
 195         "DRIVERMSG_NOTIFY",
 196         "CXN_KILLED_CMND_DATA_NOT_ON_SAME_CONN",
 197         "SOL_CMD_KILLED_DIF_ERR",
 198         "CXN_KILLED_SYN_RCVD",
 199         "CXN_KILLED_IMM_DATA_RCVD"
 200 };
 201
 202 static int beiscsi_slave_configure(struct scsi_device *sdev)
 203 {
 204         blk_queue_max_segment_size(sdev->request_queue, 65536);
 205         return 0;
 206 }
 207
 208 static int beiscsi_eh_abort(struct scsi_cmnd *sc)
 209 {
 210         struct iscsi_cls_session *cls_session;
 211         struct iscsi_task *aborted_task = (struct iscsi_task *)sc->SCp.ptr;
 212         struct beiscsi_io_task *aborted_io_task;
 213         struct iscsi_conn *conn;
 214         struct beiscsi_conn *beiscsi_conn;
 215         struct beiscsi_hba *phba;
 216         struct iscsi_session *session;
 217         struct invalidate_command_table *inv_tbl;
 218         struct be_dma_mem nonemb_cmd;
 219         unsigned int cid, tag, num_invalidate;
 220
 221         cls_session = starget_to_session(scsi_target(sc->device));
 222         session = cls_session->dd_data;
 223
 224         spin_lock_bh(&session->lock);
 225         if (!aborted_task || !aborted_task->sc) {
 226                 /* we raced */
 227                 spin_unlock_bh(&session->lock);
 228                 return SUCCESS;
 229         }
 230
 231         aborted_io_task = aborted_task->dd_data;
 232         if (!aborted_io_task->scsi_cmnd) {
 233                 /* raced or invalid command */
 234                 spin_unlock_bh(&session->lock);
 235                 return SUCCESS;
 236         }
 237         spin_unlock_bh(&session->lock);
 238         conn = aborted_task->conn;
 239         beiscsi_conn = conn->dd_data;
 240         phba = beiscsi_conn->phba;
 241
 242         /* invalidate iocb */
 243         cid = beiscsi_conn->beiscsi_conn_cid;
 244         inv_tbl = phba->inv_tbl;
 245         memset(inv_tbl, 0x0, sizeof(*inv_tbl));
 246         inv_tbl->cid = cid;
 247         inv_tbl->icd = aborted_io_task->psgl_handle->sgl_index;
 248         num_invalidate = 1;
 249         nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
 250                                 sizeof(struct invalidate_commands_params_in),
 251                                 &nonemb_cmd.dma);
 252         if (nonemb_cmd.va == NULL) {
 253                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_EH,
 254                             "BM_%d : Failed to allocate memory for"
 255                             "mgmt_invalidate_icds\n");
 256                 return FAILED;
 257         }
 258         nonemb_cmd.size = sizeof(struct invalidate_commands_params_in);
 259
 260         tag = mgmt_invalidate_icds(phba, inv_tbl, num_invalidate,
 261                                    cid, &nonemb_cmd);
 262         if (!tag) {
 263                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_EH,
 264                             "BM_%d : mgmt_invalidate_icds could not be"
 265                             "submitted\n");
 266                 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
 267                                     nonemb_cmd.va, nonemb_cmd.dma);
 268
 269                 return FAILED;
 270         }
 271
 272         beiscsi_mccq_compl(phba, tag, NULL, nonemb_cmd.va);
 273         pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
 274                             nonemb_cmd.va, nonemb_cmd.dma);
 275         return iscsi_eh_abort(sc);
 276 }
 277
 278 static int beiscsi_eh_device_reset(struct scsi_cmnd *sc)
 279 {
 280         struct iscsi_task *abrt_task;
 281         struct beiscsi_io_task *abrt_io_task;
 282         struct iscsi_conn *conn;
 283         struct beiscsi_conn *beiscsi_conn;
 284         struct beiscsi_hba *phba;
 285         struct iscsi_session *session;
 286         struct iscsi_cls_session *cls_session;
 287         struct invalidate_command_table *inv_tbl;
 288         struct be_dma_mem nonemb_cmd;
 289         unsigned int cid, tag, i, num_invalidate;
 290
 291         /* invalidate iocbs */
 292         cls_session = starget_to_session(scsi_target(sc->device));
 293         session = cls_session->dd_data;
 294         spin_lock_bh(&session->lock);
 295         if (!session->leadconn || session->state != ISCSI_STATE_LOGGED_IN) {
 296                 spin_unlock_bh(&session->lock);
 297                 return FAILED;
 298         }
 299         conn = session->leadconn;
 300         beiscsi_conn = conn->dd_data;
 301         phba = beiscsi_conn->phba;
 302         cid = beiscsi_conn->beiscsi_conn_cid;
 303         inv_tbl = phba->inv_tbl;
 304         memset(inv_tbl, 0x0, sizeof(*inv_tbl) * BE2_CMDS_PER_CXN);
 305         num_invalidate = 0;
 306         for (i = 0; i < conn->session->cmds_max; i++) {
 307                 abrt_task = conn->session->cmds[i];
 308                 abrt_io_task = abrt_task->dd_data;
 309                 if (!abrt_task->sc || abrt_task->state == ISCSI_TASK_FREE)
 310                         continue;
 311
 312                 if (abrt_task->sc->device->lun != abrt_task->sc->device->lun)
 313                         continue;
 314
 315                 inv_tbl->cid = cid;
 316                 inv_tbl->icd = abrt_io_task->psgl_handle->sgl_index;
 317                 num_invalidate++;
 318                 inv_tbl++;
 319         }
 320         spin_unlock_bh(&session->lock);
 321         inv_tbl = phba->inv_tbl;
 322
 323         nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
 324                                 sizeof(struct invalidate_commands_params_in),
 325                                 &nonemb_cmd.dma);
 326         if (nonemb_cmd.va == NULL) {
 327                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_EH,
 328                             "BM_%d : Failed to allocate memory for"
 329                             "mgmt_invalidate_icds\n");
 330                 return FAILED;
 331         }
 332         nonemb_cmd.size = sizeof(struct invalidate_commands_params_in);
 333         memset(nonemb_cmd.va, 0, nonemb_cmd.size);
 334         tag = mgmt_invalidate_icds(phba, inv_tbl, num_invalidate,
 335                                    cid, &nonemb_cmd);
 336         if (!tag) {
 337                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_EH,
 338                             "BM_%d : mgmt_invalidate_icds could not be"
 339                             " submitted\n");
 340                 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
 341                                     nonemb_cmd.va, nonemb_cmd.dma);
 342                 return FAILED;
 343         }
 344
 345         beiscsi_mccq_compl(phba, tag, NULL, nonemb_cmd.va);
 346         pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
 347                             nonemb_cmd.va, nonemb_cmd.dma);
 348         return iscsi_eh_device_reset(sc);
 349 }
 350
 351 static ssize_t beiscsi_show_boot_tgt_info(void *data, int type, char *buf)
 352 {
 353         struct beiscsi_hba *phba = data;
 354         struct mgmt_session_info *boot_sess = &phba->boot_sess;
 355         struct mgmt_conn_info *boot_conn = &boot_sess->conn_list[0];
 356         char *str = buf;
 357         int rc;
 358
 359         switch (type) {
 360         case ISCSI_BOOT_TGT_NAME:
 361                 rc = sprintf(buf, "%.*s\n",
 362                             (int)strlen(boot_sess->target_name),
 363                             (char *)&boot_sess->target_name);
 364                 break;
 365         case ISCSI_BOOT_TGT_IP_ADDR:
 366                 if (boot_conn->dest_ipaddr.ip_type == 0x1)
 367                         rc = sprintf(buf, "%pI4\n",
 368                                 (char *)&boot_conn->dest_ipaddr.addr);
 369                 else
 370                         rc = sprintf(str, "%pI6\n",
 371                                 (char *)&boot_conn->dest_ipaddr.addr);
 372                 break;
 373         case ISCSI_BOOT_TGT_PORT:
 374                 rc = sprintf(str, "%d\n", boot_conn->dest_port);
 375                 break;
 376
 377         case ISCSI_BOOT_TGT_CHAP_NAME:
 378                 rc = sprintf(str,  "%.*s\n",
 379                              boot_conn->negotiated_login_options.auth_data.chap.
 380                              target_chap_name_length,
 381                              (char *)&boot_conn->negotiated_login_options.
 382                              auth_data.chap.target_chap_name);
 383                 break;
 384         case ISCSI_BOOT_TGT_CHAP_SECRET:
 385                 rc = sprintf(str,  "%.*s\n",
 386                              boot_conn->negotiated_login_options.auth_data.chap.
 387                              target_secret_length,
 388                              (char *)&boot_conn->negotiated_login_options.
 389                              auth_data.chap.target_secret);
 390                 break;
 391         case ISCSI_BOOT_TGT_REV_CHAP_NAME:
 392                 rc = sprintf(str,  "%.*s\n",
 393                              boot_conn->negotiated_login_options.auth_data.chap.
 394                              intr_chap_name_length,
 395                              (char *)&boot_conn->negotiated_login_options.
 396                              auth_data.chap.intr_chap_name);
 397                 break;
 398         case ISCSI_BOOT_TGT_REV_CHAP_SECRET:
 399                 rc = sprintf(str,  "%.*s\n",
 400                              boot_conn->negotiated_login_options.auth_data.chap.
 401                              intr_secret_length,
 402                              (char *)&boot_conn->negotiated_login_options.
 403                              auth_data.chap.intr_secret);
 404                 break;
 405         case ISCSI_BOOT_TGT_FLAGS:
 406                 rc = sprintf(str, "2\n");
 407                 break;
 408         case ISCSI_BOOT_TGT_NIC_ASSOC:
 409                 rc = sprintf(str, "0\n");
 410                 break;
 411         default:
 412                 rc = -ENOSYS;
 413                 break;
 414         }
 415         return rc;
 416 }
 417
 418 static ssize_t beiscsi_show_boot_ini_info(void *data, int type, char *buf)
 419 {
 420         struct beiscsi_hba *phba = data;
 421         char *str = buf;
 422         int rc;
 423
 424         switch (type) {
 425         case ISCSI_BOOT_INI_INITIATOR_NAME:
 426                 rc = sprintf(str, "%s\n", phba->boot_sess.initiator_iscsiname);
 427                 break;
 428         default:
 429                 rc = -ENOSYS;
 430                 break;
 431         }
 432         return rc;
 433 }
 434
 435 static ssize_t beiscsi_show_boot_eth_info(void *data, int type, char *buf)
 436 {
 437         struct beiscsi_hba *phba = data;
 438         char *str = buf;
 439         int rc;
 440
 441         switch (type) {
 442         case ISCSI_BOOT_ETH_FLAGS:
 443                 rc = sprintf(str, "2\n");
 444                 break;
 445         case ISCSI_BOOT_ETH_INDEX:
 446                 rc = sprintf(str, "0\n");
 447                 break;
 448         case ISCSI_BOOT_ETH_MAC:
 449                 rc  = beiscsi_get_macaddr(str, phba);
 450                 break;
 451         default:
 452                 rc = -ENOSYS;
 453                 break;
 454         }
 455         return rc;
 456 }
 457
 458
 459 static umode_t beiscsi_tgt_get_attr_visibility(void *data, int type)
 460 {
 461         umode_t rc;
 462
 463         switch (type) {
 464         case ISCSI_BOOT_TGT_NAME:
 465         case ISCSI_BOOT_TGT_IP_ADDR:
 466         case ISCSI_BOOT_TGT_PORT:
 467         case ISCSI_BOOT_TGT_CHAP_NAME:
 468         case ISCSI_BOOT_TGT_CHAP_SECRET:
 469         case ISCSI_BOOT_TGT_REV_CHAP_NAME:
 470         case ISCSI_BOOT_TGT_REV_CHAP_SECRET:
 471         case ISCSI_BOOT_TGT_NIC_ASSOC:
 472         case ISCSI_BOOT_TGT_FLAGS:
 473                 rc = S_IRUGO;
 474                 break;
 475         default:
 476                 rc = 0;
 477                 break;
 478         }
 479         return rc;
 480 }
 481
 482 static umode_t beiscsi_ini_get_attr_visibility(void *data, int type)
 483 {
 484         umode_t rc;
 485
 486         switch (type) {
 487         case ISCSI_BOOT_INI_INITIATOR_NAME:
 488                 rc = S_IRUGO;
 489                 break;
 490         default:
 491                 rc = 0;
 492                 break;
 493         }
 494         return rc;
 495 }
 496
 497
 498 static umode_t beiscsi_eth_get_attr_visibility(void *data, int type)
 499 {
 500         umode_t rc;
 501
 502         switch (type) {
 503         case ISCSI_BOOT_ETH_FLAGS:
 504         case ISCSI_BOOT_ETH_MAC:
 505         case ISCSI_BOOT_ETH_INDEX:
 506                 rc = S_IRUGO;
 507                 break;
 508         default:
 509                 rc = 0;
 510                 break;
 511         }
 512         return rc;
 513 }
 514
 515 /*------------------- PCI Driver operations and data ----------------- */
 516 static DEFINE_PCI_DEVICE_TABLE(beiscsi_pci_id_table) = {
 517         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) },
 518         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID2) },
 519         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) },
 520         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) },
 521         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID3) },
 522         { PCI_DEVICE(ELX_VENDOR_ID, OC_SKH_ID1) },
 523         { 0 }
 524 };
 525 MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
 526
 527
 528 static struct scsi_host_template beiscsi_sht = {
 529         .module = THIS_MODULE,
 530         .name = "Emulex 10Gbe open-iscsi Initiator Driver",
 531         .proc_name = DRV_NAME,
 532         .queuecommand = iscsi_queuecommand,
 533         .change_queue_depth = iscsi_change_queue_depth,
 534         .slave_configure = beiscsi_slave_configure,
 535         .target_alloc = iscsi_target_alloc,
 536         .eh_abort_handler = beiscsi_eh_abort,
 537         .eh_device_reset_handler = beiscsi_eh_device_reset,
 538         .eh_target_reset_handler = iscsi_eh_session_reset,
 539         .shost_attrs = beiscsi_attrs,
 540         .sg_tablesize = BEISCSI_SGLIST_ELEMENTS,
 541         .can_queue = BE2_IO_DEPTH,
 542         .this_id = -1,
 543         .max_sectors = BEISCSI_MAX_SECTORS,
 544         .cmd_per_lun = BEISCSI_CMD_PER_LUN,
 545         .use_clustering = ENABLE_CLUSTERING,
 546         .vendor_id = SCSI_NL_VID_TYPE_PCI | BE_VENDOR_ID,
 547
 548 };
 549
 550 static struct scsi_transport_template *beiscsi_scsi_transport;
 551
 552 static struct beiscsi_hba *beiscsi_hba_alloc(struct pci_dev *pcidev)
 553 {
 554         struct beiscsi_hba *phba;
 555         struct Scsi_Host *shost;
 556
 557         shost = iscsi_host_alloc(&beiscsi_sht, sizeof(*phba), 0);
 558         if (!shost) {
 559                 dev_err(&pcidev->dev,
 560                         "beiscsi_hba_alloc - iscsi_host_alloc failed\n");
 561                 return NULL;
 562         }
 563         shost->dma_boundary = pcidev->dma_mask;
 564         shost->max_id = BE2_MAX_SESSIONS;
 565         shost->max_channel = 0;
 566         shost->max_cmd_len = BEISCSI_MAX_CMD_LEN;
 567         shost->max_lun = BEISCSI_NUM_MAX_LUN;
 568         shost->transportt = beiscsi_scsi_transport;
 569         phba = iscsi_host_priv(shost);
 570         memset(phba, 0, sizeof(*phba));
 571         phba->shost = shost;
 572         phba->pcidev = pci_dev_get(pcidev);
 573         pci_set_drvdata(pcidev, phba);
 574         phba->interface_handle = 0xFFFFFFFF;
 575
 576         if (iscsi_host_add(shost, &phba->pcidev->dev))
 577                 goto free_devices;
 578
 579         return phba;
 580
 581 free_devices:
 582         pci_dev_put(phba->pcidev);
 583         iscsi_host_free(phba->shost);
 584         return NULL;
 585 }
 586
 587 static void beiscsi_unmap_pci_function(struct beiscsi_hba *phba)
 588 {
 589         if (phba->csr_va) {
 590                 iounmap(phba->csr_va);
 591                 phba->csr_va = NULL;
 592         }
 593         if (phba->db_va) {
 594                 iounmap(phba->db_va);
 595                 phba->db_va = NULL;
 596         }
 597         if (phba->pci_va) {
 598                 iounmap(phba->pci_va);
 599                 phba->pci_va = NULL;
 600         }
 601 }
 602
 603 static int beiscsi_map_pci_bars(struct beiscsi_hba *phba,
 604                                 struct pci_dev *pcidev)
 605 {
 606         u8 __iomem *addr;
 607         int pcicfg_reg;
 608
 609         addr = ioremap_nocache(pci_resource_start(pcidev, 2),
 610                                pci_resource_len(pcidev, 2));
 611         if (addr == NULL)
 612                 return -ENOMEM;
 613         phba->ctrl.csr = addr;
 614         phba->csr_va = addr;
 615         phba->csr_pa.u.a64.address = pci_resource_start(pcidev, 2);
 616
 617         addr = ioremap_nocache(pci_resource_start(pcidev, 4), 128 * 1024);
 618         if (addr == NULL)
 619                 goto pci_map_err;
 620         phba->ctrl.db = addr;
 621         phba->db_va = addr;
 622         phba->db_pa.u.a64.address =  pci_resource_start(pcidev, 4);
 623
 624         if (phba->generation == BE_GEN2)
 625                 pcicfg_reg = 1;
 626         else
 627                 pcicfg_reg = 0;
 628
 629         addr = ioremap_nocache(pci_resource_start(pcidev, pcicfg_reg),
 630                                pci_resource_len(pcidev, pcicfg_reg));
 631
 632         if (addr == NULL)
 633                 goto pci_map_err;
 634         phba->ctrl.pcicfg = addr;
 635         phba->pci_va = addr;
 636         phba->pci_pa.u.a64.address = pci_resource_start(pcidev, pcicfg_reg);
 637         return 0;
 638
 639 pci_map_err:
 640         beiscsi_unmap_pci_function(phba);
 641         return -ENOMEM;
 642 }
 643
 644 static int beiscsi_enable_pci(struct pci_dev *pcidev)
 645 {
 646         int ret;
 647
 648         ret = pci_enable_device(pcidev);
 649         if (ret) {
 650                 dev_err(&pcidev->dev,
 651                         "beiscsi_enable_pci - enable device failed\n");
 652                 return ret;
 653         }
 654
 655         pci_set_master(pcidev);
 656         if (pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(64))) {
 657                 ret = pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(32));
 658                 if (ret) {
 659                         dev_err(&pcidev->dev, "Could not set PCI DMA Mask\n");
 660                         pci_disable_device(pcidev);
 661                         return ret;
 662                 }
 663         }
 664         return 0;
 665 }
 666
 667 static int be_ctrl_init(struct beiscsi_hba *phba, struct pci_dev *pdev)
 668 {
 669         struct be_ctrl_info *ctrl = &phba->ctrl;
 670         struct be_dma_mem *mbox_mem_alloc = &ctrl->mbox_mem_alloced;
 671         struct be_dma_mem *mbox_mem_align = &ctrl->mbox_mem;
 672         int status = 0;
 673
 674         ctrl->pdev = pdev;
 675         status = beiscsi_map_pci_bars(phba, pdev);
 676         if (status)
 677                 return status;
 678         mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16;
 679         mbox_mem_alloc->va = pci_alloc_consistent(pdev,
 680                                                   mbox_mem_alloc->size,
 681                                                   &mbox_mem_alloc->dma);
 682         if (!mbox_mem_alloc->va) {
 683                 beiscsi_unmap_pci_function(phba);
 684                 return -ENOMEM;
 685         }
 686
 687         mbox_mem_align->size = sizeof(struct be_mcc_mailbox);
 688         mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16);
 689         mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16);
 690         memset(mbox_mem_align->va, 0, sizeof(struct be_mcc_mailbox));
 691         spin_lock_init(&ctrl->mbox_lock);
 692         spin_lock_init(&phba->ctrl.mcc_lock);
 693         spin_lock_init(&phba->ctrl.mcc_cq_lock);
 694
 695         return status;
 696 }
 697
 698 static void beiscsi_get_params(struct beiscsi_hba *phba)
 699 {
 700         phba->params.ios_per_ctrl = (phba->fw_config.iscsi_icd_count
 701                                     - (phba->fw_config.iscsi_cid_count
 702                                     + BE2_TMFS
 703                                     + BE2_NOPOUT_REQ));
 704         phba->params.cxns_per_ctrl = phba->fw_config.iscsi_cid_count;
 705         phba->params.asyncpdus_per_ctrl = phba->fw_config.iscsi_cid_count * 2;
 706         phba->params.icds_per_ctrl = phba->fw_config.iscsi_icd_count;
 707         phba->params.num_sge_per_io = BE2_SGE;
 708         phba->params.defpdu_hdr_sz = BE2_DEFPDU_HDR_SZ;
 709         phba->params.defpdu_data_sz = BE2_DEFPDU_DATA_SZ;
 710         phba->params.eq_timer = 64;
 711         phba->params.num_eq_entries =
 712             (((BE2_CMDS_PER_CXN * 2 + phba->fw_config.iscsi_cid_count * 2
 713                                     + BE2_TMFS) / 512) + 1) * 512;
 714         phba->params.num_eq_entries = (phba->params.num_eq_entries < 1024)
 715                                 ? 1024 : phba->params.num_eq_entries;
 716         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
 717                     "BM_%d : phba->params.num_eq_entries=%d\n",
 718                     phba->params.num_eq_entries);
 719         phba->params.num_cq_entries =
 720             (((BE2_CMDS_PER_CXN * 2 +  phba->fw_config.iscsi_cid_count * 2
 721                                     + BE2_TMFS) / 512) + 1) * 512;
 722         phba->params.wrbs_per_cxn = 256;
 723 }
 724
 725 static void hwi_ring_eq_db(struct beiscsi_hba *phba,
 726                            unsigned int id, unsigned int clr_interrupt,
 727                            unsigned int num_processed,
 728                            unsigned char rearm, unsigned char event)
 729 {
 730         u32 val = 0;
 731         val |= id & DB_EQ_RING_ID_MASK;
 732         if (rearm)
 733                 val |= 1 << DB_EQ_REARM_SHIFT;
 734         if (clr_interrupt)
 735                 val |= 1 << DB_EQ_CLR_SHIFT;
 736         if (event)
 737                 val |= 1 << DB_EQ_EVNT_SHIFT;
 738         val |= num_processed << DB_EQ_NUM_POPPED_SHIFT;
 739         iowrite32(val, phba->db_va + DB_EQ_OFFSET);
 740 }
 741
 742 /**
 743  * be_isr_mcc - The isr routine of the driver.
 744  * @irq: Not used
 745  * @dev_id: Pointer to host adapter structure
 746  */
 747 static irqreturn_t be_isr_mcc(int irq, void *dev_id)
 748 {
 749         struct beiscsi_hba *phba;
 750         struct be_eq_entry *eqe = NULL;
 751         struct be_queue_info *eq;
 752         struct be_queue_info *mcc;
 753         unsigned int num_eq_processed;
 754         struct be_eq_obj *pbe_eq;
 755         unsigned long flags;
 756
 757         pbe_eq = dev_id;
 758         eq = &pbe_eq->q;
 759         phba =  pbe_eq->phba;
 760         mcc = &phba->ctrl.mcc_obj.cq;
 761         eqe = queue_tail_node(eq);
 762
 763         num_eq_processed = 0;
 764
 765         while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 766                                 & EQE_VALID_MASK) {
 767                 if (((eqe->dw[offsetof(struct amap_eq_entry,
 768                      resource_id) / 32] &
 769                      EQE_RESID_MASK) >> 16) == mcc->id) {
 770                         spin_lock_irqsave(&phba->isr_lock, flags);
 771                         pbe_eq->todo_mcc_cq = true;
 772                         spin_unlock_irqrestore(&phba->isr_lock, flags);
 773                 }
 774                 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 775                 queue_tail_inc(eq);
 776                 eqe = queue_tail_node(eq);
 777                 num_eq_processed++;
 778         }
 779         if (pbe_eq->todo_mcc_cq)
 780                 queue_work(phba->wq, &pbe_eq->work_cqs);
 781         if (num_eq_processed)
 782                 hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 1, 1);
 783
 784         return IRQ_HANDLED;
 785 }
 786
 787 /**
 788  * be_isr_msix - The isr routine of the driver.
 789  * @irq: Not used
 790  * @dev_id: Pointer to host adapter structure
 791  */
 792 static irqreturn_t be_isr_msix(int irq, void *dev_id)
 793 {
 794         struct beiscsi_hba *phba;
 795         struct be_eq_entry *eqe = NULL;
 796         struct be_queue_info *eq;
 797         struct be_queue_info *cq;
 798         unsigned int num_eq_processed;
 799         struct be_eq_obj *pbe_eq;
 800         unsigned long flags;
 801
 802         pbe_eq = dev_id;
 803         eq = &pbe_eq->q;
 804         cq = pbe_eq->cq;
 805         eqe = queue_tail_node(eq);
 806
 807         phba = pbe_eq->phba;
 808         num_eq_processed = 0;
 809         if (blk_iopoll_enabled) {
 810                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 811                                         & EQE_VALID_MASK) {
 812                         if (!blk_iopoll_sched_prep(&pbe_eq->iopoll))
 813                                 blk_iopoll_sched(&pbe_eq->iopoll);
 814
 815                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 816                         queue_tail_inc(eq);
 817                         eqe = queue_tail_node(eq);
 818                         num_eq_processed++;
 819                 }
 820         } else {
 821                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 822                                                 & EQE_VALID_MASK) {
 823                         spin_lock_irqsave(&phba->isr_lock, flags);
 824                         pbe_eq->todo_cq = true;
 825                         spin_unlock_irqrestore(&phba->isr_lock, flags);
 826                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 827                         queue_tail_inc(eq);
 828                         eqe = queue_tail_node(eq);
 829                         num_eq_processed++;
 830                 }
 831
 832                 if (pbe_eq->todo_cq)
 833                         queue_work(phba->wq, &pbe_eq->work_cqs);
 834         }
 835
 836         if (num_eq_processed)
 837                 hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 0, 1);
 838
 839         return IRQ_HANDLED;
 840 }
 841
 842 /**
 843  * be_isr - The isr routine of the driver.
 844  * @irq: Not used
 845  * @dev_id: Pointer to host adapter structure
 846  */
 847 static irqreturn_t be_isr(int irq, void *dev_id)
 848 {
 849         struct beiscsi_hba *phba;
 850         struct hwi_controller *phwi_ctrlr;
 851         struct hwi_context_memory *phwi_context;
 852         struct be_eq_entry *eqe = NULL;
 853         struct be_queue_info *eq;
 854         struct be_queue_info *cq;
 855         struct be_queue_info *mcc;
 856         unsigned long flags, index;
 857         unsigned int num_mcceq_processed, num_ioeq_processed;
 858         struct be_ctrl_info *ctrl;
 859         struct be_eq_obj *pbe_eq;
 860         int isr;
 861
 862         phba = dev_id;
 863         ctrl = &phba->ctrl;
 864         isr = ioread32(ctrl->csr + CEV_ISR0_OFFSET +
 865                        (PCI_FUNC(ctrl->pdev->devfn) * CEV_ISR_SIZE));
 866         if (!isr)
 867                 return IRQ_NONE;
 868
 869         phwi_ctrlr = phba->phwi_ctrlr;
 870         phwi_context = phwi_ctrlr->phwi_ctxt;
 871         pbe_eq = &phwi_context->be_eq[0];
 872
 873         eq = &phwi_context->be_eq[0].q;
 874         mcc = &phba->ctrl.mcc_obj.cq;
 875         index = 0;
 876         eqe = queue_tail_node(eq);
 877
 878         num_ioeq_processed = 0;
 879         num_mcceq_processed = 0;
 880         if (blk_iopoll_enabled) {
 881                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 882                                         & EQE_VALID_MASK) {
 883                         if (((eqe->dw[offsetof(struct amap_eq_entry,
 884                              resource_id) / 32] &
 885                              EQE_RESID_MASK) >> 16) == mcc->id) {
 886                                 spin_lock_irqsave(&phba->isr_lock, flags);
 887                                 pbe_eq->todo_mcc_cq = true;
 888                                 spin_unlock_irqrestore(&phba->isr_lock, flags);
 889                                 num_mcceq_processed++;
 890                         } else {
 891                                 if (!blk_iopoll_sched_prep(&pbe_eq->iopoll))
 892                                         blk_iopoll_sched(&pbe_eq->iopoll);
 893                                 num_ioeq_processed++;
 894                         }
 895                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 896                         queue_tail_inc(eq);
 897                         eqe = queue_tail_node(eq);
 898                 }
 899                 if (num_ioeq_processed || num_mcceq_processed) {
 900                         if (pbe_eq->todo_mcc_cq)
 901                                 queue_work(phba->wq, &pbe_eq->work_cqs);
 902
 903                         if ((num_mcceq_processed) && (!num_ioeq_processed))
 904                                 hwi_ring_eq_db(phba, eq->id, 0,
 905                                               (num_ioeq_processed +
 906                                                num_mcceq_processed) , 1, 1);
 907                         else
 908                                 hwi_ring_eq_db(phba, eq->id, 0,
 909                                                (num_ioeq_processed +
 910                                                 num_mcceq_processed), 0, 1);
 911
 912                         return IRQ_HANDLED;
 913                 } else
 914                         return IRQ_NONE;
 915         } else {
 916                 cq = &phwi_context->be_cq[0];
 917                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 918                                                 & EQE_VALID_MASK) {
 919
 920                         if (((eqe->dw[offsetof(struct amap_eq_entry,
 921                              resource_id) / 32] &
 922                              EQE_RESID_MASK) >> 16) != cq->id) {
 923                                 spin_lock_irqsave(&phba->isr_lock, flags);
 924                                 pbe_eq->todo_mcc_cq = true;
 925                                 spin_unlock_irqrestore(&phba->isr_lock, flags);
 926                         } else {
 927                                 spin_lock_irqsave(&phba->isr_lock, flags);
 928                                 pbe_eq->todo_cq = true;
 929                                 spin_unlock_irqrestore(&phba->isr_lock, flags);
 930                         }
 931                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 932                         queue_tail_inc(eq);
 933                         eqe = queue_tail_node(eq);
 934                         num_ioeq_processed++;
 935                 }
 936                 if (pbe_eq->todo_cq || pbe_eq->todo_mcc_cq)
 937                         queue_work(phba->wq, &pbe_eq->work_cqs);
 938
 939                 if (num_ioeq_processed) {
 940                         hwi_ring_eq_db(phba, eq->id, 0,
 941                                        num_ioeq_processed, 1, 1);
 942                         return IRQ_HANDLED;
 943                 } else
 944                         return IRQ_NONE;
 945         }
 946 }
 947
 948 static int beiscsi_init_irqs(struct beiscsi_hba *phba)
 949 {
 950         struct pci_dev *pcidev = phba->pcidev;
 951         struct hwi_controller *phwi_ctrlr;
 952         struct hwi_context_memory *phwi_context;
 953         int ret, msix_vec, i, j;
 954
 955         phwi_ctrlr = phba->phwi_ctrlr;
 956         phwi_context = phwi_ctrlr->phwi_ctxt;
 957
 958         if (phba->msix_enabled) {
 959                 for (i = 0; i < phba->num_cpus; i++) {
 960                         phba->msi_name[i] = kzalloc(BEISCSI_MSI_NAME,
 961                                                     GFP_KERNEL);
 962                         if (!phba->msi_name[i]) {
 963                                 ret = -ENOMEM;
 964                                 goto free_msix_irqs;
 965                         }
 966
 967                         sprintf(phba->msi_name[i], "beiscsi_%02x_%02x",
 968                                 phba->shost->host_no, i);
 969                         msix_vec = phba->msix_entries[i].vector;
 970                         ret = request_irq(msix_vec, be_isr_msix, 0,
 971                                           phba->msi_name[i],
 972                                           &phwi_context->be_eq[i]);
 973                         if (ret) {
 974                                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
 975                                             "BM_%d : beiscsi_init_irqs-Failed to"
 976                                             "register msix for i = %d\n",
 977                                             i);
 978                                 kfree(phba->msi_name[i]);
 979                                 goto free_msix_irqs;
 980                         }
 981                 }
 982                 phba->msi_name[i] = kzalloc(BEISCSI_MSI_NAME, GFP_KERNEL);
 983                 if (!phba->msi_name[i]) {
 984                         ret = -ENOMEM;
 985                         goto free_msix_irqs;
 986                 }
 987                 sprintf(phba->msi_name[i], "beiscsi_mcc_%02x",
 988                         phba->shost->host_no);
 989                 msix_vec = phba->msix_entries[i].vector;
 990                 ret = request_irq(msix_vec, be_isr_mcc, 0, phba->msi_name[i],
 991                                   &phwi_context->be_eq[i]);
 992                 if (ret) {
 993                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT ,
 994                                     "BM_%d : beiscsi_init_irqs-"
 995                                     "Failed to register beiscsi_msix_mcc\n");
 996                         kfree(phba->msi_name[i]);
 997                         goto free_msix_irqs;
 998                 }
 999
1000         } else {
1001                 ret = request_irq(pcidev->irq, be_isr, IRQF_SHARED,
1002                                   "beiscsi", phba);
1003                 if (ret) {
1004                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
1005                                     "BM_%d : beiscsi_init_irqs-"
1006                                     "Failed to register irq\\n");
1007                         return ret;
1008                 }
1009         }
1010         return 0;
1011 free_msix_irqs:
1012         for (j = i - 1; j >= 0; j--) {
1013                 kfree(phba->msi_name[j]);
1014                 msix_vec = phba->msix_entries[j].vector;
1015                 free_irq(msix_vec, &phwi_context->be_eq[j]);
1016         }
1017         return ret;
1018 }
1019
1020 static void hwi_ring_cq_db(struct beiscsi_hba *phba,
1021                            unsigned int id, unsigned int num_processed,
1022                            unsigned char rearm, unsigned char event)
1023 {
1024         u32 val = 0;
1025         val |= id & DB_CQ_RING_ID_MASK;
1026         if (rearm)
1027                 val |= 1 << DB_CQ_REARM_SHIFT;
1028         val |= num_processed << DB_CQ_NUM_POPPED_SHIFT;
1029         iowrite32(val, phba->db_va + DB_CQ_OFFSET);
1030 }
1031
1032 static unsigned int
1033 beiscsi_process_async_pdu(struct beiscsi_conn *beiscsi_conn,
1034                           struct beiscsi_hba *phba,
1035                           unsigned short cid,
1036                           struct pdu_base *ppdu,
1037                           unsigned long pdu_len,
1038                           void *pbuffer, unsigned long buf_len)
1039 {
1040         struct iscsi_conn *conn = beiscsi_conn->conn;
1041         struct iscsi_session *session = conn->session;
1042         struct iscsi_task *task;
1043         struct beiscsi_io_task *io_task;
1044         struct iscsi_hdr *login_hdr;
1045
1046         switch (ppdu->dw[offsetof(struct amap_pdu_base, opcode) / 32] &
1047                                                 PDUBASE_OPCODE_MASK) {
1048         case ISCSI_OP_NOOP_IN:
1049                 pbuffer = NULL;
1050                 buf_len = 0;
1051                 break;
1052         case ISCSI_OP_ASYNC_EVENT:
1053                 break;
1054         case ISCSI_OP_REJECT:
1055                 WARN_ON(!pbuffer);
1056                 WARN_ON(!(buf_len == 48));
1057                 beiscsi_log(phba, KERN_ERR,
1058                             BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
1059                             "BM_%d : In ISCSI_OP_REJECT\n");
1060                 break;
1061         case ISCSI_OP_LOGIN_RSP:
1062         case ISCSI_OP_TEXT_RSP:
1063                 task = conn->login_task;
1064                 io_task = task->dd_data;
1065                 login_hdr = (struct iscsi_hdr *)ppdu;
1066                 login_hdr->itt = io_task->libiscsi_itt;
1067                 break;
1068         default:
1069                 beiscsi_log(phba, KERN_WARNING,
1070                             BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
1071                             "BM_%d : Unrecognized opcode 0x%x in async msg\n",
1072                             (ppdu->
1073                              dw[offsetof(struct amap_pdu_base, opcode) / 32]
1074                              & PDUBASE_OPCODE_MASK));
1075                 return 1;
1076         }
1077
1078         spin_lock_bh(&session->lock);
1079         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)ppdu, pbuffer, buf_len);
1080         spin_unlock_bh(&session->lock);
1081         return 0;
1082 }
1083
1084 static struct sgl_handle *alloc_io_sgl_handle(struct beiscsi_hba *phba)
1085 {
1086         struct sgl_handle *psgl_handle;
1087
1088         if (phba->io_sgl_hndl_avbl) {
1089                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_IO,
1090                             "BM_%d : In alloc_io_sgl_handle,"
1091                             " io_sgl_alloc_index=%d\n",
1092                             phba->io_sgl_alloc_index);
1093
1094                 psgl_handle = phba->io_sgl_hndl_base[phba->
1095                                                 io_sgl_alloc_index];
1096                 phba->io_sgl_hndl_base[phba->io_sgl_alloc_index] = NULL;
1097                 phba->io_sgl_hndl_avbl--;
1098                 if (phba->io_sgl_alloc_index == (phba->params.
1099                                                  ios_per_ctrl - 1))
1100                         phba->io_sgl_alloc_index = 0;
1101                 else
1102                         phba->io_sgl_alloc_index++;
1103         } else
1104                 psgl_handle = NULL;
1105         return psgl_handle;
1106 }
1107
1108 static void
1109 free_io_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
1110 {
1111         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_IO,
1112                     "BM_%d : In free_,io_sgl_free_index=%d\n",
1113                     phba->io_sgl_free_index);
1114
1115         if (phba->io_sgl_hndl_base[phba->io_sgl_free_index]) {
1116                 /*
1117                  * this can happen if clean_task is called on a task that
1118                  * failed in xmit_task or alloc_pdu.
1119                  */
1120                  beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_IO,
1121                              "BM_%d : Double Free in IO SGL io_sgl_free_index=%d,"
1122                              "value there=%p\n", phba->io_sgl_free_index,
1123                              phba->io_sgl_hndl_base
1124                              [phba->io_sgl_free_index]);
1125                 return;
1126         }
1127         phba->io_sgl_hndl_base[phba->io_sgl_free_index] = psgl_handle;
1128         phba->io_sgl_hndl_avbl++;
1129         if (phba->io_sgl_free_index == (phba->params.ios_per_ctrl - 1))
1130                 phba->io_sgl_free_index = 0;
1131         else
1132                 phba->io_sgl_free_index++;
1133 }
1134
1135 /**
1136  * alloc_wrb_handle - To allocate a wrb handle
1137  * @phba: The hba pointer
1138  * @cid: The cid to use for allocation
1139  *
1140  * This happens under session_lock until submission to chip
1141  */
1142 struct wrb_handle *alloc_wrb_handle(struct beiscsi_hba *phba, unsigned int cid)
1143 {
1144         struct hwi_wrb_context *pwrb_context;
1145         struct hwi_controller *phwi_ctrlr;
1146         struct wrb_handle *pwrb_handle, *pwrb_handle_tmp;
1147
1148         phwi_ctrlr = phba->phwi_ctrlr;
1149         pwrb_context = &phwi_ctrlr->wrb_context[cid];
1150         if (pwrb_context->wrb_handles_available >= 2) {
1151                 pwrb_handle = pwrb_context->pwrb_handle_base[
1152                                             pwrb_context->alloc_index];
1153                 pwrb_context->wrb_handles_available--;
1154                 if (pwrb_context->alloc_index ==
1155                                                 (phba->params.wrbs_per_cxn - 1))
1156                         pwrb_context->alloc_index = 0;
1157                 else
1158                         pwrb_context->alloc_index++;
1159                 pwrb_handle_tmp = pwrb_context->pwrb_handle_base[
1160                                                 pwrb_context->alloc_index];
1161                 pwrb_handle->nxt_wrb_index = pwrb_handle_tmp->wrb_index;
1162         } else
1163                 pwrb_handle = NULL;
1164         return pwrb_handle;
1165 }
1166
1167 /**
1168  * free_wrb_handle - To free the wrb handle back to pool
1169  * @phba: The hba pointer
1170  * @pwrb_context: The context to free from
1171  * @pwrb_handle: The wrb_handle to free
1172  *
1173  * This happens under session_lock until submission to chip
1174  */
1175 static void
1176 free_wrb_handle(struct beiscsi_hba *phba, struct hwi_wrb_context *pwrb_context,
1177                 struct wrb_handle *pwrb_handle)
1178 {
1179         pwrb_context->pwrb_handle_base[pwrb_context->free_index] = pwrb_handle;
1180         pwrb_context->wrb_handles_available++;
1181         if (pwrb_context->free_index == (phba->params.wrbs_per_cxn - 1))
1182                 pwrb_context->free_index = 0;
1183         else
1184                 pwrb_context->free_index++;
1185
1186         beiscsi_log(phba, KERN_INFO,
1187                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
1188                     "BM_%d : FREE WRB: pwrb_handle=%p free_index=0x%x"
1189                     "wrb_handles_available=%d\n",
1190                     pwrb_handle, pwrb_context->free_index,
1191                     pwrb_context->wrb_handles_available);
1192 }
1193
1194 static struct sgl_handle *alloc_mgmt_sgl_handle(struct beiscsi_hba *phba)
1195 {
1196         struct sgl_handle *psgl_handle;
1197
1198         if (phba->eh_sgl_hndl_avbl) {
1199                 psgl_handle = phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index];
1200                 phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index] = NULL;
1201                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG,
1202                             "BM_%d : mgmt_sgl_alloc_index=%d=0x%x\n",
1203                             phba->eh_sgl_alloc_index,
1204                             phba->eh_sgl_alloc_index);
1205
1206                 phba->eh_sgl_hndl_avbl--;
1207                 if (phba->eh_sgl_alloc_index ==
1208                     (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl -
1209                      1))
1210                         phba->eh_sgl_alloc_index = 0;
1211                 else
1212                         phba->eh_sgl_alloc_index++;
1213         } else
1214                 psgl_handle = NULL;
1215         return psgl_handle;
1216 }
1217
1218 void
1219 free_mgmt_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
1220 {
1221
1222         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG,
1223                     "BM_%d : In  free_mgmt_sgl_handle,"
1224                     "eh_sgl_free_index=%d\n",
1225                     phba->eh_sgl_free_index);
1226
1227         if (phba->eh_sgl_hndl_base[phba->eh_sgl_free_index]) {
1228                 /*
1229                  * this can happen if clean_task is called on a task that
1230                  * failed in xmit_task or alloc_pdu.
1231                  */
1232                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_CONFIG,
1233                             "BM_%d : Double Free in eh SGL ,"
1234                             "eh_sgl_free_index=%d\n",
1235                             phba->eh_sgl_free_index);
1236                 return;
1237         }
1238         phba->eh_sgl_hndl_base[phba->eh_sgl_free_index] = psgl_handle;
1239         phba->eh_sgl_hndl_avbl++;
1240         if (phba->eh_sgl_free_index ==
1241             (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl - 1))
1242                 phba->eh_sgl_free_index = 0;
1243         else
1244                 phba->eh_sgl_free_index++;
1245 }
1246
1247 static void
1248 be_complete_io(struct beiscsi_conn *beiscsi_conn,
1249                 struct iscsi_task *task,
1250                 struct common_sol_cqe *csol_cqe)
1251 {
1252         struct beiscsi_io_task *io_task = task->dd_data;
1253         struct be_status_bhs *sts_bhs =
1254                                 (struct be_status_bhs *)io_task->cmd_bhs;
1255         struct iscsi_conn *conn = beiscsi_conn->conn;
1256         unsigned char *sense;
1257         u32 resid = 0, exp_cmdsn, max_cmdsn;
1258         u8 rsp, status, flags;
1259
1260         exp_cmdsn = csol_cqe->exp_cmdsn;
1261         max_cmdsn = (csol_cqe->exp_cmdsn +
1262                      csol_cqe->cmd_wnd - 1);
1263         rsp = csol_cqe->i_resp;
1264         status = csol_cqe->i_sts;
1265         flags = csol_cqe->i_flags;
1266         resid = csol_cqe->res_cnt;
1267
1268         if (!task->sc) {
1269                 if (io_task->scsi_cmnd)
1270                         scsi_dma_unmap(io_task->scsi_cmnd);
1271
1272                 return;
1273         }
1274         task->sc->result = (DID_OK << 16) | status;
1275         if (rsp != ISCSI_STATUS_CMD_COMPLETED) {
1276                 task->sc->result = DID_ERROR << 16;
1277                 goto unmap;
1278         }
1279
1280         /* bidi not initially supported */
1281         if (flags & (ISCSI_FLAG_CMD_UNDERFLOW | ISCSI_FLAG_CMD_OVERFLOW)) {
1282                 if (!status && (flags & ISCSI_FLAG_CMD_OVERFLOW))
1283                         task->sc->result = DID_ERROR << 16;
1284
1285                 if (flags & ISCSI_FLAG_CMD_UNDERFLOW) {
1286                         scsi_set_resid(task->sc, resid);
1287                         if (!status && (scsi_bufflen(task->sc) - resid <
1288                             task->sc->underflow))
1289                                 task->sc->result = DID_ERROR << 16;
1290                 }
1291         }
1292
1293         if (status == SAM_STAT_CHECK_CONDITION) {
1294                 u16 sense_len;
1295                 unsigned short *slen = (unsigned short *)sts_bhs->sense_info;
1296
1297                 sense = sts_bhs->sense_info + sizeof(unsigned short);
1298                 sense_len = be16_to_cpu(*slen);
1299                 memcpy(task->sc->sense_buffer, sense,
1300                        min_t(u16, sense_len, SCSI_SENSE_BUFFERSIZE));
1301         }
1302
1303         if (io_task->cmd_bhs->iscsi_hdr.flags & ISCSI_FLAG_CMD_READ)
1304                 conn->rxdata_octets += resid;
1305 unmap:
1306         scsi_dma_unmap(io_task->scsi_cmnd);
1307         iscsi_complete_scsi_task(task, exp_cmdsn, max_cmdsn);
1308 }
1309
1310 static void
1311 be_complete_logout(struct beiscsi_conn *beiscsi_conn,
1312                     struct iscsi_task *task,
1313                     struct common_sol_cqe *csol_cqe)
1314 {
1315         struct iscsi_logout_rsp *hdr;
1316         struct beiscsi_io_task *io_task = task->dd_data;
1317         struct iscsi_conn *conn = beiscsi_conn->conn;
1318
1319         hdr = (struct iscsi_logout_rsp *)task->hdr;
1320         hdr->opcode = ISCSI_OP_LOGOUT_RSP;
1321         hdr->t2wait = 5;
1322         hdr->t2retain = 0;
1323         hdr->flags = csol_cqe->i_flags;
1324         hdr->response = csol_cqe->i_resp;
1325         hdr->exp_cmdsn = csol_cqe->exp_cmdsn;
1326         hdr->max_cmdsn = (csol_cqe->exp_cmdsn + csol_cqe->cmd_wnd - 1);
1327
1328         hdr->dlength[0] = 0;
1329         hdr->dlength[1] = 0;
1330         hdr->dlength[2] = 0;
1331         hdr->hlength = 0;
1332         hdr->itt = io_task->libiscsi_itt;
1333         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
1334 }
1335
1336 static void
1337 be_complete_tmf(struct beiscsi_conn *beiscsi_conn,
1338                  struct iscsi_task *task,
1339                  struct common_sol_cqe *csol_cqe)
1340 {
1341         struct iscsi_tm_rsp *hdr;
1342         struct iscsi_conn *conn = beiscsi_conn->conn;
1343         struct beiscsi_io_task *io_task = task->dd_data;
1344
1345         hdr = (struct iscsi_tm_rsp *)task->hdr;
1346         hdr->opcode = ISCSI_OP_SCSI_TMFUNC_RSP;
1347         hdr->flags = csol_cqe->i_flags;
1348         hdr->response = csol_cqe->i_resp;
1349         hdr->exp_cmdsn = csol_cqe->exp_cmdsn;
1350         hdr->max_cmdsn = (csol_cqe->exp_cmdsn +
1351                           csol_cqe->cmd_wnd - 1);
1352
1353         hdr->itt = io_task->libiscsi_itt;
1354         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
1355 }
1356
1357 static void
1358 hwi_complete_drvr_msgs(struct beiscsi_conn *beiscsi_conn,
1359                        struct beiscsi_hba *phba, struct sol_cqe *psol)
1360 {
1361         struct hwi_wrb_context *pwrb_context;
1362         struct wrb_handle *pwrb_handle = NULL;
1363         struct hwi_controller *phwi_ctrlr;
1364         struct iscsi_task *task;
1365         struct beiscsi_io_task *io_task;
1366         struct iscsi_conn *conn = beiscsi_conn->conn;
1367         struct iscsi_session *session = conn->session;
1368         uint16_t wrb_index, cid;
1369
1370         phwi_ctrlr = phba->phwi_ctrlr;
1371         if (chip_skh_r(phba->pcidev)) {
1372                 wrb_index = AMAP_GET_BITS(struct amap_it_dmsg_cqe_v2,
1373                                           wrb_idx, psol);
1374                 cid = AMAP_GET_BITS(struct amap_it_dmsg_cqe_v2,
1375                                     cid, psol);
1376         } else {
1377                 wrb_index = AMAP_GET_BITS(struct amap_it_dmsg_cqe,
1378                                           wrb_idx, psol);
1379                 cid = AMAP_GET_BITS(struct amap_it_dmsg_cqe,
1380                                     cid, psol);
1381         }
1382
1383         pwrb_context = &phwi_ctrlr->wrb_context[
1384                         cid - phba->fw_config.iscsi_cid_start];
1385         pwrb_handle = pwrb_context->pwrb_handle_basestd[wrb_index];
1386         task = pwrb_handle->pio_handle;
1387
1388         io_task = task->dd_data;
1389         spin_lock_bh(&phba->mgmt_sgl_lock);
1390         free_mgmt_sgl_handle(phba, io_task->psgl_handle);
1391         spin_unlock_bh(&phba->mgmt_sgl_lock);
1392         spin_lock_bh(&session->lock);
1393         free_wrb_handle(phba, pwrb_context, pwrb_handle);
1394         spin_unlock_bh(&session->lock);
1395 }
1396
1397 static void
1398 be_complete_nopin_resp(struct beiscsi_conn *beiscsi_conn,
1399                         struct iscsi_task *task,
1400                         struct common_sol_cqe *csol_cqe)
1401 {
1402         struct iscsi_nopin *hdr;
1403         struct iscsi_conn *conn = beiscsi_conn->conn;
1404         struct beiscsi_io_task *io_task = task->dd_data;
1405
1406         hdr = (struct iscsi_nopin *)task->hdr;
1407         hdr->flags = csol_cqe->i_flags;
1408         hdr->exp_cmdsn = cpu_to_be32(csol_cqe->exp_cmdsn);
1409         hdr->max_cmdsn = be32_to_cpu(hdr->exp_cmdsn +
1410                          csol_cqe->cmd_wnd - 1);
1411
1412         hdr->opcode = ISCSI_OP_NOOP_IN;
1413         hdr->itt = io_task->libiscsi_itt;
1414         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
1415 }
1416
1417 static void adapter_get_sol_cqe(struct beiscsi_hba *phba,
1418                 struct sol_cqe *psol,
1419                 struct common_sol_cqe *csol_cqe)
1420 {
1421         if (chip_skh_r(phba->pcidev)) {
1422                 csol_cqe->exp_cmdsn = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1423                                                     i_exp_cmd_sn, psol);
1424                 csol_cqe->res_cnt = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1425                                                   i_res_cnt, psol);
1426                 csol_cqe->wrb_index = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1427                                                     wrb_index, psol);
1428                 csol_cqe->cid = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1429                                               cid, psol);
1430                 csol_cqe->hw_sts = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1431                                                  hw_sts, psol);
1432                 csol_cqe->cmd_wnd = AMAP_GET_BITS(struct amap_sol_cqe,
1433                                                   i_cmd_wnd, psol);
1434                 if (AMAP_GET_BITS(struct amap_sol_cqe_v2,
1435                                   cmd_cmpl, psol))
1436                         csol_cqe->i_sts = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1437                                                         i_sts, psol);
1438                 else
1439                         csol_cqe->i_resp = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1440                                                          i_sts, psol);
1441                 if (AMAP_GET_BITS(struct amap_sol_cqe_v2,
1442                                   u, psol))
1443                         csol_cqe->i_flags = ISCSI_FLAG_CMD_UNDERFLOW;
1444
1445                 if (AMAP_GET_BITS(struct amap_sol_cqe_v2,
1446                                   o, psol))
1447                         csol_cqe->i_flags |= ISCSI_FLAG_CMD_OVERFLOW;
1448         } else {
1449                 csol_cqe->exp_cmdsn = AMAP_GET_BITS(struct amap_sol_cqe,
1450                                                     i_exp_cmd_sn, psol);
1451                 csol_cqe->res_cnt = AMAP_GET_BITS(struct amap_sol_cqe,
1452                                                   i_res_cnt, psol);
1453                 csol_cqe->cmd_wnd = AMAP_GET_BITS(struct amap_sol_cqe,
1454                                                   i_cmd_wnd, psol);
1455                 csol_cqe->wrb_index = AMAP_GET_BITS(struct amap_sol_cqe,
1456                                                     wrb_index, psol);
1457                 csol_cqe->cid = AMAP_GET_BITS(struct amap_sol_cqe,
1458                                               cid, psol);
1459                 csol_cqe->hw_sts = AMAP_GET_BITS(struct amap_sol_cqe,
1460                                                  hw_sts, psol);
1461                 csol_cqe->i_resp = AMAP_GET_BITS(struct amap_sol_cqe,
1462                                                  i_resp, psol);
1463                 csol_cqe->i_sts = AMAP_GET_BITS(struct amap_sol_cqe,
1464                                                 i_sts, psol);
1465                 csol_cqe->i_flags = AMAP_GET_BITS(struct amap_sol_cqe,
1466                                                   i_flags, psol);
1467         }
1468 }
1469
1470
1471 static void hwi_complete_cmd(struct beiscsi_conn *beiscsi_conn,
1472                              struct beiscsi_hba *phba, struct sol_cqe *psol)
1473 {
1474         struct hwi_wrb_context *pwrb_context;
1475         struct wrb_handle *pwrb_handle;
1476         struct iscsi_wrb *pwrb = NULL;
1477         struct hwi_controller *phwi_ctrlr;
1478         struct iscsi_task *task;
1479         unsigned int type;
1480         struct iscsi_conn *conn = beiscsi_conn->conn;
1481         struct iscsi_session *session = conn->session;
1482         struct common_sol_cqe csol_cqe = {0};
1483
1484         phwi_ctrlr = phba->phwi_ctrlr;
1485
1486         /* Copy the elements to a common structure */
1487         adapter_get_sol_cqe(phba, psol, &csol_cqe);
1488
1489         pwrb_context = &phwi_ctrlr->wrb_context[
1490                         csol_cqe.cid - phba->fw_config.iscsi_cid_start];
1491
1492         pwrb_handle = pwrb_context->pwrb_handle_basestd[
1493                       csol_cqe.wrb_index];
1494
1495         task = pwrb_handle->pio_handle;
1496         pwrb = pwrb_handle->pwrb;
1497         type = ((struct beiscsi_io_task *)task->dd_data)->wrb_type;
1498
1499         spin_lock_bh(&session->lock);
1500         switch (type) {
1501         case HWH_TYPE_IO:
1502         case HWH_TYPE_IO_RD:
1503                 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) ==
1504                      ISCSI_OP_NOOP_OUT)
1505                         be_complete_nopin_resp(beiscsi_conn, task, &csol_cqe);
1506                 else
1507                         be_complete_io(beiscsi_conn, task, &csol_cqe);
1508                 break;
1509
1510         case HWH_TYPE_LOGOUT:
1511                 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGOUT)
1512                         be_complete_logout(beiscsi_conn, task, &csol_cqe);
1513                 else
1514                         be_complete_tmf(beiscsi_conn, task, &csol_cqe);
1515                 break;
1516
1517         case HWH_TYPE_LOGIN:
1518                 beiscsi_log(phba, KERN_ERR,
1519                             BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
1520                             "BM_%d :\t\t No HWH_TYPE_LOGIN Expected in"
1521                             " hwi_complete_cmd- Solicited path\n");
1522                 break;
1523
1524         case HWH_TYPE_NOP:
1525                 be_complete_nopin_resp(beiscsi_conn, task, &csol_cqe);
1526                 break;
1527
1528         default:
1529                 beiscsi_log(phba, KERN_WARNING,
1530                             BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
1531                             "BM_%d : In hwi_complete_cmd, unknown type = %d"
1532                             "wrb_index 0x%x CID 0x%x\n", type,
1533                             csol_cqe.wrb_index,
1534                             csol_cqe.cid);
1535                 break;
1536         }
1537
1538         spin_unlock_bh(&session->lock);
1539 }
1540
1541 static struct list_head *hwi_get_async_busy_list(struct hwi_async_pdu_context
1542                                           *pasync_ctx, unsigned int is_header,
1543                                           unsigned int host_write_ptr)
1544 {
1545         if (is_header)
1546                 return &pasync_ctx->async_entry[host_write_ptr].
1547                     header_busy_list;
1548         else
1549                 return &pasync_ctx->async_entry[host_write_ptr].data_busy_list;
1550 }
1551
1552 static struct async_pdu_handle *
1553 hwi_get_async_handle(struct beiscsi_hba *phba,
1554                      struct beiscsi_conn *beiscsi_conn,
1555                      struct hwi_async_pdu_context *pasync_ctx,
1556                      struct i_t_dpdu_cqe *pdpdu_cqe, unsigned int *pcq_index)
1557 {
1558         struct be_bus_address phys_addr;
1559         struct list_head *pbusy_list;
1560         struct async_pdu_handle *pasync_handle = NULL;
1561         unsigned char is_header = 0;
1562         unsigned int index, dpl;
1563
1564         if (chip_skh_r(phba->pcidev)) {
1565                 dpl = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe_v2,
1566                                     dpl, pdpdu_cqe);
1567                 index = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe_v2,
1568                                       index, pdpdu_cqe);
1569         } else {
1570                 dpl = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe,
1571                                     dpl, pdpdu_cqe);
1572                 index = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe,
1573                                       index, pdpdu_cqe);
1574         }
1575
1576         phys_addr.u.a32.address_lo =
1577                 (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1578                                         db_addr_lo) / 32] - dpl);
1579         phys_addr.u.a32.address_hi =
1580                 pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1581                                        db_addr_hi) / 32];
1582
1583         phys_addr.u.a64.address =
1584                         *((unsigned long long *)(&phys_addr.u.a64.address));
1585
1586         switch (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, code) / 32]
1587                         & PDUCQE_CODE_MASK) {
1588         case UNSOL_HDR_NOTIFY:
1589                 is_header = 1;
1590
1591                  pbusy_list = hwi_get_async_busy_list(pasync_ctx,
1592                                                       is_header, index);
1593                 break;
1594         case UNSOL_DATA_NOTIFY:
1595                  pbusy_list = hwi_get_async_busy_list(pasync_ctx,
1596                                                       is_header, index);
1597                 break;
1598         default:
1599                 pbusy_list = NULL;
1600                 beiscsi_log(phba, KERN_WARNING,
1601                             BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
1602                             "BM_%d : Unexpected code=%d\n",
1603                             pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1604                             code) / 32] & PDUCQE_CODE_MASK);
1605                 return NULL;
1606         }
1607
1608         WARN_ON(list_empty(pbusy_list));
1609         list_for_each_entry(pasync_handle, pbusy_list, link) {
1610                 if (pasync_handle->pa.u.a64.address == phys_addr.u.a64.address)
1611                         break;
1612         }
1613
1614         WARN_ON(!pasync_handle);
1615
1616         pasync_handle->cri = (unsigned short)beiscsi_conn->beiscsi_conn_cid -
1617                                              phba->fw_config.iscsi_cid_start;
1618         pasync_handle->is_header = is_header;
1619         pasync_handle->buffer_len = dpl;
1620         *pcq_index = index;
1621
1622         return pasync_handle;
1623 }
1624
1625 static unsigned int
1626 hwi_update_async_writables(struct beiscsi_hba *phba,
1627                             struct hwi_async_pdu_context *pasync_ctx,
1628                             unsigned int is_header, unsigned int cq_index)
1629 {
1630         struct list_head *pbusy_list;
1631         struct async_pdu_handle *pasync_handle;
1632         unsigned int num_entries, writables = 0;
1633         unsigned int *pep_read_ptr, *pwritables;
1634
1635         num_entries = pasync_ctx->num_entries;
1636         if (is_header) {
1637                 pep_read_ptr = &pasync_ctx->async_header.ep_read_ptr;
1638                 pwritables = &pasync_ctx->async_header.writables;
1639         } else {
1640                 pep_read_ptr = &pasync_ctx->async_data.ep_read_ptr;
1641                 pwritables = &pasync_ctx->async_data.writables;
1642         }
1643
1644         while ((*pep_read_ptr) != cq_index) {
1645                 (*pep_read_ptr)++;
1646                 *pep_read_ptr = (*pep_read_ptr) % num_entries;
1647
1648                 pbusy_list = hwi_get_async_busy_list(pasync_ctx, is_header,
1649                                                      *pep_read_ptr);
1650                 if (writables == 0)
1651                         WARN_ON(list_empty(pbusy_list));
1652
1653                 if (!list_empty(pbusy_list)) {
1654                         pasync_handle = list_entry(pbusy_list->next,
1655                                                    struct async_pdu_handle,
1656                                                    link);
1657                         WARN_ON(!pasync_handle);
1658                         pasync_handle->consumed = 1;
1659                 }
1660
1661                 writables++;
1662         }
1663
1664         if (!writables) {
1665                 beiscsi_log(phba, KERN_ERR,
1666                             BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
1667                             "BM_%d : Duplicate notification received - index 0x%x!!\n",
1668                             cq_index);
1669                 WARN_ON(1);
1670         }
1671
1672         *pwritables = *pwritables + writables;
1673         return 0;
1674 }
1675
1676 static void hwi_free_async_msg(struct beiscsi_hba *phba,
1677                                        unsigned int cri)
1678 {
1679         struct hwi_controller *phwi_ctrlr;
1680         struct hwi_async_pdu_context *pasync_ctx;
1681         struct async_pdu_handle *pasync_handle, *tmp_handle;
1682         struct list_head *plist;
1683
1684         phwi_ctrlr = phba->phwi_ctrlr;
1685         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1686
1687         plist  = &pasync_ctx->async_entry[cri].wait_queue.list;
1688
1689         list_for_each_entry_safe(pasync_handle, tmp_handle, plist, link) {
1690                 list_del(&pasync_handle->link);
1691
1692                 if (pasync_handle->is_header) {
1693                         list_add_tail(&pasync_handle->link,
1694                                       &pasync_ctx->async_header.free_list);
1695                         pasync_ctx->async_header.free_entries++;
1696                 } else {
1697                         list_add_tail(&pasync_handle->link,
1698                                       &pasync_ctx->async_data.free_list);
1699                         pasync_ctx->async_data.free_entries++;
1700                 }
1701         }
1702
1703         INIT_LIST_HEAD(&pasync_ctx->async_entry[cri].wait_queue.list);
1704         pasync_ctx->async_entry[cri].wait_queue.hdr_received = 0;
1705         pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0;
1706 }
1707
1708 static struct phys_addr *
1709 hwi_get_ring_address(struct hwi_async_pdu_context *pasync_ctx,
1710                      unsigned int is_header, unsigned int host_write_ptr)
1711 {
1712         struct phys_addr *pasync_sge = NULL;
1713
1714         if (is_header)
1715                 pasync_sge = pasync_ctx->async_header.ring_base;
1716         else
1717                 pasync_sge = pasync_ctx->async_data.ring_base;
1718
1719         return pasync_sge + host_write_ptr;
1720 }
1721
1722 static void hwi_post_async_buffers(struct beiscsi_hba *phba,
1723                                    unsigned int is_header)
1724 {
1725         struct hwi_controller *phwi_ctrlr;
1726         struct hwi_async_pdu_context *pasync_ctx;
1727         struct async_pdu_handle *pasync_handle;
1728         struct list_head *pfree_link, *pbusy_list;
1729         struct phys_addr *pasync_sge;
1730         unsigned int ring_id, num_entries;
1731         unsigned int host_write_num;
1732         unsigned int writables;
1733         unsigned int i = 0;
1734         u32 doorbell = 0;
1735
1736         phwi_ctrlr = phba->phwi_ctrlr;
1737         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1738         num_entries = pasync_ctx->num_entries;
1739
1740         if (is_header) {
1741                 writables = min(pasync_ctx->async_header.writables,
1742                                 pasync_ctx->async_header.free_entries);
1743                 pfree_link = pasync_ctx->async_header.free_list.next;
1744                 host_write_num = pasync_ctx->async_header.host_write_ptr;
1745                 ring_id = phwi_ctrlr->default_pdu_hdr.id;
1746         } else {
1747                 writables = min(pasync_ctx->async_data.writables,
1748                                 pasync_ctx->async_data.free_entries);
1749                 pfree_link = pasync_ctx->async_data.free_list.next;
1750                 host_write_num = pasync_ctx->async_data.host_write_ptr;
1751                 ring_id = phwi_ctrlr->default_pdu_data.id;
1752         }
1753
1754         writables = (writables / 8) * 8;
1755         if (writables) {
1756                 for (i = 0; i < writables; i++) {
1757                         pbusy_list =
1758                             hwi_get_async_busy_list(pasync_ctx, is_header,
1759                                                     host_write_num);
1760                         pasync_handle =
1761                             list_entry(pfree_link, struct async_pdu_handle,
1762                                                                 link);
1763                         WARN_ON(!pasync_handle);
1764                         pasync_handle->consumed = 0;
1765
1766                         pfree_link = pfree_link->next;
1767
1768                         pasync_sge = hwi_get_ring_address(pasync_ctx,
1769                                                 is_header, host_write_num);
1770
1771                         pasync_sge->hi = pasync_handle->pa.u.a32.address_lo;
1772                         pasync_sge->lo = pasync_handle->pa.u.a32.address_hi;
1773
1774                         list_move(&pasync_handle->link, pbusy_list);
1775
1776                         host_write_num++;
1777                         host_write_num = host_write_num % num_entries;
1778                 }
1779
1780                 if (is_header) {
1781                         pasync_ctx->async_header.host_write_ptr =
1782                                                         host_write_num;
1783                         pasync_ctx->async_header.free_entries -= writables;
1784                         pasync_ctx->async_header.writables -= writables;
1785                         pasync_ctx->async_header.busy_entries += writables;
1786                 } else {
1787                         pasync_ctx->async_data.host_write_ptr = host_write_num;
1788                         pasync_ctx->async_data.free_entries -= writables;
1789                         pasync_ctx->async_data.writables -= writables;
1790                         pasync_ctx->async_data.busy_entries += writables;
1791                 }
1792
1793                 doorbell |= ring_id & DB_DEF_PDU_RING_ID_MASK;
1794                 doorbell |= 1 << DB_DEF_PDU_REARM_SHIFT;
1795                 doorbell |= 0 << DB_DEF_PDU_EVENT_SHIFT;
1796                 doorbell |= (writables & DB_DEF_PDU_CQPROC_MASK)
1797                                         << DB_DEF_PDU_CQPROC_SHIFT;
1798
1799                 iowrite32(doorbell, phba->db_va + DB_RXULP0_OFFSET);
1800         }
1801 }
1802
1803 static void hwi_flush_default_pdu_buffer(struct beiscsi_hba *phba,
1804                                          struct beiscsi_conn *beiscsi_conn,
1805                                          struct i_t_dpdu_cqe *pdpdu_cqe)
1806 {
1807         struct hwi_controller *phwi_ctrlr;
1808         struct hwi_async_pdu_context *pasync_ctx;
1809         struct async_pdu_handle *pasync_handle = NULL;
1810         unsigned int cq_index = -1;
1811
1812         phwi_ctrlr = phba->phwi_ctrlr;
1813         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1814
1815         pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx,
1816                                              pdpdu_cqe, &cq_index);
1817         BUG_ON(pasync_handle->is_header != 0);
1818         if (pasync_handle->consumed == 0)
1819                 hwi_update_async_writables(phba, pasync_ctx,
1820                                            pasync_handle->is_header, cq_index);
1821
1822         hwi_free_async_msg(phba, pasync_handle->cri);
1823         hwi_post_async_buffers(phba, pasync_handle->is_header);
1824 }
1825
1826 static unsigned int
1827 hwi_fwd_async_msg(struct beiscsi_conn *beiscsi_conn,
1828                   struct beiscsi_hba *phba,
1829                   struct hwi_async_pdu_context *pasync_ctx, unsigned short cri)
1830 {
1831         struct list_head *plist;
1832         struct async_pdu_handle *pasync_handle;
1833         void *phdr = NULL;
1834         unsigned int hdr_len = 0, buf_len = 0;
1835         unsigned int status, index = 0, offset = 0;
1836         void *pfirst_buffer = NULL;
1837         unsigned int num_buf = 0;
1838
1839         plist = &pasync_ctx->async_entry[cri].wait_queue.list;
1840
1841         list_for_each_entry(pasync_handle, plist, link) {
1842                 if (index == 0) {
1843                         phdr = pasync_handle->pbuffer;
1844                         hdr_len = pasync_handle->buffer_len;
1845                 } else {
1846                         buf_len = pasync_handle->buffer_len;
1847                         if (!num_buf) {
1848                                 pfirst_buffer = pasync_handle->pbuffer;
1849                                 num_buf++;
1850                         }
1851                         memcpy(pfirst_buffer + offset,
1852                                pasync_handle->pbuffer, buf_len);
1853                         offset += buf_len;
1854                 }
1855                 index++;
1856         }
1857
1858         status = beiscsi_process_async_pdu(beiscsi_conn, phba,
1859                                            (beiscsi_conn->beiscsi_conn_cid -
1860                                             phba->fw_config.iscsi_cid_start),
1861                                             phdr, hdr_len, pfirst_buffer,
1862                                             offset);
1863
1864         hwi_free_async_msg(phba, cri);
1865         return 0;
1866 }
1867
1868 static unsigned int
1869 hwi_gather_async_pdu(struct beiscsi_conn *beiscsi_conn,
1870                      struct beiscsi_hba *phba,
1871                      struct async_pdu_handle *pasync_handle)
1872 {
1873         struct hwi_async_pdu_context *pasync_ctx;
1874         struct hwi_controller *phwi_ctrlr;
1875         unsigned int bytes_needed = 0, status = 0;
1876         unsigned short cri = pasync_handle->cri;
1877         struct pdu_base *ppdu;
1878
1879         phwi_ctrlr = phba->phwi_ctrlr;
1880         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1881
1882         list_del(&pasync_handle->link);
1883         if (pasync_handle->is_header) {
1884                 pasync_ctx->async_header.busy_entries--;
1885                 if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) {
1886                         hwi_free_async_msg(phba, cri);
1887                         BUG();
1888                 }
1889
1890                 pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0;
1891                 pasync_ctx->async_entry[cri].wait_queue.hdr_received = 1;
1892                 pasync_ctx->async_entry[cri].wait_queue.hdr_len =
1893                                 (unsigned short)pasync_handle->buffer_len;
1894                 list_add_tail(&pasync_handle->link,
1895                               &pasync_ctx->async_entry[cri].wait_queue.list);
1896
1897                 ppdu = pasync_handle->pbuffer;
1898                 bytes_needed = ((((ppdu->dw[offsetof(struct amap_pdu_base,
1899                         data_len_hi) / 32] & PDUBASE_DATALENHI_MASK) << 8) &
1900                         0xFFFF0000) | ((be16_to_cpu((ppdu->
1901                         dw[offsetof(struct amap_pdu_base, data_len_lo) / 32]
1902                         & PDUBASE_DATALENLO_MASK) >> 16)) & 0x0000FFFF));
1903
1904                 if (status == 0) {
1905                         pasync_ctx->async_entry[cri].wait_queue.bytes_needed =
1906                             bytes_needed;
1907
1908                         if (bytes_needed == 0)
1909                                 status = hwi_fwd_async_msg(beiscsi_conn, phba,
1910                                                            pasync_ctx, cri);
1911                 }
1912         } else {
1913                 pasync_ctx->async_data.busy_entries--;
1914                 if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) {
1915                         list_add_tail(&pasync_handle->link,
1916                                       &pasync_ctx->async_entry[cri].wait_queue.
1917                                       list);
1918                         pasync_ctx->async_entry[cri].wait_queue.
1919                                 bytes_received +=
1920                                 (unsigned short)pasync_handle->buffer_len;
1921
1922                         if (pasync_ctx->async_entry[cri].wait_queue.
1923                             bytes_received >=
1924                             pasync_ctx->async_entry[cri].wait_queue.
1925                             bytes_needed)
1926                                 status = hwi_fwd_async_msg(beiscsi_conn, phba,
1927                                                            pasync_ctx, cri);
1928                 }
1929         }
1930         return status;
1931 }
1932
1933 static void hwi_process_default_pdu_ring(struct beiscsi_conn *beiscsi_conn,
1934                                          struct beiscsi_hba *phba,
1935                                          struct i_t_dpdu_cqe *pdpdu_cqe)
1936 {
1937         struct hwi_controller *phwi_ctrlr;
1938         struct hwi_async_pdu_context *pasync_ctx;
1939         struct async_pdu_handle *pasync_handle = NULL;
1940         unsigned int cq_index = -1;
1941
1942         phwi_ctrlr = phba->phwi_ctrlr;
1943         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1944         pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx,
1945                                              pdpdu_cqe, &cq_index);
1946
1947         if (pasync_handle->consumed == 0)
1948                 hwi_update_async_writables(phba, pasync_ctx,
1949                                            pasync_handle->is_header, cq_index);
1950
1951         hwi_gather_async_pdu(beiscsi_conn, phba, pasync_handle);
1952         hwi_post_async_buffers(phba, pasync_handle->is_header);
1953 }
1954
1955 static void  beiscsi_process_mcc_isr(struct beiscsi_hba *phba)
1956 {
1957         struct be_queue_info *mcc_cq;
1958         struct  be_mcc_compl *mcc_compl;
1959         unsigned int num_processed = 0;
1960
1961         mcc_cq = &phba->ctrl.mcc_obj.cq;
1962         mcc_compl = queue_tail_node(mcc_cq);
1963         mcc_compl->flags = le32_to_cpu(mcc_compl->flags);
1964         while (mcc_compl->flags & CQE_FLAGS_VALID_MASK) {
1965
1966                 if (num_processed >= 32) {
1967                         hwi_ring_cq_db(phba, mcc_cq->id,
1968                                         num_processed, 0, 0);
1969                         num_processed = 0;
1970                 }
1971                 if (mcc_compl->flags & CQE_FLAGS_ASYNC_MASK) {
1972                         /* Interpret flags as an async trailer */
1973                         if (is_link_state_evt(mcc_compl->flags))
1974                                 /* Interpret compl as a async link evt */
1975                                 beiscsi_async_link_state_process(phba,
1976                                 (struct be_async_event_link_state *) mcc_compl);
1977                         else
1978                                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_MBOX,
1979                                             "BM_%d :  Unsupported Async Event, flags"
1980                                             " = 0x%08x\n",
1981                                             mcc_compl->flags);
1982                 } else if (mcc_compl->flags & CQE_FLAGS_COMPLETED_MASK) {
1983                         be_mcc_compl_process_isr(&phba->ctrl, mcc_compl);
1984                         atomic_dec(&phba->ctrl.mcc_obj.q.used);
1985                 }
1986
1987                 mcc_compl->flags = 0;
1988                 queue_tail_inc(mcc_cq);
1989                 mcc_compl = queue_tail_node(mcc_cq);
1990                 mcc_compl->flags = le32_to_cpu(mcc_compl->flags);
1991                 num_processed++;
1992         }
1993
1994         if (num_processed > 0)
1995                 hwi_ring_cq_db(phba, mcc_cq->id, num_processed, 1, 0);
1996
1997 }
1998
1999 /**
2000  * beiscsi_process_cq()- Process the Completion Queue
2001  * @pbe_eq: Event Q on which the Completion has come
2002  *
2003  * return
2004  *     Number of Completion Entries processed.
2005  **/
2006 static unsigned int beiscsi_process_cq(struct be_eq_obj *pbe_eq)
2007 {
2008         struct be_queue_info *cq;
2009         struct sol_cqe *sol;
2010         struct dmsg_cqe *dmsg;
2011         unsigned int num_processed = 0;
2012         unsigned int tot_nump = 0;
2013         unsigned short code = 0, cid = 0;
2014         struct beiscsi_conn *beiscsi_conn;
2015         struct beiscsi_endpoint *beiscsi_ep;
2016         struct iscsi_endpoint *ep;
2017         struct beiscsi_hba *phba;
2018
2019         cq = pbe_eq->cq;
2020         sol = queue_tail_node(cq);
2021         phba = pbe_eq->phba;
2022
2023         while (sol->dw[offsetof(struct amap_sol_cqe, valid) / 32] &
2024                CQE_VALID_MASK) {
2025                 be_dws_le_to_cpu(sol, sizeof(struct sol_cqe));
2026
2027                  code = (sol->dw[offsetof(struct amap_sol_cqe, code) /
2028                          32] & CQE_CODE_MASK);
2029
2030                  /* Get the CID */
2031                 if (chip_skh_r(phba->pcidev)) {
2032                         if ((code == DRIVERMSG_NOTIFY) ||
2033                             (code == UNSOL_HDR_NOTIFY) ||
2034                             (code == UNSOL_DATA_NOTIFY))
2035                                 cid = AMAP_GET_BITS(
2036                                                     struct amap_i_t_dpdu_cqe_v2,
2037                                                     cid, sol);
2038                          else
2039                                  cid = AMAP_GET_BITS(struct amap_sol_cqe_v2,
2040                                                      cid, sol);
2041                    } else
2042                          cid = AMAP_GET_BITS(struct amap_sol_cqe, cid, sol);
2043
2044                 ep = phba->ep_array[cid - phba->fw_config.iscsi_cid_start];
2045                 beiscsi_ep = ep->dd_data;
2046                 beiscsi_conn = beiscsi_ep->conn;
2047
2048                 if (num_processed >= 32) {
2049                         hwi_ring_cq_db(phba, cq->id,
2050                                         num_processed, 0, 0);
2051                         tot_nump += num_processed;
2052                         num_processed = 0;
2053                 }
2054
2055                 switch (code) {
2056                 case SOL_CMD_COMPLETE:
2057                         hwi_complete_cmd(beiscsi_conn, phba, sol);
2058                         break;
2059                 case DRIVERMSG_NOTIFY:
2060                         beiscsi_log(phba, KERN_INFO,
2061                                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
2062                                     "BM_%d : Received %s[%d] on CID : %d\n",
2063                                     cqe_desc[code], code, cid);
2064
2065                         dmsg = (struct dmsg_cqe *)sol;
2066                         hwi_complete_drvr_msgs(beiscsi_conn, phba, sol);
2067                         break;
2068                 case UNSOL_HDR_NOTIFY:
2069                         beiscsi_log(phba, KERN_INFO,
2070                                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
2071                                     "BM_%d : Received %s[%d] on CID : %d\n",
2072                                     cqe_desc[code], code, cid);
2073
2074                         hwi_process_default_pdu_ring(beiscsi_conn, phba,
2075                                              (struct i_t_dpdu_cqe *)sol);
2076                         break;
2077                 case UNSOL_DATA_NOTIFY:
2078                         beiscsi_log(phba, KERN_INFO,
2079                                     BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
2080                                     "BM_%d : Received %s[%d] on CID : %d\n",
2081                                     cqe_desc[code], code, cid);
2082
2083                         hwi_process_default_pdu_ring(beiscsi_conn, phba,
2084                                              (struct i_t_dpdu_cqe *)sol);
2085                         break;
2086                 case CXN_INVALIDATE_INDEX_NOTIFY:
2087                 case CMD_INVALIDATED_NOTIFY:
2088                 case CXN_INVALIDATE_NOTIFY:
2089                         beiscsi_log(phba, KERN_ERR,
2090                                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
2091                                     "BM_%d : Ignoring %s[%d] on CID : %d\n",
2092                                     cqe_desc[code], code, cid);
2093                         break;
2094                 case SOL_CMD_KILLED_DATA_DIGEST_ERR:
2095                 case CMD_KILLED_INVALID_STATSN_RCVD:
2096                 case CMD_KILLED_INVALID_R2T_RCVD:
2097                 case CMD_CXN_KILLED_LUN_INVALID:
2098                 case CMD_CXN_KILLED_ICD_INVALID:
2099                 case CMD_CXN_KILLED_ITT_INVALID:
2100                 case CMD_CXN_KILLED_SEQ_OUTOFORDER:
2101                 case CMD_CXN_KILLED_INVALID_DATASN_RCVD:
2102                         beiscsi_log(phba, KERN_ERR,
2103                                     BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
2104                                     "BM_%d : Cmd Notification %s[%d] on CID : %d\n",
2105                                     cqe_desc[code], code,  cid);
2106                         break;
2107                 case UNSOL_DATA_DIGEST_ERROR_NOTIFY:
2108                         beiscsi_log(phba, KERN_ERR,
2109                                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
2110                                     "BM_%d :  Dropping %s[%d] on DPDU ring on CID : %d\n",
2111                                     cqe_desc[code], code, cid);
2112                         hwi_flush_default_pdu_buffer(phba, beiscsi_conn,
2113                                              (struct i_t_dpdu_cqe *) sol);
2114                         break;
2115                 case CXN_KILLED_PDU_SIZE_EXCEEDS_DSL:
2116                 case CXN_KILLED_BURST_LEN_MISMATCH:
2117                 case CXN_KILLED_AHS_RCVD:
2118                 case CXN_KILLED_HDR_DIGEST_ERR:
2119                 case CXN_KILLED_UNKNOWN_HDR:
2120                 case CXN_KILLED_STALE_ITT_TTT_RCVD:
2121                 case CXN_KILLED_INVALID_ITT_TTT_RCVD:
2122                 case CXN_KILLED_TIMED_OUT:
2123                 case CXN_KILLED_FIN_RCVD:
2124                 case CXN_KILLED_RST_SENT:
2125                 case CXN_KILLED_RST_RCVD:
2126                 case CXN_KILLED_BAD_UNSOL_PDU_RCVD:
2127                 case CXN_KILLED_BAD_WRB_INDEX_ERROR:
2128                 case CXN_KILLED_OVER_RUN_RESIDUAL:
2129                 case CXN_KILLED_UNDER_RUN_RESIDUAL:
2130                 case CXN_KILLED_CMND_DATA_NOT_ON_SAME_CONN:
2131                         beiscsi_log(phba, KERN_ERR,
2132                                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
2133                                     "BM_%d : Event %s[%d] received on CID : %d\n",
2134                                     cqe_desc[code], code, cid);
2135                         if (beiscsi_conn)
2136                                 iscsi_conn_failure(beiscsi_conn->conn,
2137                                                    ISCSI_ERR_CONN_FAILED);
2138                         break;
2139                 default:
2140                         beiscsi_log(phba, KERN_ERR,
2141                                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
2142                                     "BM_%d : Invalid CQE Event Received Code : %d"
2143                                     "CID 0x%x...\n",
2144                                     code, cid);
2145                         break;
2146                 }
2147
2148                 AMAP_SET_BITS(struct amap_sol_cqe, valid, sol, 0);
2149                 queue_tail_inc(cq);
2150                 sol = queue_tail_node(cq);
2151                 num_processed++;
2152         }
2153
2154         if (num_processed > 0) {
2155                 tot_nump += num_processed;
2156                 hwi_ring_cq_db(phba, cq->id, num_processed, 1, 0);
2157         }
2158         return tot_nump;
2159 }
2160
2161 void beiscsi_process_all_cqs(struct work_struct *work)
2162 {
2163         unsigned long flags;
2164         struct hwi_controller *phwi_ctrlr;
2165         struct hwi_context_memory *phwi_context;
2166         struct beiscsi_hba *phba;
2167         struct be_eq_obj *pbe_eq =
2168             container_of(work, struct be_eq_obj, work_cqs);
2169
2170         phba = pbe_eq->phba;
2171         phwi_ctrlr = phba->phwi_ctrlr;
2172         phwi_context = phwi_ctrlr->phwi_ctxt;
2173
2174         if (pbe_eq->todo_mcc_cq) {
2175                 spin_lock_irqsave(&phba->isr_lock, flags);
2176                 pbe_eq->todo_mcc_cq = false;
2177                 spin_unlock_irqrestore(&phba->isr_lock, flags);
2178                 beiscsi_process_mcc_isr(phba);
2179         }
2180
2181         if (pbe_eq->todo_cq) {
2182                 spin_lock_irqsave(&phba->isr_lock, flags);
2183                 pbe_eq->todo_cq = false;
2184                 spin_unlock_irqrestore(&phba->isr_lock, flags);
2185                 beiscsi_process_cq(pbe_eq);
2186         }
2187
2188         /* rearm EQ for further interrupts */
2189         hwi_ring_eq_db(phba, pbe_eq->q.id, 0, 0, 1, 1);
2190 }
2191
2192 static int be_iopoll(struct blk_iopoll *iop, int budget)
2193 {
2194         static unsigned int ret;
2195         struct beiscsi_hba *phba;
2196         struct be_eq_obj *pbe_eq;
2197
2198         pbe_eq = container_of(iop, struct be_eq_obj, iopoll);
2199         ret = beiscsi_process_cq(pbe_eq);
2200         if (ret < budget) {
2201                 phba = pbe_eq->phba;
2202                 blk_iopoll_complete(iop);
2203                 beiscsi_log(phba, KERN_INFO,
2204                             BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
2205                             "BM_%d : rearm pbe_eq->q.id =%d\n",
2206                             pbe_eq->q.id);
2207                 hwi_ring_eq_db(phba, pbe_eq->q.id, 0, 0, 1, 1);
2208         }
2209         return ret;
2210 }
2211
2212 static void
2213 hwi_write_sgl_v2(struct iscsi_wrb *pwrb, struct scatterlist *sg,
2214                   unsigned int num_sg, struct beiscsi_io_task *io_task)
2215 {
2216         struct iscsi_sge *psgl;
2217         unsigned int sg_len, index;
2218         unsigned int sge_len = 0;
2219         unsigned long long addr;
2220         struct scatterlist *l_sg;
2221         unsigned int offset;
2222
2223         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, iscsi_bhs_addr_lo, pwrb,
2224                       io_task->bhs_pa.u.a32.address_lo);
2225         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, iscsi_bhs_addr_hi, pwrb,
2226                       io_task->bhs_pa.u.a32.address_hi);
2227
2228         l_sg = sg;
2229         for (index = 0; (index < num_sg) && (index < 2); index++,
2230                         sg = sg_next(sg)) {
2231                 if (index == 0) {
2232                         sg_len = sg_dma_len(sg);
2233                         addr = (u64) sg_dma_address(sg);
2234                         AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
2235                                       sge0_addr_lo, pwrb,
2236                                       lower_32_bits(addr));
2237                         AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
2238                                       sge0_addr_hi, pwrb,
2239                                       upper_32_bits(addr));
2240                         AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
2241                                       sge0_len, pwrb,
2242                                       sg_len);
2243                         sge_len = sg_len;
2244                 } else {
2245                         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge1_r2t_offset,
2246                                       pwrb, sge_len);
2247                         sg_len = sg_dma_len(sg);
2248                         addr = (u64) sg_dma_address(sg);
2249                         AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
2250                                       sge1_addr_lo, pwrb,
2251                                       lower_32_bits(addr));
2252                         AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
2253                                       sge1_addr_hi, pwrb,
2254                                       upper_32_bits(addr));
2255                         AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
2256                                       sge1_len, pwrb,
2257                                       sg_len);
2258                 }
2259         }
2260         psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
2261         memset(psgl, 0, sizeof(*psgl) * BE2_SGE);
2262
2263         AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len - 2);
2264
2265         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2266                       io_task->bhs_pa.u.a32.address_hi);
2267         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2268                       io_task->bhs_pa.u.a32.address_lo);
2269
2270         if (num_sg == 1) {
2271                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge0_last, pwrb,
2272                               1);
2273                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge1_last, pwrb,
2274                               0);
2275         } else if (num_sg == 2) {
2276                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge0_last, pwrb,
2277                               0);
2278                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge1_last, pwrb,
2279                               1);
2280         } else {
2281                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge0_last, pwrb,
2282                               0);
2283                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge1_last, pwrb,
2284                               0);
2285         }
2286
2287         sg = l_sg;
2288         psgl++;
2289         psgl++;
2290         offset = 0;
2291         for (index = 0; index < num_sg; index++, sg = sg_next(sg), psgl++) {
2292                 sg_len = sg_dma_len(sg);
2293                 addr = (u64) sg_dma_address(sg);
2294                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2295                               lower_32_bits(addr));
2296                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2297                               upper_32_bits(addr));
2298                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, sg_len);
2299                 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, offset);
2300                 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
2301                 offset += sg_len;
2302         }
2303         psgl--;
2304         AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
2305 }
2306
2307 static void
2308 hwi_write_sgl(struct iscsi_wrb *pwrb, struct scatterlist *sg,
2309               unsigned int num_sg, struct beiscsi_io_task *io_task)
2310 {
2311         struct iscsi_sge *psgl;
2312         unsigned int sg_len, index;
2313         unsigned int sge_len = 0;
2314         unsigned long long addr;
2315         struct scatterlist *l_sg;
2316         unsigned int offset;
2317
2318         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb,
2319                                       io_task->bhs_pa.u.a32.address_lo);
2320         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb,
2321                                       io_task->bhs_pa.u.a32.address_hi);
2322
2323         l_sg = sg;
2324         for (index = 0; (index < num_sg) && (index < 2); index++,
2325                                                          sg = sg_next(sg)) {
2326                 if (index == 0) {
2327                         sg_len = sg_dma_len(sg);
2328                         addr = (u64) sg_dma_address(sg);
2329                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb,
2330                                                 ((u32)(addr & 0xFFFFFFFF)));
2331                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb,
2332                                                         ((u32)(addr >> 32)));
2333                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb,
2334                                                         sg_len);
2335                         sge_len = sg_len;
2336                 } else {
2337                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_r2t_offset,
2338                                                         pwrb, sge_len);
2339                         sg_len = sg_dma_len(sg);
2340                         addr = (u64) sg_dma_address(sg);
2341                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_lo, pwrb,
2342                                                 ((u32)(addr & 0xFFFFFFFF)));
2343                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_hi, pwrb,
2344                                                         ((u32)(addr >> 32)));
2345                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_len, pwrb,
2346                                                         sg_len);
2347                 }
2348         }
2349         psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
2350         memset(psgl, 0, sizeof(*psgl) * BE2_SGE);
2351
2352         AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len - 2);
2353
2354         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2355                         io_task->bhs_pa.u.a32.address_hi);
2356         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2357                         io_task->bhs_pa.u.a32.address_lo);
2358
2359         if (num_sg == 1) {
2360                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
2361                                                                 1);
2362                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
2363                                                                 0);
2364         } else if (num_sg == 2) {
2365                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
2366                                                                 0);
2367                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
2368                                                                 1);
2369         } else {
2370                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
2371                                                                 0);
2372                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
2373                                                                 0);
2374         }
2375         sg = l_sg;
2376         psgl++;
2377         psgl++;
2378         offset = 0;
2379         for (index = 0; index < num_sg; index++, sg = sg_next(sg), psgl++) {
2380                 sg_len = sg_dma_len(sg);
2381                 addr = (u64) sg_dma_address(sg);
2382                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2383                                                 (addr & 0xFFFFFFFF));
2384                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2385                                                 (addr >> 32));
2386                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, sg_len);
2387                 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, offset);
2388                 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
2389                 offset += sg_len;
2390         }
2391         psgl--;
2392         AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
2393 }
2394
2395 /**
2396  * hwi_write_buffer()- Populate the WRB with task info
2397  * @pwrb: ptr to the WRB entry
2398  * @task: iscsi task which is to be executed
2399  **/
2400 static void hwi_write_buffer(struct iscsi_wrb *pwrb, struct iscsi_task *task)
2401 {
2402         struct iscsi_sge *psgl;
2403         struct beiscsi_io_task *io_task = task->dd_data;
2404         struct beiscsi_conn *beiscsi_conn = io_task->conn;
2405         struct beiscsi_hba *phba = beiscsi_conn->phba;
2406         uint8_t dsp_value = 0;
2407
2408         io_task->bhs_len = sizeof(struct be_nonio_bhs) - 2;
2409         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb,
2410                                 io_task->bhs_pa.u.a32.address_lo);
2411         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb,
2412                                 io_task->bhs_pa.u.a32.address_hi);
2413
2414         if (task->data) {
2415
2416                 /* Check for the data_count */
2417                 dsp_value = (task->data_count) ? 1 : 0;
2418
2419                 if (chip_skh_r(phba->pcidev))
2420                         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, dsp,
2421                                       pwrb, dsp_value);
2422                 else
2423                         AMAP_SET_BITS(struct amap_iscsi_wrb, dsp,
2424                                       pwrb, dsp_value);
2425
2426                 /* Map addr only if there is data_count */
2427                 if (dsp_value) {
2428                         io_task->mtask_addr = pci_map_single(phba->pcidev,
2429                                                              task->data,
2430                                                              task->data_count,
2431                                                              PCI_DMA_TODEVICE);
2432                         io_task->mtask_data_count = task->data_count;
2433                 } else
2434                         io_task->mtask_addr = 0;
2435
2436                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb,
2437                               lower_32_bits(io_task->mtask_addr));
2438                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb,
2439                               upper_32_bits(io_task->mtask_addr));
2440                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb,
2441                                                 task->data_count);
2442
2443                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb, 1);
2444         } else {
2445                 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
2446                 io_task->mtask_addr = 0;
2447         }
2448
2449         psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
2450
2451         AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len);
2452
2453         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2454                       io_task->bhs_pa.u.a32.address_hi);
2455         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2456                       io_task->bhs_pa.u.a32.address_lo);
2457         if (task->data) {
2458                 psgl++;
2459                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl, 0);
2460                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl, 0);
2461                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0);
2462                 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, 0);
2463                 AMAP_SET_BITS(struct amap_iscsi_sge, rsvd0, psgl, 0);
2464                 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
2465
2466                 psgl++;
2467                 if (task->data) {
2468                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2469                                       lower_32_bits(io_task->mtask_addr));
2470                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2471                                       upper_32_bits(io_task->mtask_addr));
2472                 }
2473                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0x106);
2474         }
2475         AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
2476 }
2477
2478 static void beiscsi_find_mem_req(struct beiscsi_hba *phba)
2479 {
2480         unsigned int num_cq_pages, num_async_pdu_buf_pages;
2481         unsigned int num_async_pdu_data_pages, wrb_sz_per_cxn;
2482         unsigned int num_async_pdu_buf_sgl_pages, num_async_pdu_data_sgl_pages;
2483
2484         num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \
2485                                       sizeof(struct sol_cqe));
2486         num_async_pdu_buf_pages =
2487                         PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
2488                                        phba->params.defpdu_hdr_sz);
2489         num_async_pdu_buf_sgl_pages =
2490                         PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
2491                                        sizeof(struct phys_addr));
2492         num_async_pdu_data_pages =
2493                         PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
2494                                        phba->params.defpdu_data_sz);
2495         num_async_pdu_data_sgl_pages =
2496                         PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
2497                                        sizeof(struct phys_addr));
2498
2499         phba->params.hwi_ws_sz = sizeof(struct hwi_controller);
2500
2501         phba->mem_req[ISCSI_MEM_GLOBAL_HEADER] = 2 *
2502                                                  BE_ISCSI_PDU_HEADER_SIZE;
2503         phba->mem_req[HWI_MEM_ADDN_CONTEXT] =
2504                                             sizeof(struct hwi_context_memory);
2505
2506
2507         phba->mem_req[HWI_MEM_WRB] = sizeof(struct iscsi_wrb)
2508             * (phba->params.wrbs_per_cxn)
2509             * phba->params.cxns_per_ctrl;
2510         wrb_sz_per_cxn =  sizeof(struct wrb_handle) *
2511                                  (phba->params.wrbs_per_cxn);
2512         phba->mem_req[HWI_MEM_WRBH] = roundup_pow_of_two((wrb_sz_per_cxn) *
2513                                 phba->params.cxns_per_ctrl);
2514
2515         phba->mem_req[HWI_MEM_SGLH] = sizeof(struct sgl_handle) *
2516                 phba->params.icds_per_ctrl;
2517         phba->mem_req[HWI_MEM_SGE] = sizeof(struct iscsi_sge) *
2518                 phba->params.num_sge_per_io * phba->params.icds_per_ctrl;
2519
2520         phba->mem_req[HWI_MEM_ASYNC_HEADER_BUF] =
2521                 num_async_pdu_buf_pages * PAGE_SIZE;
2522         phba->mem_req[HWI_MEM_ASYNC_DATA_BUF] =
2523                 num_async_pdu_data_pages * PAGE_SIZE;
2524         phba->mem_req[HWI_MEM_ASYNC_HEADER_RING] =
2525                 num_async_pdu_buf_sgl_pages * PAGE_SIZE;
2526         phba->mem_req[HWI_MEM_ASYNC_DATA_RING] =
2527                 num_async_pdu_data_sgl_pages * PAGE_SIZE;
2528         phba->mem_req[HWI_MEM_ASYNC_HEADER_HANDLE] =
2529                 phba->params.asyncpdus_per_ctrl *
2530                 sizeof(struct async_pdu_handle);
2531         phba->mem_req[HWI_MEM_ASYNC_DATA_HANDLE] =
2532                 phba->params.asyncpdus_per_ctrl *
2533                 sizeof(struct async_pdu_handle);
2534         phba->mem_req[HWI_MEM_ASYNC_PDU_CONTEXT] =
2535                 sizeof(struct hwi_async_pdu_context) +
2536                 (phba->params.cxns_per_ctrl * sizeof(struct hwi_async_entry));
2537 }
2538
2539 static int beiscsi_alloc_mem(struct beiscsi_hba *phba)
2540 {
2541         struct be_mem_descriptor *mem_descr;
2542         dma_addr_t bus_add;
2543         struct mem_array *mem_arr, *mem_arr_orig;
2544         unsigned int i, j, alloc_size, curr_alloc_size;
2545
2546         phba->phwi_ctrlr = kzalloc(phba->params.hwi_ws_sz, GFP_KERNEL);
2547         if (!phba->phwi_ctrlr)
2548                 return -ENOMEM;
2549
2550         phba->init_mem = kcalloc(SE_MEM_MAX, sizeof(*mem_descr),
2551                                  GFP_KERNEL);
2552         if (!phba->init_mem) {
2553                 kfree(phba->phwi_ctrlr);
2554                 return -ENOMEM;
2555         }
2556
2557         mem_arr_orig = kmalloc(sizeof(*mem_arr_orig) * BEISCSI_MAX_FRAGS_INIT,
2558                                GFP_KERNEL);
2559         if (!mem_arr_orig) {
2560                 kfree(phba->init_mem);
2561                 kfree(phba->phwi_ctrlr);
2562                 return -ENOMEM;
2563         }
2564
2565         mem_descr = phba->init_mem;
2566         for (i = 0; i < SE_MEM_MAX; i++) {
2567                 j = 0;
2568                 mem_arr = mem_arr_orig;
2569                 alloc_size = phba->mem_req[i];
2570                 memset(mem_arr, 0, sizeof(struct mem_array) *
2571                        BEISCSI_MAX_FRAGS_INIT);
2572                 curr_alloc_size = min(be_max_phys_size * 1024, alloc_size);
2573                 do {
2574                         mem_arr->virtual_address = pci_alloc_consistent(
2575                                                         phba->pcidev,
2576                                                         curr_alloc_size,
2577                                                         &bus_add);
2578                         if (!mem_arr->virtual_address) {
2579                                 if (curr_alloc_size <= BE_MIN_MEM_SIZE)
2580                                         goto free_mem;
2581                                 if (curr_alloc_size -
2582                                         rounddown_pow_of_two(curr_alloc_size))
2583                                         curr_alloc_size = rounddown_pow_of_two
2584                                                              (curr_alloc_size);
2585                                 else
2586                                         curr_alloc_size = curr_alloc_size / 2;
2587                         } else {
2588                                 mem_arr->bus_address.u.
2589                                     a64.address = (__u64) bus_add;
2590                                 mem_arr->size = curr_alloc_size;
2591                                 alloc_size -= curr_alloc_size;
2592                                 curr_alloc_size = min(be_max_phys_size *
2593                                                       1024, alloc_size);
2594                                 j++;
2595                                 mem_arr++;
2596                         }
2597                 } while (alloc_size);
2598                 mem_descr->num_elements = j;
2599                 mem_descr->size_in_bytes = phba->mem_req[i];
2600                 mem_descr->mem_array = kmalloc(sizeof(*mem_arr) * j,
2601                                                GFP_KERNEL);
2602                 if (!mem_descr->mem_array)
2603                         goto free_mem;
2604
2605                 memcpy(mem_descr->mem_array, mem_arr_orig,
2606                        sizeof(struct mem_array) * j);
2607                 mem_descr++;
2608         }
2609         kfree(mem_arr_orig);
2610         return 0;
2611 free_mem:
2612         mem_descr->num_elements = j;
2613         while ((i) || (j)) {
2614                 for (j = mem_descr->num_elements; j > 0; j--) {
2615                         pci_free_consistent(phba->pcidev,
2616                                             mem_descr->mem_array[j - 1].size,
2617                                             mem_descr->mem_array[j - 1].
2618                                             virtual_address,
2619                                             (unsigned long)mem_descr->
2620                                             mem_array[j - 1].
2621                                             bus_address.u.a64.address);
2622                 }
2623                 if (i) {
2624                         i--;
2625                         kfree(mem_descr->mem_array);
2626                         mem_descr--;
2627                 }
2628         }
2629         kfree(mem_arr_orig);
2630         kfree(phba->init_mem);
2631         kfree(phba->phwi_ctrlr);
2632         return -ENOMEM;
2633 }
2634
2635 static int beiscsi_get_memory(struct beiscsi_hba *phba)
2636 {
2637         beiscsi_find_mem_req(phba);
2638         return beiscsi_alloc_mem(phba);
2639 }
2640
2641 static void iscsi_init_global_templates(struct beiscsi_hba *phba)
2642 {
2643         struct pdu_data_out *pdata_out;
2644         struct pdu_nop_out *pnop_out;
2645         struct be_mem_descriptor *mem_descr;
2646
2647         mem_descr = phba->init_mem;
2648         mem_descr += ISCSI_MEM_GLOBAL_HEADER;
2649         pdata_out =
2650             (struct pdu_data_out *)mem_descr->mem_array[0].virtual_address;
2651         memset(pdata_out, 0, BE_ISCSI_PDU_HEADER_SIZE);
2652
2653         AMAP_SET_BITS(struct amap_pdu_data_out, opcode, pdata_out,
2654                       IIOC_SCSI_DATA);
2655
2656         pnop_out =
2657             (struct pdu_nop_out *)((unsigned char *)mem_descr->mem_array[0].
2658                                    virtual_address + BE_ISCSI_PDU_HEADER_SIZE);
2659
2660         memset(pnop_out, 0, BE_ISCSI_PDU_HEADER_SIZE);
2661         AMAP_SET_BITS(struct amap_pdu_nop_out, ttt, pnop_out, 0xFFFFFFFF);
2662         AMAP_SET_BITS(struct amap_pdu_nop_out, f_bit, pnop_out, 1);
2663         AMAP_SET_BITS(struct amap_pdu_nop_out, i_bit, pnop_out, 0);
2664 }
2665
2666 static int beiscsi_init_wrb_handle(struct beiscsi_hba *phba)
2667 {
2668         struct be_mem_descriptor *mem_descr_wrbh, *mem_descr_wrb;
2669         struct wrb_handle *pwrb_handle = NULL;
2670         struct hwi_controller *phwi_ctrlr;
2671         struct hwi_wrb_context *pwrb_context;
2672         struct iscsi_wrb *pwrb = NULL;
2673         unsigned int num_cxn_wrbh = 0;
2674         unsigned int num_cxn_wrb = 0, j, idx = 0, index;
2675
2676         mem_descr_wrbh = phba->init_mem;
2677         mem_descr_wrbh += HWI_MEM_WRBH;
2678
2679         mem_descr_wrb = phba->init_mem;
2680         mem_descr_wrb += HWI_MEM_WRB;
2681         phwi_ctrlr = phba->phwi_ctrlr;
2682
2683         for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) {
2684                 pwrb_context = &phwi_ctrlr->wrb_context[index];
2685                 pwrb_context->pwrb_handle_base =
2686                                 kzalloc(sizeof(struct wrb_handle *) *
2687                                         phba->params.wrbs_per_cxn, GFP_KERNEL);
2688                 if (!pwrb_context->pwrb_handle_base) {
2689                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
2690                                     "BM_%d : Mem Alloc Failed. Failing to load\n");
2691                         goto init_wrb_hndl_failed;
2692                 }
2693                 pwrb_context->pwrb_handle_basestd =
2694                                 kzalloc(sizeof(struct wrb_handle *) *
2695                                         phba->params.wrbs_per_cxn, GFP_KERNEL);
2696                 if (!pwrb_context->pwrb_handle_basestd) {
2697                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
2698                                     "BM_%d : Mem Alloc Failed. Failing to load\n");
2699                         goto init_wrb_hndl_failed;
2700                 }
2701                 if (!num_cxn_wrbh) {
2702                         pwrb_handle =
2703                                 mem_descr_wrbh->mem_array[idx].virtual_address;
2704                         num_cxn_wrbh = ((mem_descr_wrbh->mem_array[idx].size) /
2705                                         ((sizeof(struct wrb_handle)) *
2706                                          phba->params.wrbs_per_cxn));
2707                         idx++;
2708                 }
2709                 pwrb_context->alloc_index = 0;
2710                 pwrb_context->wrb_handles_available = 0;
2711                 pwrb_context->free_index = 0;
2712
2713                 if (num_cxn_wrbh) {
2714                         for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2715                                 pwrb_context->pwrb_handle_base[j] = pwrb_handle;
2716                                 pwrb_context->pwrb_handle_basestd[j] =
2717                                                                 pwrb_handle;
2718                                 pwrb_context->wrb_handles_available++;
2719                                 pwrb_handle->wrb_index = j;
2720                                 pwrb_handle++;
2721                         }
2722                         num_cxn_wrbh--;
2723                 }
2724         }
2725         idx = 0;
2726         for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) {
2727                 pwrb_context = &phwi_ctrlr->wrb_context[index];
2728                 if (!num_cxn_wrb) {
2729                         pwrb = mem_descr_wrb->mem_array[idx].virtual_address;
2730                         num_cxn_wrb = (mem_descr_wrb->mem_array[idx].size) /
2731                                 ((sizeof(struct iscsi_wrb) *
2732                                   phba->params.wrbs_per_cxn));
2733                         idx++;
2734                 }
2735
2736                 if (num_cxn_wrb) {
2737                         for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2738                                 pwrb_handle = pwrb_context->pwrb_handle_base[j];
2739                                 pwrb_handle->pwrb = pwrb;
2740                                 pwrb++;
2741                         }
2742                         num_cxn_wrb--;
2743                 }
2744         }
2745         return 0;
2746 init_wrb_hndl_failed:
2747         for (j = index; j > 0; j--) {
2748                 pwrb_context = &phwi_ctrlr->wrb_context[j];
2749                 kfree(pwrb_context->pwrb_handle_base);
2750                 kfree(pwrb_context->pwrb_handle_basestd);
2751         }
2752         return -ENOMEM;
2753 }
2754
2755 static void hwi_init_async_pdu_ctx(struct beiscsi_hba *phba)
2756 {
2757         struct hwi_controller *phwi_ctrlr;
2758         struct hba_parameters *p = &phba->params;
2759         struct hwi_async_pdu_context *pasync_ctx;
2760         struct async_pdu_handle *pasync_header_h, *pasync_data_h;
2761         unsigned int index, idx, num_per_mem, num_async_data;
2762         struct be_mem_descriptor *mem_descr;
2763
2764         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2765         mem_descr += HWI_MEM_ASYNC_PDU_CONTEXT;
2766
2767         phwi_ctrlr = phba->phwi_ctrlr;
2768         phwi_ctrlr->phwi_ctxt->pasync_ctx = (struct hwi_async_pdu_context *)
2769                                 mem_descr->mem_array[0].virtual_address;
2770         pasync_ctx = phwi_ctrlr->phwi_ctxt->pasync_ctx;
2771         memset(pasync_ctx, 0, sizeof(*pasync_ctx));
2772
2773         pasync_ctx->num_entries = p->asyncpdus_per_ctrl;
2774         pasync_ctx->buffer_size = p->defpdu_hdr_sz;
2775
2776         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2777         mem_descr += HWI_MEM_ASYNC_HEADER_BUF;
2778         if (mem_descr->mem_array[0].virtual_address) {
2779                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
2780                             "BM_%d : hwi_init_async_pdu_ctx"
2781                             " HWI_MEM_ASYNC_HEADER_BUF va=%p\n",
2782                             mem_descr->mem_array[0].virtual_address);
2783         } else
2784                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
2785                             "BM_%d : No Virtual address\n");
2786
2787         pasync_ctx->async_header.va_base =
2788                         mem_descr->mem_array[0].virtual_address;
2789
2790         pasync_ctx->async_header.pa_base.u.a64.address =
2791                         mem_descr->mem_array[0].bus_address.u.a64.address;
2792
2793         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2794         mem_descr += HWI_MEM_ASYNC_HEADER_RING;
2795         if (mem_descr->mem_array[0].virtual_address) {
2796                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
2797                             "BM_%d : hwi_init_async_pdu_ctx"
2798                             " HWI_MEM_ASYNC_HEADER_RING va=%p\n",
2799                             mem_descr->mem_array[0].virtual_address);
2800         } else
2801                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
2802                             "BM_%d : No Virtual address\n");
2803
2804         pasync_ctx->async_header.ring_base =
2805                         mem_descr->mem_array[0].virtual_address;
2806
2807         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2808         mem_descr += HWI_MEM_ASYNC_HEADER_HANDLE;
2809         if (mem_descr->mem_array[0].virtual_address) {
2810                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
2811                             "BM_%d : hwi_init_async_pdu_ctx"
2812                             " HWI_MEM_ASYNC_HEADER_HANDLE va=%p\n",
2813                             mem_descr->mem_array[0].virtual_address);
2814         } else
2815                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
2816                             "BM_%d : No Virtual address\n");
2817
2818         pasync_ctx->async_header.handle_base =
2819                         mem_descr->mem_array[0].virtual_address;
2820         pasync_ctx->async_header.writables = 0;
2821         INIT_LIST_HEAD(&pasync_ctx->async_header.free_list);
2822
2823
2824         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2825         mem_descr += HWI_MEM_ASYNC_DATA_RING;
2826         if (mem_descr->mem_array[0].virtual_address) {
2827                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
2828                             "BM_%d : hwi_init_async_pdu_ctx"
2829                             " HWI_MEM_ASYNC_DATA_RING va=%p\n",
2830                             mem_descr->mem_array[0].virtual_address);
2831         } else
2832                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
2833                             "BM_%d : No Virtual address\n");
2834
2835         pasync_ctx->async_data.ring_base =
2836                         mem_descr->mem_array[0].virtual_address;
2837
2838         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2839         mem_descr += HWI_MEM_ASYNC_DATA_HANDLE;
2840         if (!mem_descr->mem_array[0].virtual_address)
2841                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
2842                             "BM_%d : No Virtual address\n");
2843
2844         pasync_ctx->async_data.handle_base =
2845                         mem_descr->mem_array[0].virtual_address;
2846         pasync_ctx->async_data.writables = 0;
2847         INIT_LIST_HEAD(&pasync_ctx->async_data.free_list);
2848
2849         pasync_header_h =
2850                 (struct async_pdu_handle *)pasync_ctx->async_header.handle_base;
2851         pasync_data_h =
2852                 (struct async_pdu_handle *)pasync_ctx->async_data.handle_base;
2853
2854         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2855         mem_descr += HWI_MEM_ASYNC_DATA_BUF;
2856         if (mem_descr->mem_array[0].virtual_address) {
2857                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
2858                             "BM_%d : hwi_init_async_pdu_ctx"
2859                             " HWI_MEM_ASYNC_DATA_BUF va=%p\n",
2860                             mem_descr->mem_array[0].virtual_address);
2861         } else
2862                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
2863                             "BM_%d : No Virtual address\n");
2864
2865         idx = 0;
2866         pasync_ctx->async_data.va_base =
2867                         mem_descr->mem_array[idx].virtual_address;
2868         pasync_ctx->async_data.pa_base.u.a64.address =
2869                         mem_descr->mem_array[idx].bus_address.u.a64.address;
2870
2871         num_async_data = ((mem_descr->mem_array[idx].size) /
2872                                 phba->params.defpdu_data_sz);
2873         num_per_mem = 0;
2874
2875         for (index = 0; index < p->asyncpdus_per_ctrl; index++) {
2876                 pasync_header_h->cri = -1;
2877                 pasync_header_h->index = (char)index;
2878                 INIT_LIST_HEAD(&pasync_header_h->link);
2879                 pasync_header_h->pbuffer =
2880                         (void *)((unsigned long)
2881                         (pasync_ctx->async_header.va_base) +
2882                         (p->defpdu_hdr_sz * index));
2883
2884                 pasync_header_h->pa.u.a64.address =
2885                         pasync_ctx->async_header.pa_base.u.a64.address +
2886                         (p->defpdu_hdr_sz * index);
2887
2888                 list_add_tail(&pasync_header_h->link,
2889                                 &pasync_ctx->async_header.free_list);
2890                 pasync_header_h++;
2891                 pasync_ctx->async_header.free_entries++;
2892                 pasync_ctx->async_header.writables++;
2893
2894                 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].wait_queue.list);
2895                 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].
2896                                header_busy_list);
2897                 pasync_data_h->cri = -1;
2898                 pasync_data_h->index = (char)index;
2899                 INIT_LIST_HEAD(&pasync_data_h->link);
2900
2901                 if (!num_async_data) {
2902                         num_per_mem = 0;
2903                         idx++;
2904                         pasync_ctx->async_data.va_base =
2905                                 mem_descr->mem_array[idx].virtual_address;
2906                         pasync_ctx->async_data.pa_base.u.a64.address =
2907                                 mem_descr->mem_array[idx].
2908                                 bus_address.u.a64.address;
2909
2910                         num_async_data = ((mem_descr->mem_array[idx].size) /
2911                                         phba->params.defpdu_data_sz);
2912                 }
2913                 pasync_data_h->pbuffer =
2914                         (void *)((unsigned long)
2915                         (pasync_ctx->async_data.va_base) +
2916                         (p->defpdu_data_sz * num_per_mem));
2917
2918                 pasync_data_h->pa.u.a64.address =
2919                     pasync_ctx->async_data.pa_base.u.a64.address +
2920                     (p->defpdu_data_sz * num_per_mem);
2921                 num_per_mem++;
2922                 num_async_data--;
2923
2924                 list_add_tail(&pasync_data_h->link,
2925                               &pasync_ctx->async_data.free_list);
2926                 pasync_data_h++;
2927                 pasync_ctx->async_data.free_entries++;
2928                 pasync_ctx->async_data.writables++;
2929
2930                 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].data_busy_list);
2931         }
2932
2933         pasync_ctx->async_header.host_write_ptr = 0;
2934         pasync_ctx->async_header.ep_read_ptr = -1;
2935         pasync_ctx->async_data.host_write_ptr = 0;
2936         pasync_ctx->async_data.ep_read_ptr = -1;
2937 }
2938
2939 static int
2940 be_sgl_create_contiguous(void *virtual_address,
2941                          u64 physical_address, u32 length,
2942                          struct be_dma_mem *sgl)
2943 {
2944         WARN_ON(!virtual_address);
2945         WARN_ON(!physical_address);
2946         WARN_ON(!length > 0);
2947         WARN_ON(!sgl);
2948
2949         sgl->va = virtual_address;
2950         sgl->dma = (unsigned long)physical_address;
2951         sgl->size = length;
2952
2953         return 0;
2954 }
2955
2956 static void be_sgl_destroy_contiguous(struct be_dma_mem *sgl)
2957 {
2958         memset(sgl, 0, sizeof(*sgl));
2959 }
2960
2961 static void
2962 hwi_build_be_sgl_arr(struct beiscsi_hba *phba,
2963                      struct mem_array *pmem, struct be_dma_mem *sgl)
2964 {
2965         if (sgl->va)
2966                 be_sgl_destroy_contiguous(sgl);
2967
2968         be_sgl_create_contiguous(pmem->virtual_address,
2969                                  pmem->bus_address.u.a64.address,
2970                                  pmem->size, sgl);
2971 }
2972
2973 static void
2974 hwi_build_be_sgl_by_offset(struct beiscsi_hba *phba,
2975                            struct mem_array *pmem, struct be_dma_mem *sgl)
2976 {
2977         if (sgl->va)
2978                 be_sgl_destroy_contiguous(sgl);
2979
2980         be_sgl_create_contiguous((unsigned char *)pmem->virtual_address,
2981                                  pmem->bus_address.u.a64.address,
2982                                  pmem->size, sgl);
2983 }
2984
2985 static int be_fill_queue(struct be_queue_info *q,
2986                 u16 len, u16 entry_size, void *vaddress)
2987 {
2988         struct be_dma_mem *mem = &q->dma_mem;
2989
2990         memset(q, 0, sizeof(*q));
2991         q->len = len;
2992         q->entry_size = entry_size;
2993         mem->size = len * entry_size;
2994         mem->va = vaddress;
2995         if (!mem->va)
2996                 return -ENOMEM;
2997         memset(mem->va, 0, mem->size);
2998         return 0;
2999 }
3000
3001 static int beiscsi_create_eqs(struct beiscsi_hba *phba,
3002                              struct hwi_context_memory *phwi_context)
3003 {
3004         unsigned int i, num_eq_pages;
3005         int ret = 0, eq_for_mcc;
3006         struct be_queue_info *eq;
3007         struct be_dma_mem *mem;
3008         void *eq_vaddress;
3009         dma_addr_t paddr;
3010
3011         num_eq_pages = PAGES_REQUIRED(phba->params.num_eq_entries * \
3012                                       sizeof(struct be_eq_entry));
3013
3014         if (phba->msix_enabled)
3015                 eq_for_mcc = 1;
3016         else
3017                 eq_for_mcc = 0;
3018         for (i = 0; i < (phba->num_cpus + eq_for_mcc); i++) {
3019                 eq = &phwi_context->be_eq[i].q;
3020                 mem = &eq->dma_mem;
3021                 phwi_context->be_eq[i].phba = phba;
3022                 eq_vaddress = pci_alloc_consistent(phba->pcidev,
3023                                                      num_eq_pages * PAGE_SIZE,
3024                                                      &paddr);
3025                 if (!eq_vaddress)
3026                         goto create_eq_error;
3027
3028                 mem->va = eq_vaddress;
3029                 ret = be_fill_queue(eq, phba->params.num_eq_entries,
3030                                     sizeof(struct be_eq_entry), eq_vaddress);
3031                 if (ret) {
3032                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3033                                     "BM_%d : be_fill_queue Failed for EQ\n");
3034                         goto create_eq_error;
3035                 }
3036
3037                 mem->dma = paddr;
3038                 ret = beiscsi_cmd_eq_create(&phba->ctrl, eq,
3039                                             phwi_context->cur_eqd);
3040                 if (ret) {
3041                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3042                                     "BM_%d : beiscsi_cmd_eq_create"
3043                                     "Failed for EQ\n");
3044                         goto create_eq_error;
3045                 }
3046
3047                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3048                             "BM_%d : eqid = %d\n",
3049                             phwi_context->be_eq[i].q.id);
3050         }
3051         return 0;
3052 create_eq_error:
3053         for (i = 0; i < (phba->num_cpus + eq_for_mcc); i++) {
3054                 eq = &phwi_context->be_eq[i].q;
3055                 mem = &eq->dma_mem;
3056                 if (mem->va)
3057                         pci_free_consistent(phba->pcidev, num_eq_pages
3058                                             * PAGE_SIZE,
3059                                             mem->va, mem->dma);
3060         }
3061         return ret;
3062 }
3063
3064 static int beiscsi_create_cqs(struct beiscsi_hba *phba,
3065                              struct hwi_context_memory *phwi_context)
3066 {
3067         unsigned int i, num_cq_pages;
3068         int ret = 0;
3069         struct be_queue_info *cq, *eq;
3070         struct be_dma_mem *mem;
3071         struct be_eq_obj *pbe_eq;
3072         void *cq_vaddress;
3073         dma_addr_t paddr;
3074
3075         num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \
3076                                       sizeof(struct sol_cqe));
3077
3078         for (i = 0; i < phba->num_cpus; i++) {
3079                 cq = &phwi_context->be_cq[i];
3080                 eq = &phwi_context->be_eq[i].q;
3081                 pbe_eq = &phwi_context->be_eq[i];
3082                 pbe_eq->cq = cq;
3083                 pbe_eq->phba = phba;
3084                 mem = &cq->dma_mem;
3085                 cq_vaddress = pci_alloc_consistent(phba->pcidev,
3086                                                      num_cq_pages * PAGE_SIZE,
3087                                                      &paddr);
3088                 if (!cq_vaddress)
3089                         goto create_cq_error;
3090                 ret = be_fill_queue(cq, phba->params.num_cq_entries,
3091                                     sizeof(struct sol_cqe), cq_vaddress);
3092                 if (ret) {
3093                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3094                                     "BM_%d : be_fill_queue Failed "
3095                                     "for ISCSI CQ\n");
3096                         goto create_cq_error;
3097                 }
3098
3099                 mem->dma = paddr;
3100                 ret = beiscsi_cmd_cq_create(&phba->ctrl, cq, eq, false,
3101                                             false, 0);
3102                 if (ret) {
3103                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3104                                     "BM_%d : beiscsi_cmd_eq_create"
3105                                     "Failed for ISCSI CQ\n");
3106                         goto create_cq_error;
3107                 }
3108                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3109                             "BM_%d : iscsi cq_id is %d for eq_id %d\n"
3110                             "iSCSI CQ CREATED\n", cq->id, eq->id);
3111         }
3112         return 0;
3113
3114 create_cq_error:
3115         for (i = 0; i < phba->num_cpus; i++) {
3116                 cq = &phwi_context->be_cq[i];
3117                 mem = &cq->dma_mem;
3118                 if (mem->va)
3119                         pci_free_consistent(phba->pcidev, num_cq_pages
3120                                             * PAGE_SIZE,
3121                                             mem->va, mem->dma);
3122         }
3123         return ret;
3124
3125 }
3126
3127 static int
3128 beiscsi_create_def_hdr(struct beiscsi_hba *phba,
3129                        struct hwi_context_memory *phwi_context,
3130                        struct hwi_controller *phwi_ctrlr,
3131                        unsigned int def_pdu_ring_sz)
3132 {
3133         unsigned int idx;
3134         int ret;
3135         struct be_queue_info *dq, *cq;
3136         struct be_dma_mem *mem;
3137         struct be_mem_descriptor *mem_descr;
3138         void *dq_vaddress;
3139
3140         idx = 0;
3141         dq = &phwi_context->be_def_hdrq;
3142         cq = &phwi_context->be_cq[0];
3143         mem = &dq->dma_mem;
3144         mem_descr = phba->init_mem;
3145         mem_descr += HWI_MEM_ASYNC_HEADER_RING;
3146         dq_vaddress = mem_descr->mem_array[idx].virtual_address;
3147         ret = be_fill_queue(dq, mem_descr->mem_array[0].size /
3148                             sizeof(struct phys_addr),
3149                             sizeof(struct phys_addr), dq_vaddress);
3150         if (ret) {
3151                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3152                             "BM_%d : be_fill_queue Failed for DEF PDU HDR\n");
3153                 return ret;
3154         }
3155         mem->dma = (unsigned long)mem_descr->mem_array[idx].
3156                                   bus_address.u.a64.address;
3157         ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dq,
3158                                               def_pdu_ring_sz,
3159                                               phba->params.defpdu_hdr_sz);
3160         if (ret) {
3161                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3162                             "BM_%d : be_cmd_create_default_pdu_queue Failed DEFHDR\n");
3163                 return ret;
3164         }
3165         phwi_ctrlr->default_pdu_hdr.id = phwi_context->be_def_hdrq.id;
3166         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3167                     "BM_%d : iscsi def pdu id is %d\n",
3168                     phwi_context->be_def_hdrq.id);
3169
3170         hwi_post_async_buffers(phba, 1);
3171         return 0;
3172 }
3173
3174 static int
3175 beiscsi_create_def_data(struct beiscsi_hba *phba,
3176                         struct hwi_context_memory *phwi_context,
3177                         struct hwi_controller *phwi_ctrlr,
3178                         unsigned int def_pdu_ring_sz)
3179 {
3180         unsigned int idx;
3181         int ret;
3182         struct be_queue_info *dataq, *cq;
3183         struct be_dma_mem *mem;
3184         struct be_mem_descriptor *mem_descr;
3185         void *dq_vaddress;
3186
3187         idx = 0;
3188         dataq = &phwi_context->be_def_dataq;
3189         cq = &phwi_context->be_cq[0];
3190         mem = &dataq->dma_mem;
3191         mem_descr = phba->init_mem;
3192         mem_descr += HWI_MEM_ASYNC_DATA_RING;
3193         dq_vaddress = mem_descr->mem_array[idx].virtual_address;
3194         ret = be_fill_queue(dataq, mem_descr->mem_array[0].size /
3195                             sizeof(struct phys_addr),
3196                             sizeof(struct phys_addr), dq_vaddress);
3197         if (ret) {
3198                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3199                             "BM_%d : be_fill_queue Failed for DEF PDU DATA\n");
3200                 return ret;
3201         }
3202         mem->dma = (unsigned long)mem_descr->mem_array[idx].
3203                                   bus_address.u.a64.address;
3204         ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dataq,
3205                                               def_pdu_ring_sz,
3206                                               phba->params.defpdu_data_sz);
3207         if (ret) {
3208                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3209                             "BM_%d be_cmd_create_default_pdu_queue"
3210                             " Failed for DEF PDU DATA\n");
3211                 return ret;
3212         }
3213         phwi_ctrlr->default_pdu_data.id = phwi_context->be_def_dataq.id;
3214         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3215                     "BM_%d : iscsi def data id is %d\n",
3216                     phwi_context->be_def_dataq.id);
3217
3218         hwi_post_async_buffers(phba, 0);
3219         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3220                     "BM_%d : DEFAULT PDU DATA RING CREATED\n");
3221
3222         return 0;
3223 }
3224
3225 static int
3226 beiscsi_post_pages(struct beiscsi_hba *phba)
3227 {
3228         struct be_mem_descriptor *mem_descr;
3229         struct mem_array *pm_arr;
3230         unsigned int page_offset, i;
3231         struct be_dma_mem sgl;
3232         int status;
3233
3234         mem_descr = phba->init_mem;
3235         mem_descr += HWI_MEM_SGE;
3236         pm_arr = mem_descr->mem_array;
3237
3238         page_offset = (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io *
3239                         phba->fw_config.iscsi_icd_start) / PAGE_SIZE;
3240         for (i = 0; i < mem_descr->num_elements; i++) {
3241                 hwi_build_be_sgl_arr(phba, pm_arr, &sgl);
3242                 status = be_cmd_iscsi_post_sgl_pages(&phba->ctrl, &sgl,
3243                                                 page_offset,
3244                                                 (pm_arr->size / PAGE_SIZE));
3245                 page_offset += pm_arr->size / PAGE_SIZE;
3246                 if (status != 0) {
3247                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3248                                     "BM_%d : post sgl failed.\n");
3249                         return status;
3250                 }
3251                 pm_arr++;
3252         }
3253         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3254                     "BM_%d : POSTED PAGES\n");
3255         return 0;
3256 }
3257
3258 static void be_queue_free(struct beiscsi_hba *phba, struct be_queue_info *q)
3259 {
3260         struct be_dma_mem *mem = &q->dma_mem;
3261         if (mem->va) {
3262                 pci_free_consistent(phba->pcidev, mem->size,
3263                         mem->va, mem->dma);
3264                 mem->va = NULL;
3265         }
3266 }
3267
3268 static int be_queue_alloc(struct beiscsi_hba *phba, struct be_queue_info *q,
3269                 u16 len, u16 entry_size)
3270 {
3271         struct be_dma_mem *mem = &q->dma_mem;
3272
3273         memset(q, 0, sizeof(*q));
3274         q->len = len;
3275         q->entry_size = entry_size;
3276         mem->size = len * entry_size;
3277         mem->va = pci_alloc_consistent(phba->pcidev, mem->size, &mem->dma);
3278         if (!mem->va)
3279                 return -ENOMEM;
3280         memset(mem->va, 0, mem->size);
3281         return 0;
3282 }
3283
3284 static int
3285 beiscsi_create_wrb_rings(struct beiscsi_hba *phba,
3286                          struct hwi_context_memory *phwi_context,
3287                          struct hwi_controller *phwi_ctrlr)
3288 {
3289         unsigned int wrb_mem_index, offset, size, num_wrb_rings;
3290         u64 pa_addr_lo;
3291         unsigned int idx, num, i;
3292         struct mem_array *pwrb_arr;
3293         void *wrb_vaddr;
3294         struct be_dma_mem sgl;
3295         struct be_mem_descriptor *mem_descr;
3296         int status;
3297
3298         idx = 0;
3299         mem_descr = phba->init_mem;
3300         mem_descr += HWI_MEM_WRB;
3301         pwrb_arr = kmalloc(sizeof(*pwrb_arr) * phba->params.cxns_per_ctrl,
3302                            GFP_KERNEL);
3303         if (!pwrb_arr) {
3304                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3305                             "BM_%d : Memory alloc failed in create wrb ring.\n");
3306                 return -ENOMEM;
3307         }
3308         wrb_vaddr = mem_descr->mem_array[idx].virtual_address;
3309         pa_addr_lo = mem_descr->mem_array[idx].bus_address.u.a64.address;
3310         num_wrb_rings = mem_descr->mem_array[idx].size /
3311                 (phba->params.wrbs_per_cxn * sizeof(struct iscsi_wrb));
3312
3313         for (num = 0; num < phba->params.cxns_per_ctrl; num++) {
3314                 if (num_wrb_rings) {
3315                         pwrb_arr[num].virtual_address = wrb_vaddr;
3316                         pwrb_arr[num].bus_address.u.a64.address = pa_addr_lo;
3317                         pwrb_arr[num].size = phba->params.wrbs_per_cxn *
3318                                             sizeof(struct iscsi_wrb);
3319                         wrb_vaddr += pwrb_arr[num].size;
3320                         pa_addr_lo += pwrb_arr[num].size;
3321                         num_wrb_rings--;
3322                 } else {
3323                         idx++;
3324                         wrb_vaddr = mem_descr->mem_array[idx].virtual_address;
3325                         pa_addr_lo = mem_descr->mem_array[idx].\
3326                                         bus_address.u.a64.address;
3327                         num_wrb_rings = mem_descr->mem_array[idx].size /
3328                                         (phba->params.wrbs_per_cxn *
3329                                         sizeof(struct iscsi_wrb));
3330                         pwrb_arr[num].virtual_address = wrb_vaddr;
3331                         pwrb_arr[num].bus_address.u.a64.address\
3332                                                 = pa_addr_lo;
3333                         pwrb_arr[num].size = phba->params.wrbs_per_cxn *
3334                                                  sizeof(struct iscsi_wrb);
3335                         wrb_vaddr += pwrb_arr[num].size;
3336                         pa_addr_lo   += pwrb_arr[num].size;
3337                         num_wrb_rings--;
3338                 }
3339         }
3340         for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
3341                 wrb_mem_index = 0;
3342                 offset = 0;
3343                 size = 0;
3344
3345                 hwi_build_be_sgl_by_offset(phba, &pwrb_arr[i], &sgl);
3346                 status = be_cmd_wrbq_create(&phba->ctrl, &sgl,
3347                                             &phwi_context->be_wrbq[i]);
3348                 if (status != 0) {
3349                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3350                                     "BM_%d : wrbq create failed.");
3351                         kfree(pwrb_arr);
3352                         return status;
3353                 }
3354                 phwi_ctrlr->wrb_context[i * 2].cid = phwi_context->be_wrbq[i].
3355                                                                    id;
3356         }
3357         kfree(pwrb_arr);
3358         return 0;
3359 }
3360
3361 static void free_wrb_handles(struct beiscsi_hba *phba)
3362 {
3363         unsigned int index;
3364         struct hwi_controller *phwi_ctrlr;
3365         struct hwi_wrb_context *pwrb_context;
3366
3367         phwi_ctrlr = phba->phwi_ctrlr;
3368         for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) {
3369                 pwrb_context = &phwi_ctrlr->wrb_context[index];
3370                 kfree(pwrb_context->pwrb_handle_base);
3371                 kfree(pwrb_context->pwrb_handle_basestd);
3372         }
3373 }
3374
3375 static void be_mcc_queues_destroy(struct beiscsi_hba *phba)
3376 {
3377         struct be_queue_info *q;
3378         struct be_ctrl_info *ctrl = &phba->ctrl;
3379
3380         q = &phba->ctrl.mcc_obj.q;
3381         if (q->created)
3382                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_MCCQ);
3383         be_queue_free(phba, q);
3384
3385         q = &phba->ctrl.mcc_obj.cq;
3386         if (q->created)
3387                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ);
3388         be_queue_free(phba, q);
3389 }
3390
3391 static void hwi_cleanup(struct beiscsi_hba *phba)
3392 {
3393         struct be_queue_info *q;
3394         struct be_ctrl_info *ctrl = &phba->ctrl;
3395         struct hwi_controller *phwi_ctrlr;
3396         struct hwi_context_memory *phwi_context;
3397         int i, eq_num;
3398
3399         phwi_ctrlr = phba->phwi_ctrlr;
3400         phwi_context = phwi_ctrlr->phwi_ctxt;
3401         for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
3402                 q = &phwi_context->be_wrbq[i];
3403                 if (q->created)
3404                         beiscsi_cmd_q_destroy(ctrl, q, QTYPE_WRBQ);
3405         }
3406         free_wrb_handles(phba);
3407
3408         q = &phwi_context->be_def_hdrq;
3409         if (q->created)
3410                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ);
3411
3412         q = &phwi_context->be_def_dataq;
3413         if (q->created)
3414                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ);
3415
3416         beiscsi_cmd_q_destroy(ctrl, NULL, QTYPE_SGL);
3417
3418         for (i = 0; i < (phba->num_cpus); i++) {
3419                 q = &phwi_context->be_cq[i];
3420                 if (q->created)
3421                         beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ);
3422         }
3423         if (phba->msix_enabled)
3424                 eq_num = 1;
3425         else
3426                 eq_num = 0;
3427         for (i = 0; i < (phba->num_cpus + eq_num); i++) {
3428                 q = &phwi_context->be_eq[i].q;
3429                 if (q->created)
3430                         beiscsi_cmd_q_destroy(ctrl, q, QTYPE_EQ);
3431         }
3432         be_mcc_queues_destroy(phba);
3433         be_cmd_fw_uninit(ctrl);
3434 }
3435
3436 static int be_mcc_queues_create(struct beiscsi_hba *phba,
3437                                 struct hwi_context_memory *phwi_context)
3438 {
3439         struct be_queue_info *q, *cq;
3440         struct be_ctrl_info *ctrl = &phba->ctrl;
3441
3442         /* Alloc MCC compl queue */
3443         cq = &phba->ctrl.mcc_obj.cq;
3444         if (be_queue_alloc(phba, cq, MCC_CQ_LEN,
3445                         sizeof(struct be_mcc_compl)))
3446                 goto err;
3447         /* Ask BE to create MCC compl queue; */
3448         if (phba->msix_enabled) {
3449                 if (beiscsi_cmd_cq_create(ctrl, cq, &phwi_context->be_eq
3450                                          [phba->num_cpus].q, false, true, 0))
3451                 goto mcc_cq_free;
3452         } else {
3453                 if (beiscsi_cmd_cq_create(ctrl, cq, &phwi_context->be_eq[0].q,
3454                                           false, true, 0))
3455                 goto mcc_cq_free;
3456         }
3457
3458         /* Alloc MCC queue */
3459         q = &phba->ctrl.mcc_obj.q;
3460         if (be_queue_alloc(phba, q, MCC_Q_LEN, sizeof(struct be_mcc_wrb)))
3461                 goto mcc_cq_destroy;
3462
3463         /* Ask BE to create MCC queue */
3464         if (beiscsi_cmd_mccq_create(phba, q, cq))
3465                 goto mcc_q_free;
3466
3467         return 0;
3468
3469 mcc_q_free:
3470         be_queue_free(phba, q);
3471 mcc_cq_destroy:
3472         beiscsi_cmd_q_destroy(ctrl, cq, QTYPE_CQ);
3473 mcc_cq_free:
3474         be_queue_free(phba, cq);
3475 err:
3476         return -ENOMEM;
3477 }
3478
3479 /**
3480  * find_num_cpus()- Get the CPU online count
3481  * @phba: ptr to priv structure
3482  *
3483  * CPU count is used for creating EQ.
3484  **/
3485 static void find_num_cpus(struct beiscsi_hba *phba)
3486 {
3487         int  num_cpus = 0;
3488
3489         num_cpus = num_online_cpus();
3490
3491         switch (phba->generation) {
3492         case BE_GEN2:
3493         case BE_GEN3:
3494                 phba->num_cpus = (num_cpus > BEISCSI_MAX_NUM_CPUS) ?
3495                                   BEISCSI_MAX_NUM_CPUS : num_cpus;
3496                 break;
3497         case BE_GEN4:
3498                 phba->num_cpus = (num_cpus > OC_SKH_MAX_NUM_CPUS) ?
3499                                   OC_SKH_MAX_NUM_CPUS : num_cpus;
3500                 break;
3501         default:
3502                 phba->num_cpus = 1;
3503         }
3504 }
3505
3506 static int hwi_init_port(struct beiscsi_hba *phba)
3507 {
3508         struct hwi_controller *phwi_ctrlr;
3509         struct hwi_context_memory *phwi_context;
3510         unsigned int def_pdu_ring_sz;
3511         struct be_ctrl_info *ctrl = &phba->ctrl;
3512         int status;
3513
3514         def_pdu_ring_sz =
3515                 phba->params.asyncpdus_per_ctrl * sizeof(struct phys_addr);
3516         phwi_ctrlr = phba->phwi_ctrlr;
3517         phwi_context = phwi_ctrlr->phwi_ctxt;
3518         phwi_context->max_eqd = 0;
3519         phwi_context->min_eqd = 0;
3520         phwi_context->cur_eqd = 64;
3521         be_cmd_fw_initialize(&phba->ctrl);
3522
3523         status = beiscsi_create_eqs(phba, phwi_context);
3524         if (status != 0) {
3525                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3526                             "BM_%d : EQ not created\n");
3527                 goto error;
3528         }
3529
3530         status = be_mcc_queues_create(phba, phwi_context);
3531         if (status != 0)
3532                 goto error;
3533
3534         status = mgmt_check_supported_fw(ctrl, phba);
3535         if (status != 0) {
3536                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3537                             "BM_%d : Unsupported fw version\n");
3538                 goto error;
3539         }
3540
3541         status = beiscsi_create_cqs(phba, phwi_context);
3542         if (status != 0) {
3543                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3544                             "BM_%d : CQ not created\n");
3545                 goto error;
3546         }
3547
3548         status = beiscsi_create_def_hdr(phba, phwi_context, phwi_ctrlr,
3549                                         def_pdu_ring_sz);
3550         if (status != 0) {
3551                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3552                             "BM_%d : Default Header not created\n");
3553                 goto error;
3554         }
3555
3556         status = beiscsi_create_def_data(phba, phwi_context,
3557                                          phwi_ctrlr, def_pdu_ring_sz);
3558         if (status != 0) {
3559                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3560                             "BM_%d : Default Data not created\n");
3561                 goto error;
3562         }
3563
3564         status = beiscsi_post_pages(phba);
3565         if (status != 0) {
3566                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3567                             "BM_%d : Post SGL Pages Failed\n");
3568                 goto error;
3569         }
3570
3571         status = beiscsi_create_wrb_rings(phba, phwi_context, phwi_ctrlr);
3572         if (status != 0) {
3573                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3574                             "BM_%d : WRB Rings not created\n");
3575                 goto error;
3576         }
3577
3578         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3579                     "BM_%d : hwi_init_port success\n");
3580         return 0;
3581
3582 error:
3583         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3584                     "BM_%d : hwi_init_port failed");
3585         hwi_cleanup(phba);
3586         return status;
3587 }
3588
3589 static int hwi_init_controller(struct beiscsi_hba *phba)
3590 {
3591         struct hwi_controller *phwi_ctrlr;
3592
3593         phwi_ctrlr = phba->phwi_ctrlr;
3594         if (1 == phba->init_mem[HWI_MEM_ADDN_CONTEXT].num_elements) {
3595                 phwi_ctrlr->phwi_ctxt = (struct hwi_context_memory *)phba->
3596                     init_mem[HWI_MEM_ADDN_CONTEXT].mem_array[0].virtual_address;
3597                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3598                             "BM_%d :  phwi_ctrlr->phwi_ctxt=%p\n",
3599                             phwi_ctrlr->phwi_ctxt);
3600         } else {
3601                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3602                             "BM_%d : HWI_MEM_ADDN_CONTEXT is more "
3603                             "than one element.Failing to load\n");
3604                 return -ENOMEM;
3605         }
3606
3607         iscsi_init_global_templates(phba);
3608         if (beiscsi_init_wrb_handle(phba))
3609                 return -ENOMEM;
3610
3611         hwi_init_async_pdu_ctx(phba);
3612         if (hwi_init_port(phba) != 0) {
3613                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3614                             "BM_%d : hwi_init_controller failed\n");
3615
3616                 return -ENOMEM;
3617         }
3618         return 0;
3619 }
3620
3621 static void beiscsi_free_mem(struct beiscsi_hba *phba)
3622 {
3623         struct be_mem_descriptor *mem_descr;
3624         int i, j;
3625
3626         mem_descr = phba->init_mem;
3627         i = 0;
3628         j = 0;
3629         for (i = 0; i < SE_MEM_MAX; i++) {
3630                 for (j = mem_descr->num_elements; j > 0; j--) {
3631                         pci_free_consistent(phba->pcidev,
3632                           mem_descr->mem_array[j - 1].size,
3633                           mem_descr->mem_array[j - 1].virtual_address,
3634                           (unsigned long)mem_descr->mem_array[j - 1].
3635                           bus_address.u.a64.address);
3636                 }
3637                 kfree(mem_descr->mem_array);
3638                 mem_descr++;
3639         }
3640         kfree(phba->init_mem);
3641         kfree(phba->phwi_ctrlr);
3642 }
3643
3644 static int beiscsi_init_controller(struct beiscsi_hba *phba)
3645 {
3646         int ret = -ENOMEM;
3647
3648         ret = beiscsi_get_memory(phba);
3649         if (ret < 0) {
3650                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3651                             "BM_%d : beiscsi_dev_probe -"
3652                             "Failed in beiscsi_alloc_memory\n");
3653                 return ret;
3654         }
3655
3656         ret = hwi_init_controller(phba);
3657         if (ret)
3658                 goto free_init;
3659         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3660                     "BM_%d : Return success from beiscsi_init_controller");
3661
3662         return 0;
3663
3664 free_init:
3665         beiscsi_free_mem(phba);
3666         return ret;
3667 }
3668
3669 static int beiscsi_init_sgl_handle(struct beiscsi_hba *phba)
3670 {
3671         struct be_mem_descriptor *mem_descr_sglh, *mem_descr_sg;
3672         struct sgl_handle *psgl_handle;
3673         struct iscsi_sge *pfrag;
3674         unsigned int arr_index, i, idx;
3675
3676         phba->io_sgl_hndl_avbl = 0;
3677         phba->eh_sgl_hndl_avbl = 0;
3678
3679         mem_descr_sglh = phba->init_mem;
3680         mem_descr_sglh += HWI_MEM_SGLH;
3681         if (1 == mem_descr_sglh->num_elements) {
3682                 phba->io_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) *
3683                                                  phba->params.ios_per_ctrl,
3684                                                  GFP_KERNEL);
3685                 if (!phba->io_sgl_hndl_base) {
3686                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3687                                     "BM_%d : Mem Alloc Failed. Failing to load\n");
3688                         return -ENOMEM;
3689                 }
3690                 phba->eh_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) *
3691                                                  (phba->params.icds_per_ctrl -
3692                                                  phba->params.ios_per_ctrl),
3693                                                  GFP_KERNEL);
3694                 if (!phba->eh_sgl_hndl_base) {
3695                         kfree(phba->io_sgl_hndl_base);
3696                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3697                                     "BM_%d : Mem Alloc Failed. Failing to load\n");
3698                         return -ENOMEM;
3699                 }
3700         } else {
3701                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3702                             "BM_%d : HWI_MEM_SGLH is more than one element."
3703                             "Failing to load\n");
3704                 return -ENOMEM;
3705         }
3706
3707         arr_index = 0;
3708         idx = 0;
3709         while (idx < mem_descr_sglh->num_elements) {
3710                 psgl_handle = mem_descr_sglh->mem_array[idx].virtual_address;
3711
3712                 for (i = 0; i < (mem_descr_sglh->mem_array[idx].size /
3713                       sizeof(struct sgl_handle)); i++) {
3714                         if (arr_index < phba->params.ios_per_ctrl) {
3715                                 phba->io_sgl_hndl_base[arr_index] = psgl_handle;
3716                                 phba->io_sgl_hndl_avbl++;
3717                                 arr_index++;
3718                         } else {
3719                                 phba->eh_sgl_hndl_base[arr_index -
3720                                         phba->params.ios_per_ctrl] =
3721                                                                 psgl_handle;
3722                                 arr_index++;
3723                                 phba->eh_sgl_hndl_avbl++;
3724                         }
3725                         psgl_handle++;
3726                 }
3727                 idx++;
3728         }
3729         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3730                     "BM_%d : phba->io_sgl_hndl_avbl=%d"
3731                     "phba->eh_sgl_hndl_avbl=%d\n",
3732                     phba->io_sgl_hndl_avbl,
3733                     phba->eh_sgl_hndl_avbl);
3734
3735         mem_descr_sg = phba->init_mem;
3736         mem_descr_sg += HWI_MEM_SGE;
3737         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3738                     "\n BM_%d : mem_descr_sg->num_elements=%d\n",
3739                     mem_descr_sg->num_elements);
3740
3741         arr_index = 0;
3742         idx = 0;
3743         while (idx < mem_descr_sg->num_elements) {
3744                 pfrag = mem_descr_sg->mem_array[idx].virtual_address;
3745
3746                 for (i = 0;
3747                      i < (mem_descr_sg->mem_array[idx].size) /
3748                      (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io);
3749                      i++) {
3750                         if (arr_index < phba->params.ios_per_ctrl)
3751                                 psgl_handle = phba->io_sgl_hndl_base[arr_index];
3752                         else
3753                                 psgl_handle = phba->eh_sgl_hndl_base[arr_index -
3754                                                 phba->params.ios_per_ctrl];
3755                         psgl_handle->pfrag = pfrag;
3756                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, pfrag, 0);
3757                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, pfrag, 0);
3758                         pfrag += phba->params.num_sge_per_io;
3759                         psgl_handle->sgl_index =
3760                                 phba->fw_config.iscsi_icd_start + arr_index++;
3761                 }
3762                 idx++;
3763         }
3764         phba->io_sgl_free_index = 0;
3765         phba->io_sgl_alloc_index = 0;
3766         phba->eh_sgl_free_index = 0;
3767         phba->eh_sgl_alloc_index = 0;
3768         return 0;
3769 }
3770
3771 static int hba_setup_cid_tbls(struct beiscsi_hba *phba)
3772 {
3773         int i, new_cid;
3774
3775         phba->cid_array = kzalloc(sizeof(void *) * phba->params.cxns_per_ctrl,
3776                                   GFP_KERNEL);
3777         if (!phba->cid_array) {
3778                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3779                             "BM_%d : Failed to allocate memory in "
3780                             "hba_setup_cid_tbls\n");
3781                 return -ENOMEM;
3782         }
3783         phba->ep_array = kzalloc(sizeof(struct iscsi_endpoint *) *
3784                                  phba->params.cxns_per_ctrl * 2, GFP_KERNEL);
3785         if (!phba->ep_array) {
3786                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3787                             "BM_%d : Failed to allocate memory in "
3788                             "hba_setup_cid_tbls\n");
3789                 kfree(phba->cid_array);
3790                 return -ENOMEM;
3791         }
3792         new_cid = phba->fw_config.iscsi_cid_start;
3793         for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
3794                 phba->cid_array[i] = new_cid;
3795                 new_cid += 2;
3796         }
3797         phba->avlbl_cids = phba->params.cxns_per_ctrl;
3798         return 0;
3799 }
3800
3801 static void hwi_enable_intr(struct beiscsi_hba *phba)
3802 {
3803         struct be_ctrl_info *ctrl = &phba->ctrl;
3804         struct hwi_controller *phwi_ctrlr;
3805         struct hwi_context_memory *phwi_context;
3806         struct be_queue_info *eq;
3807         u8 __iomem *addr;
3808         u32 reg, i;
3809         u32 enabled;
3810
3811         phwi_ctrlr = phba->phwi_ctrlr;
3812         phwi_context = phwi_ctrlr->phwi_ctxt;
3813
3814         addr = (u8 __iomem *) ((u8 __iomem *) ctrl->pcicfg +
3815                         PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET);
3816         reg = ioread32(addr);
3817
3818         enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3819         if (!enabled) {
3820                 reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3821                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3822                             "BM_%d : reg =x%08x addr=%p\n", reg, addr);
3823                 iowrite32(reg, addr);
3824         }
3825
3826         if (!phba->msix_enabled) {
3827                 eq = &phwi_context->be_eq[0].q;
3828                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3829                             "BM_%d : eq->id=%d\n", eq->id);
3830
3831                 hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1);
3832         } else {
3833                 for (i = 0; i <= phba->num_cpus; i++) {
3834                         eq = &phwi_context->be_eq[i].q;
3835                         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3836                                     "BM_%d : eq->id=%d\n", eq->id);
3837                         hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1);
3838                 }
3839         }
3840 }
3841
3842 static void hwi_disable_intr(struct beiscsi_hba *phba)
3843 {
3844         struct be_ctrl_info *ctrl = &phba->ctrl;
3845
3846         u8 __iomem *addr = ctrl->pcicfg + PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET;
3847         u32 reg = ioread32(addr);
3848
3849         u32 enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3850         if (enabled) {
3851                 reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3852                 iowrite32(reg, addr);
3853         } else
3854                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
3855                             "BM_%d : In hwi_disable_intr, Already Disabled\n");
3856 }
3857
3858 /**
3859  * beiscsi_get_boot_info()- Get the boot session info
3860  * @phba: The device priv structure instance
3861  *
3862  * Get the boot target info and store in driver priv structure
3863  *
3864  * return values
3865  *      Success: 0
3866  *      Failure: Non-Zero Value
3867  **/
3868 static int beiscsi_get_boot_info(struct beiscsi_hba *phba)
3869 {
3870         struct be_cmd_get_session_resp *session_resp;
3871         struct be_dma_mem nonemb_cmd;
3872         unsigned int tag;
3873         unsigned int s_handle;
3874         int ret = -ENOMEM;
3875
3876         /* Get the session handle of the boot target */
3877         ret = be_mgmt_get_boot_shandle(phba, &s_handle);
3878         if (ret) {
3879                 beiscsi_log(phba, KERN_ERR,
3880                             BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG,
3881                             "BM_%d : No boot session\n");
3882                 return ret;
3883         }
3884         nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
3885                                 sizeof(*session_resp),
3886                                 &nonemb_cmd.dma);
3887         if (nonemb_cmd.va == NULL) {
3888                 beiscsi_log(phba, KERN_ERR,
3889                             BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG,
3890                             "BM_%d : Failed to allocate memory for"
3891                             "beiscsi_get_session_info\n");
3892
3893                 return -ENOMEM;
3894         }
3895
3896         memset(nonemb_cmd.va, 0, sizeof(*session_resp));
3897         tag = mgmt_get_session_info(phba, s_handle,
3898                                     &nonemb_cmd);
3899         if (!tag) {
3900                 beiscsi_log(phba, KERN_ERR,
3901                             BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG,
3902                             "BM_%d : beiscsi_get_session_info"
3903                             " Failed\n");
3904
3905                 goto boot_freemem;
3906         }
3907
3908         ret = beiscsi_mccq_compl(phba, tag, NULL, nonemb_cmd.va);
3909         if (ret) {
3910                 beiscsi_log(phba, KERN_ERR,
3911                             BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG,
3912                             "BM_%d : beiscsi_get_session_info Failed");
3913                 goto boot_freemem;
3914         }
3915
3916         session_resp = nonemb_cmd.va ;
3917
3918         memcpy(&phba->boot_sess, &session_resp->session_info,
3919                sizeof(struct mgmt_session_info));
3920         ret = 0;
3921
3922 boot_freemem:
3923         pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
3924                     nonemb_cmd.va, nonemb_cmd.dma);
3925         return ret;
3926 }
3927
3928 static void beiscsi_boot_release(void *data)
3929 {
3930         struct beiscsi_hba *phba = data;
3931
3932         scsi_host_put(phba->shost);
3933 }
3934
3935 static int beiscsi_setup_boot_info(struct beiscsi_hba *phba)
3936 {
3937         struct iscsi_boot_kobj *boot_kobj;
3938
3939         /* get boot info using mgmt cmd */
3940         if (beiscsi_get_boot_info(phba))
3941                 /* Try to see if we can carry on without this */
3942                 return 0;
3943
3944         phba->boot_kset = iscsi_boot_create_host_kset(phba->shost->host_no);
3945         if (!phba->boot_kset)
3946                 return -ENOMEM;
3947
3948         /* get a ref because the show function will ref the phba */
3949         if (!scsi_host_get(phba->shost))
3950                 goto free_kset;
3951         boot_kobj = iscsi_boot_create_target(phba->boot_kset, 0, phba,
3952                                              beiscsi_show_boot_tgt_info,
3953                                              beiscsi_tgt_get_attr_visibility,
3954                                              beiscsi_boot_release);
3955         if (!boot_kobj)
3956                 goto put_shost;
3957
3958         if (!scsi_host_get(phba->shost))
3959                 goto free_kset;
3960         boot_kobj = iscsi_boot_create_initiator(phba->boot_kset, 0, phba,
3961                                                 beiscsi_show_boot_ini_info,
3962                                                 beiscsi_ini_get_attr_visibility,
3963                                                 beiscsi_boot_release);
3964         if (!boot_kobj)
3965                 goto put_shost;
3966
3967         if (!scsi_host_get(phba->shost))
3968                 goto free_kset;
3969         boot_kobj = iscsi_boot_create_ethernet(phba->boot_kset, 0, phba,
3970                                                beiscsi_show_boot_eth_info,
3971                                                beiscsi_eth_get_attr_visibility,
3972                                                beiscsi_boot_release);
3973         if (!boot_kobj)
3974                 goto put_shost;
3975         return 0;
3976
3977 put_shost:
3978         scsi_host_put(phba->shost);
3979 free_kset:
3980         iscsi_boot_destroy_kset(phba->boot_kset);
3981         return -ENOMEM;
3982 }
3983
3984 static int beiscsi_init_port(struct beiscsi_hba *phba)
3985 {
3986         int ret;
3987
3988         ret = beiscsi_init_controller(phba);
3989         if (ret < 0) {
3990                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3991                             "BM_%d : beiscsi_dev_probe - Failed in"
3992                             "beiscsi_init_controller\n");
3993                 return ret;
3994         }
3995         ret = beiscsi_init_sgl_handle(phba);
3996         if (ret < 0) {
3997                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3998                             "BM_%d : beiscsi_dev_probe - Failed in"
3999                             "beiscsi_init_sgl_handle\n");
4000                 goto do_cleanup_ctrlr;
4001         }
4002
4003         if (hba_setup_cid_tbls(phba)) {
4004                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4005                             "BM_%d : Failed in hba_setup_cid_tbls\n");
4006                 kfree(phba->io_sgl_hndl_base);
4007                 kfree(phba->eh_sgl_hndl_base);
4008                 goto do_cleanup_ctrlr;
4009         }
4010
4011         return ret;
4012
4013 do_cleanup_ctrlr:
4014         hwi_cleanup(phba);
4015         return ret;
4016 }
4017
4018 static void hwi_purge_eq(struct beiscsi_hba *phba)
4019 {
4020         struct hwi_controller *phwi_ctrlr;
4021         struct hwi_context_memory *phwi_context;
4022         struct be_queue_info *eq;
4023         struct be_eq_entry *eqe = NULL;
4024         int i, eq_msix;
4025         unsigned int num_processed;
4026
4027         phwi_ctrlr = phba->phwi_ctrlr;
4028         phwi_context = phwi_ctrlr->phwi_ctxt;
4029         if (phba->msix_enabled)
4030                 eq_msix = 1;
4031         else
4032                 eq_msix = 0;
4033
4034         for (i = 0; i < (phba->num_cpus + eq_msix); i++) {
4035                 eq = &phwi_context->be_eq[i].q;
4036                 eqe = queue_tail_node(eq);
4037                 num_processed = 0;
4038                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
4039                                         & EQE_VALID_MASK) {
4040                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
4041                         queue_tail_inc(eq);
4042                         eqe = queue_tail_node(eq);
4043                         num_processed++;
4044                 }
4045
4046                 if (num_processed)
4047                         hwi_ring_eq_db(phba, eq->id, 1, num_processed, 1, 1);
4048         }
4049 }
4050
4051 static void beiscsi_clean_port(struct beiscsi_hba *phba)
4052 {
4053         int mgmt_status;
4054
4055         mgmt_status = mgmt_epfw_cleanup(phba, CMD_CONNECTION_CHUTE_0);
4056         if (mgmt_status)
4057                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
4058                             "BM_%d : mgmt_epfw_cleanup FAILED\n");
4059
4060         hwi_purge_eq(phba);
4061         hwi_cleanup(phba);
4062         kfree(phba->io_sgl_hndl_base);
4063         kfree(phba->eh_sgl_hndl_base);
4064         kfree(phba->cid_array);
4065         kfree(phba->ep_array);
4066 }
4067
4068 /**
4069  * beiscsi_free_mgmt_task_handles()- Free driver CXN resources
4070  * @beiscsi_conn: ptr to the conn to be cleaned up
4071  *
4072  * Free driver mgmt resources binded to CXN.
4073  **/
4074 void
4075 beiscsi_free_mgmt_task_handles(struct beiscsi_conn *beiscsi_conn)
4076 {
4077         struct beiscsi_io_task *io_task;
4078         struct beiscsi_hba *phba = beiscsi_conn->phba;
4079         struct hwi_wrb_context *pwrb_context;
4080         struct hwi_controller *phwi_ctrlr;
4081
4082         phwi_ctrlr = phba->phwi_ctrlr;
4083         pwrb_context = &phwi_ctrlr->wrb_context
4084                        [beiscsi_conn->beiscsi_conn_cid
4085                        - phba->fw_config.iscsi_cid_start];
4086         io_task = beiscsi_conn->task->dd_data;
4087
4088         if (io_task->pwrb_handle) {
4089                 memset(io_task->pwrb_handle->pwrb, 0,
4090                        sizeof(struct iscsi_wrb));
4091                 free_wrb_handle(phba, pwrb_context,
4092                                 io_task->pwrb_handle);
4093                 io_task->pwrb_handle = NULL;
4094         }
4095
4096         if (io_task->psgl_handle) {
4097                 spin_lock_bh(&phba->mgmt_sgl_lock);
4098                 free_mgmt_sgl_handle(phba,
4099                                      io_task->psgl_handle);
4100                 spin_unlock_bh(&phba->mgmt_sgl_lock);
4101                 io_task->psgl_handle = NULL;
4102         }
4103
4104         if (io_task->mtask_addr)
4105                 pci_unmap_single(phba->pcidev,
4106                                  io_task->mtask_addr,
4107                                  io_task->mtask_data_count,
4108                                  PCI_DMA_TODEVICE);
4109 }
4110
4111 /**
4112  * beiscsi_cleanup_task()- Free driver resources of the task
4113  * @task: ptr to the iscsi task
4114  *
4115  **/
4116 static void beiscsi_cleanup_task(struct iscsi_task *task)
4117 {
4118         struct beiscsi_io_task *io_task = task->dd_data;
4119         struct iscsi_conn *conn = task->conn;
4120         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
4121         struct beiscsi_hba *phba = beiscsi_conn->phba;
4122         struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess;
4123         struct hwi_wrb_context *pwrb_context;
4124         struct hwi_controller *phwi_ctrlr;
4125
4126         phwi_ctrlr = phba->phwi_ctrlr;
4127         pwrb_context = &phwi_ctrlr->wrb_context[beiscsi_conn->beiscsi_conn_cid
4128                         - phba->fw_config.iscsi_cid_start];
4129
4130         if (io_task->cmd_bhs) {
4131                 pci_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs,
4132                               io_task->bhs_pa.u.a64.address);
4133                 io_task->cmd_bhs = NULL;
4134         }
4135
4136         if (task->sc) {
4137                 if (io_task->pwrb_handle) {
4138                         free_wrb_handle(phba, pwrb_context,
4139                                         io_task->pwrb_handle);
4140                         io_task->pwrb_handle = NULL;
4141                 }
4142
4143                 if (io_task->psgl_handle) {
4144                         spin_lock(&phba->io_sgl_lock);
4145                         free_io_sgl_handle(phba, io_task->psgl_handle);
4146                         spin_unlock(&phba->io_sgl_lock);
4147                         io_task->psgl_handle = NULL;
4148                 }
4149         } else {
4150                 if (!beiscsi_conn->login_in_progress)
4151                         beiscsi_free_mgmt_task_handles(beiscsi_conn);
4152         }
4153 }
4154
4155 void
4156 beiscsi_offload_connection(struct beiscsi_conn *beiscsi_conn,
4157                            struct beiscsi_offload_params *params)
4158 {
4159         struct wrb_handle *pwrb_handle;
4160         struct beiscsi_hba *phba = beiscsi_conn->phba;
4161         struct iscsi_task *task = beiscsi_conn->task;
4162         struct iscsi_session *session = task->conn->session;
4163         u32 doorbell = 0;
4164
4165         /*
4166          * We can always use 0 here because it is reserved by libiscsi for
4167          * login/startup related tasks.
4168          */
4169         beiscsi_conn->login_in_progress = 0;
4170         spin_lock_bh(&session->lock);
4171         beiscsi_cleanup_task(task);
4172         spin_unlock_bh(&session->lock);
4173
4174         pwrb_handle = alloc_wrb_handle(phba, (beiscsi_conn->beiscsi_conn_cid -
4175                                        phba->fw_config.iscsi_cid_start));
4176
4177         /* Check for the adapter family */
4178         if (chip_skh_r(phba->pcidev))
4179                 beiscsi_offload_cxn_v2(params, pwrb_handle);
4180         else
4181                 beiscsi_offload_cxn_v0(params, pwrb_handle,
4182                                        phba->init_mem);
4183
4184         be_dws_le_to_cpu(pwrb_handle->pwrb,
4185                          sizeof(struct iscsi_target_context_update_wrb));
4186
4187         doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
4188         doorbell |= (pwrb_handle->wrb_index & DB_DEF_PDU_WRB_INDEX_MASK)
4189                              << DB_DEF_PDU_WRB_INDEX_SHIFT;
4190         doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
4191
4192         iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
4193 }
4194
4195 static void beiscsi_parse_pdu(struct iscsi_conn *conn, itt_t itt,
4196                               int *index, int *age)
4197 {
4198         *index = (int)itt;
4199         if (age)
4200                 *age = conn->session->age;
4201 }
4202
4203 /**
4204  * beiscsi_alloc_pdu - allocates pdu and related resources
4205  * @task: libiscsi task
4206  * @opcode: opcode of pdu for task
4207  *
4208  * This is called with the session lock held. It will allocate
4209  * the wrb and sgl if needed for the command. And it will prep
4210  * the pdu's itt. beiscsi_parse_pdu will later translate
4211  * the pdu itt to the libiscsi task itt.
4212  */
4213 static int beiscsi_alloc_pdu(struct iscsi_task *task, uint8_t opcode)
4214 {
4215         struct beiscsi_io_task *io_task = task->dd_data;
4216         struct iscsi_conn *conn = task->conn;
4217         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
4218         struct beiscsi_hba *phba = beiscsi_conn->phba;
4219         struct hwi_wrb_context *pwrb_context;
4220         struct hwi_controller *phwi_ctrlr;
4221         itt_t itt;
4222         struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess;
4223         dma_addr_t paddr;
4224
4225         io_task->cmd_bhs = pci_pool_alloc(beiscsi_sess->bhs_pool,
4226                                           GFP_ATOMIC, &paddr);
4227         if (!io_task->cmd_bhs)
4228                 return -ENOMEM;
4229         io_task->bhs_pa.u.a64.address = paddr;
4230         io_task->libiscsi_itt = (itt_t)task->itt;
4231         io_task->conn = beiscsi_conn;
4232
4233         task->hdr = (struct iscsi_hdr *)&io_task->cmd_bhs->iscsi_hdr;
4234         task->hdr_max = sizeof(struct be_cmd_bhs);
4235         io_task->psgl_handle = NULL;
4236         io_task->pwrb_handle = NULL;
4237
4238         if (task->sc) {
4239                 spin_lock(&phba->io_sgl_lock);
4240                 io_task->psgl_handle = alloc_io_sgl_handle(phba);
4241                 spin_unlock(&phba->io_sgl_lock);
4242                 if (!io_task->psgl_handle) {
4243                         beiscsi_log(phba, KERN_ERR,
4244                                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
4245                                     "BM_%d : Alloc of IO_SGL_ICD Failed"
4246                                     "for the CID : %d\n",
4247                                     beiscsi_conn->beiscsi_conn_cid);
4248                         goto free_hndls;
4249                 }
4250                 io_task->pwrb_handle = alloc_wrb_handle(phba,
4251                                         beiscsi_conn->beiscsi_conn_cid -
4252                                         phba->fw_config.iscsi_cid_start);
4253                 if (!io_task->pwrb_handle) {
4254                         beiscsi_log(phba, KERN_ERR,
4255                                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
4256                                     "BM_%d : Alloc of WRB_HANDLE Failed"
4257                                     "for the CID : %d\n",
4258                                     beiscsi_conn->beiscsi_conn_cid);
4259                         goto free_io_hndls;
4260                 }
4261         } else {
4262                 io_task->scsi_cmnd = NULL;
4263                 if ((opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGIN) {
4264                         beiscsi_conn->task = task;
4265                         if (!beiscsi_conn->login_in_progress) {
4266                                 spin_lock(&phba->mgmt_sgl_lock);
4267                                 io_task->psgl_handle = (struct sgl_handle *)
4268                                                 alloc_mgmt_sgl_handle(phba);
4269                                 spin_unlock(&phba->mgmt_sgl_lock);
4270                                 if (!io_task->psgl_handle) {
4271                                         beiscsi_log(phba, KERN_ERR,
4272                                                     BEISCSI_LOG_IO |
4273                                                     BEISCSI_LOG_CONFIG,
4274                                                     "BM_%d : Alloc of MGMT_SGL_ICD Failed"
4275                                                     "for the CID : %d\n",
4276                                                     beiscsi_conn->
4277                                                     beiscsi_conn_cid);
4278                                         goto free_hndls;
4279                                 }
4280
4281                                 beiscsi_conn->login_in_progress = 1;
4282                                 beiscsi_conn->plogin_sgl_handle =
4283                                                         io_task->psgl_handle;
4284                                 io_task->pwrb_handle =
4285                                         alloc_wrb_handle(phba,
4286                                         beiscsi_conn->beiscsi_conn_cid -
4287                                         phba->fw_config.iscsi_cid_start);
4288                                 if (!io_task->pwrb_handle) {
4289                                         beiscsi_log(phba, KERN_ERR,
4290                                                     BEISCSI_LOG_IO |
4291                                                     BEISCSI_LOG_CONFIG,
4292                                                     "BM_%d : Alloc of WRB_HANDLE Failed"
4293                                                     "for the CID : %d\n",
4294                                                     beiscsi_conn->
4295                                                     beiscsi_conn_cid);
4296                                         goto free_mgmt_hndls;
4297                                 }
4298                                 beiscsi_conn->plogin_wrb_handle =
4299                                                         io_task->pwrb_handle;
4300
4301                         } else {
4302                                 io_task->psgl_handle =
4303                                                 beiscsi_conn->plogin_sgl_handle;
4304                                 io_task->pwrb_handle =
4305                                                 beiscsi_conn->plogin_wrb_handle;
4306                         }
4307                 } else {
4308                         spin_lock(&phba->mgmt_sgl_lock);
4309                         io_task->psgl_handle = alloc_mgmt_sgl_handle(phba);
4310                         spin_unlock(&phba->mgmt_sgl_lock);
4311                         if (!io_task->psgl_handle) {
4312                                 beiscsi_log(phba, KERN_ERR,
4313                                             BEISCSI_LOG_IO |
4314                                             BEISCSI_LOG_CONFIG,
4315                                             "BM_%d : Alloc of MGMT_SGL_ICD Failed"
4316                                             "for the CID : %d\n",
4317                                             beiscsi_conn->
4318                                             beiscsi_conn_cid);
4319                                 goto free_hndls;
4320                         }
4321                         io_task->pwrb_handle =
4322                                         alloc_wrb_handle(phba,
4323                                         beiscsi_conn->beiscsi_conn_cid -
4324                                         phba->fw_config.iscsi_cid_start);
4325                         if (!io_task->pwrb_handle) {
4326                                 beiscsi_log(phba, KERN_ERR,
4327                                             BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
4328                                             "BM_%d : Alloc of WRB_HANDLE Failed"
4329                                             "for the CID : %d\n",
4330                                             beiscsi_conn->beiscsi_conn_cid);
4331                                 goto free_mgmt_hndls;
4332                         }
4333
4334                 }
4335         }
4336         itt = (itt_t) cpu_to_be32(((unsigned int)io_task->pwrb_handle->
4337                                  wrb_index << 16) | (unsigned int)
4338                                 (io_task->psgl_handle->sgl_index));
4339         io_task->pwrb_handle->pio_handle = task;
4340
4341         io_task->cmd_bhs->iscsi_hdr.itt = itt;
4342         return 0;
4343
4344 free_io_hndls:
4345         spin_lock(&phba->io_sgl_lock);
4346         free_io_sgl_handle(phba, io_task->psgl_handle);
4347         spin_unlock(&phba->io_sgl_lock);
4348         goto free_hndls;
4349 free_mgmt_hndls:
4350         spin_lock(&phba->mgmt_sgl_lock);
4351         free_mgmt_sgl_handle(phba, io_task->psgl_handle);
4352         spin_unlock(&phba->mgmt_sgl_lock);
4353 free_hndls:
4354         phwi_ctrlr = phba->phwi_ctrlr;
4355         pwrb_context = &phwi_ctrlr->wrb_context[
4356                         beiscsi_conn->beiscsi_conn_cid -
4357                         phba->fw_config.iscsi_cid_start];
4358         if (io_task->pwrb_handle)
4359                 free_wrb_handle(phba, pwrb_context, io_task->pwrb_handle);
4360         io_task->pwrb_handle = NULL;
4361         pci_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs,
4362                       io_task->bhs_pa.u.a64.address);
4363         io_task->cmd_bhs = NULL;
4364         return -ENOMEM;
4365 }
4366 int beiscsi_iotask_v2(struct iscsi_task *task, struct scatterlist *sg,
4367                        unsigned int num_sg, unsigned int xferlen,
4368                        unsigned int writedir)
4369 {
4370
4371         struct beiscsi_io_task *io_task = task->dd_data;
4372         struct iscsi_conn *conn = task->conn;
4373         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
4374         struct beiscsi_hba *phba = beiscsi_conn->phba;
4375         struct iscsi_wrb *pwrb = NULL;
4376         unsigned int doorbell = 0;
4377
4378         pwrb = io_task->pwrb_handle->pwrb;
4379         memset(pwrb, 0, sizeof(*pwrb));
4380
4381         io_task->cmd_bhs->iscsi_hdr.exp_statsn = 0;
4382         io_task->bhs_len = sizeof(struct be_cmd_bhs);
4383
4384         if (writedir) {
4385                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, type, pwrb,
4386                               INI_WR_CMD);
4387                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, dsp, pwrb, 1);
4388         } else {
4389                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, type, pwrb,
4390                               INI_RD_CMD);
4391                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, dsp, pwrb, 0);
4392         }
4393
4394         io_task->wrb_type = AMAP_GET_BITS(struct amap_iscsi_wrb_v2,
4395                                           type, pwrb);
4396
4397         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, lun, pwrb,
4398                       cpu_to_be16(*(unsigned short *)
4399                       &io_task->cmd_bhs->iscsi_hdr.lun));
4400         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, r2t_exp_dtl, pwrb, xferlen);
4401         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, wrb_idx, pwrb,
4402                       io_task->pwrb_handle->wrb_index);
4403         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, cmdsn_itt, pwrb,
4404                       be32_to_cpu(task->cmdsn));
4405         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sgl_idx, pwrb,
4406                       io_task->psgl_handle->sgl_index);
4407
4408         hwi_write_sgl_v2(pwrb, sg, num_sg, io_task);
4409         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, ptr2nextwrb, pwrb,
4410                       io_task->pwrb_handle->nxt_wrb_index);
4411
4412         be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb));
4413
4414         doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
4415         doorbell |= (io_task->pwrb_handle->wrb_index &
4416                      DB_DEF_PDU_WRB_INDEX_MASK) <<
4417                      DB_DEF_PDU_WRB_INDEX_SHIFT;
4418         doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
4419         iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
4420         return 0;
4421 }
4422
4423 static int beiscsi_iotask(struct iscsi_task *task, struct scatterlist *sg,
4424                           unsigned int num_sg, unsigned int xferlen,
4425                           unsigned int writedir)
4426 {
4427
4428         struct beiscsi_io_task *io_task = task->dd_data;
4429         struct iscsi_conn *conn = task->conn;
4430         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
4431         struct beiscsi_hba *phba = beiscsi_conn->phba;
4432         struct iscsi_wrb *pwrb = NULL;
4433         unsigned int doorbell = 0;
4434
4435         pwrb = io_task->pwrb_handle->pwrb;
4436         io_task->cmd_bhs->iscsi_hdr.exp_statsn = 0;
4437         io_task->bhs_len = sizeof(struct be_cmd_bhs);
4438
4439         if (writedir) {
4440                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4441                               INI_WR_CMD);
4442                 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1);
4443         } else {
4444                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4445                               INI_RD_CMD);
4446                 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
4447         }
4448
4449         io_task->wrb_type = AMAP_GET_BITS(struct amap_iscsi_wrb,
4450                                           type, pwrb);
4451
4452         AMAP_SET_BITS(struct amap_iscsi_wrb, lun, pwrb,
4453                       cpu_to_be16(*(unsigned short *)
4454                                   &io_task->cmd_bhs->iscsi_hdr.lun));
4455         AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb, xferlen);
4456         AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb,
4457                       io_task->pwrb_handle->wrb_index);
4458         AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb,
4459                       be32_to_cpu(task->cmdsn));
4460         AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb,
4461                       io_task->psgl_handle->sgl_index);
4462
4463         hwi_write_sgl(pwrb, sg, num_sg, io_task);
4464
4465         AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb,
4466                       io_task->pwrb_handle->nxt_wrb_index);
4467         be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb));
4468
4469         doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
4470         doorbell |= (io_task->pwrb_handle->wrb_index &
4471                      DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT;
4472         doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
4473
4474         iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
4475         return 0;
4476 }
4477
4478 static int beiscsi_mtask(struct iscsi_task *task)
4479 {
4480         struct beiscsi_io_task *io_task = task->dd_data;
4481         struct iscsi_conn *conn = task->conn;
4482         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
4483         struct beiscsi_hba *phba = beiscsi_conn->phba;
4484         struct iscsi_wrb *pwrb = NULL;
4485         unsigned int doorbell = 0;
4486         unsigned int cid;
4487         unsigned int pwrb_typeoffset = 0;
4488
4489         cid = beiscsi_conn->beiscsi_conn_cid;
4490         pwrb = io_task->pwrb_handle->pwrb;
4491         memset(pwrb, 0, sizeof(*pwrb));
4492
4493         if (chip_skh_r(phba->pcidev)) {
4494                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, cmdsn_itt, pwrb,
4495                               be32_to_cpu(task->cmdsn));
4496                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, wrb_idx, pwrb,
4497                               io_task->pwrb_handle->wrb_index);
4498                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sgl_idx, pwrb,
4499                               io_task->psgl_handle->sgl_index);
4500                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, r2t_exp_dtl, pwrb,
4501                               task->data_count);
4502                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, ptr2nextwrb, pwrb,
4503                               io_task->pwrb_handle->nxt_wrb_index);
4504                 pwrb_typeoffset = SKH_WRB_TYPE_OFFSET;
4505         } else {
4506                 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb,
4507                               be32_to_cpu(task->cmdsn));
4508                 AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb,
4509                               io_task->pwrb_handle->wrb_index);
4510                 AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb,
4511                               io_task->psgl_handle->sgl_index);
4512                 AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb,
4513                               task->data_count);
4514                 AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb,
4515                               io_task->pwrb_handle->nxt_wrb_index);
4516                 pwrb_typeoffset = BE_WRB_TYPE_OFFSET;
4517         }
4518
4519
4520         switch (task->hdr->opcode & ISCSI_OPCODE_MASK) {
4521         case ISCSI_OP_LOGIN:
4522                 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb, 1);
4523                 ADAPTER_SET_WRB_TYPE(pwrb, TGT_DM_CMD, pwrb_typeoffset);
4524                 hwi_write_buffer(pwrb, task);
4525                 break;
4526         case ISCSI_OP_NOOP_OUT:
4527                 if (task->hdr->ttt != ISCSI_RESERVED_TAG) {
4528                         ADAPTER_SET_WRB_TYPE(pwrb, TGT_DM_CMD, pwrb_typeoffset);
4529                         if (chip_skh_r(phba->pcidev))
4530                                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
4531                                               dmsg, pwrb, 1);
4532                         else
4533                                 AMAP_SET_BITS(struct amap_iscsi_wrb,
4534                                               dmsg, pwrb, 1);
4535                 } else {
4536                         ADAPTER_SET_WRB_TYPE(pwrb, INI_RD_CMD, pwrb_typeoffset);
4537                         if (chip_skh_r(phba->pcidev))
4538                                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
4539                                               dmsg, pwrb, 0);
4540                         else
4541                                 AMAP_SET_BITS(struct amap_iscsi_wrb,
4542                                               dmsg, pwrb, 0);
4543                 }
4544                 hwi_write_buffer(pwrb, task);
4545                 break;
4546         case ISCSI_OP_TEXT:
4547                 ADAPTER_SET_WRB_TYPE(pwrb, TGT_DM_CMD, pwrb_typeoffset);
4548                 hwi_write_buffer(pwrb, task);
4549                 break;
4550         case ISCSI_OP_SCSI_TMFUNC:
4551                 ADAPTER_SET_WRB_TYPE(pwrb, INI_TMF_CMD, pwrb_typeoffset);
4552                 hwi_write_buffer(pwrb, task);
4553                 break;
4554         case ISCSI_OP_LOGOUT:
4555                 ADAPTER_SET_WRB_TYPE(pwrb, HWH_TYPE_LOGOUT, pwrb_typeoffset);
4556                 hwi_write_buffer(pwrb, task);
4557                 break;
4558
4559         default:
4560                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
4561                             "BM_%d : opcode =%d Not supported\n",
4562                             task->hdr->opcode & ISCSI_OPCODE_MASK);
4563
4564                 return -EINVAL;
4565         }
4566
4567         /* Set the task type */
4568         io_task->wrb_type = (chip_skh_r(phba->pcidev)) ?
4569                 AMAP_GET_BITS(struct amap_iscsi_wrb_v2, type, pwrb) :
4570                 AMAP_GET_BITS(struct amap_iscsi_wrb, type, pwrb);
4571
4572         doorbell |= cid & DB_WRB_POST_CID_MASK;
4573         doorbell |= (io_task->pwrb_handle->wrb_index &
4574                      DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT;
4575         doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
4576         iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
4577         return 0;
4578 }
4579
4580 static int beiscsi_task_xmit(struct iscsi_task *task)
4581 {
4582         struct beiscsi_io_task *io_task = task->dd_data;
4583         struct scsi_cmnd *sc = task->sc;
4584         struct beiscsi_hba *phba = NULL;
4585         struct scatterlist *sg;
4586         int num_sg;
4587         unsigned int  writedir = 0, xferlen = 0;
4588
4589         phba = ((struct beiscsi_conn *)task->conn->dd_data)->phba;
4590
4591         if (!sc)
4592                 return beiscsi_mtask(task);
4593
4594         io_task->scsi_cmnd = sc;
4595         num_sg = scsi_dma_map(sc);
4596         if (num_sg < 0) {
4597                 struct iscsi_conn *conn = task->conn;
4598                 struct beiscsi_hba *phba = NULL;
4599
4600                 phba = ((struct beiscsi_conn *)conn->dd_data)->phba;
4601                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_IO,
4602                             "BM_%d : scsi_dma_map Failed\n");
4603
4604                 return num_sg;
4605         }
4606         xferlen = scsi_bufflen(sc);
4607         sg = scsi_sglist(sc);
4608         if (sc->sc_data_direction == DMA_TO_DEVICE)
4609                 writedir = 1;
4610          else
4611                 writedir = 0;
4612
4613          return phba->iotask_fn(task, sg, num_sg, xferlen, writedir);
4614 }
4615
4616 /**
4617  * beiscsi_bsg_request - handle bsg request from ISCSI transport
4618  * @job: job to handle
4619  */
4620 static int beiscsi_bsg_request(struct bsg_job *job)
4621 {
4622         struct Scsi_Host *shost;
4623         struct beiscsi_hba *phba;
4624         struct iscsi_bsg_request *bsg_req = job->request;
4625         int rc = -EINVAL;
4626         unsigned int tag;
4627         struct be_dma_mem nonemb_cmd;
4628         struct be_cmd_resp_hdr *resp;
4629         struct iscsi_bsg_reply *bsg_reply = job->reply;
4630         unsigned short status, extd_status;
4631
4632         shost = iscsi_job_to_shost(job);
4633         phba = iscsi_host_priv(shost);
4634
4635         switch (bsg_req->msgcode) {
4636         case ISCSI_BSG_HST_VENDOR:
4637                 nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
4638                                         job->request_payload.payload_len,
4639                                         &nonemb_cmd.dma);
4640                 if (nonemb_cmd.va == NULL) {
4641                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
4642                                     "BM_%d : Failed to allocate memory for "
4643                                     "beiscsi_bsg_request\n");
4644                         return -ENOMEM;
4645                 }
4646                 tag = mgmt_vendor_specific_fw_cmd(&phba->ctrl, phba, job,
4647                                                   &nonemb_cmd);
4648                 if (!tag) {
4649                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
4650                                     "BM_%d : MBX Tag Allocation Failed\n");
4651
4652                         pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
4653                                             nonemb_cmd.va, nonemb_cmd.dma);
4654                         return -EAGAIN;
4655                 }
4656
4657                 rc = wait_event_interruptible_timeout(
4658                                         phba->ctrl.mcc_wait[tag],
4659                                         phba->ctrl.mcc_numtag[tag],
4660                                         msecs_to_jiffies(
4661                                         BEISCSI_HOST_MBX_TIMEOUT));
4662                 extd_status = (phba->ctrl.mcc_numtag[tag] & 0x0000FF00) >> 8;
4663                 status = phba->ctrl.mcc_numtag[tag] & 0x000000FF;
4664                 free_mcc_tag(&phba->ctrl, tag);
4665                 resp = (struct be_cmd_resp_hdr *)nonemb_cmd.va;
4666                 sg_copy_from_buffer(job->reply_payload.sg_list,
4667                                     job->reply_payload.sg_cnt,
4668                                     nonemb_cmd.va, (resp->response_length
4669                                     + sizeof(*resp)));
4670                 bsg_reply->reply_payload_rcv_len = resp->response_length;
4671                 bsg_reply->result = status;
4672                 bsg_job_done(job, bsg_reply->result,
4673                              bsg_reply->reply_payload_rcv_len);
4674                 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
4675                                     nonemb_cmd.va, nonemb_cmd.dma);
4676                 if (status || extd_status) {
4677                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
4678                                     "BM_%d : MBX Cmd Failed"
4679                                     " status = %d extd_status = %d\n",
4680                                     status, extd_status);
4681
4682                         return -EIO;
4683                 } else {
4684                         rc = 0;
4685                 }
4686                 break;
4687
4688         default:
4689                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
4690                                 "BM_%d : Unsupported bsg command: 0x%x\n",
4691                                 bsg_req->msgcode);
4692                 break;
4693         }
4694
4695         return rc;
4696 }
4697
4698 void beiscsi_hba_attrs_init(struct beiscsi_hba *phba)
4699 {
4700         /* Set the logging parameter */
4701         beiscsi_log_enable_init(phba, beiscsi_log_enable);
4702 }
4703
4704 /*
4705  * beiscsi_quiesce()- Cleanup Driver resources
4706  * @phba: Instance Priv structure
4707  *
4708  * Free the OS and HW resources held by the driver
4709  **/
4710 static void beiscsi_quiesce(struct beiscsi_hba *phba)
4711 {
4712         struct hwi_controller *phwi_ctrlr;
4713         struct hwi_context_memory *phwi_context;
4714         struct be_eq_obj *pbe_eq;
4715         unsigned int i, msix_vec;
4716
4717         phwi_ctrlr = phba->phwi_ctrlr;
4718         phwi_context = phwi_ctrlr->phwi_ctxt;
4719         hwi_disable_intr(phba);
4720         if (phba->msix_enabled) {
4721                 for (i = 0; i <= phba->num_cpus; i++) {
4722                         msix_vec = phba->msix_entries[i].vector;
4723                         free_irq(msix_vec, &phwi_context->be_eq[i]);
4724                         kfree(phba->msi_name[i]);
4725                 }
4726         } else
4727                 if (phba->pcidev->irq)
4728                         free_irq(phba->pcidev->irq, phba);
4729         pci_disable_msix(phba->pcidev);
4730         destroy_workqueue(phba->wq);
4731         if (blk_iopoll_enabled)
4732                 for (i = 0; i < phba->num_cpus; i++) {
4733                         pbe_eq = &phwi_context->be_eq[i];
4734                         blk_iopoll_disable(&pbe_eq->iopoll);
4735                 }
4736
4737         beiscsi_clean_port(phba);
4738         beiscsi_free_mem(phba);
4739
4740         beiscsi_unmap_pci_function(phba);
4741         pci_free_consistent(phba->pcidev,
4742                             phba->ctrl.mbox_mem_alloced.size,
4743                             phba->ctrl.mbox_mem_alloced.va,
4744                             phba->ctrl.mbox_mem_alloced.dma);
4745
4746         cancel_delayed_work_sync(&phba->beiscsi_hw_check_task);
4747 }
4748
4749 static void beiscsi_remove(struct pci_dev *pcidev)
4750 {
4751
4752         struct beiscsi_hba *phba = NULL;
4753
4754         phba = pci_get_drvdata(pcidev);
4755         if (!phba) {
4756                 dev_err(&pcidev->dev, "beiscsi_remove called with no phba\n");
4757                 return;
4758         }
4759
4760         beiscsi_destroy_def_ifaces(phba);
4761         beiscsi_quiesce(phba);
4762         iscsi_boot_destroy_kset(phba->boot_kset);
4763         iscsi_host_remove(phba->shost);
4764         pci_dev_put(phba->pcidev);
4765         iscsi_host_free(phba->shost);
4766         pci_disable_device(pcidev);
4767 }
4768
4769 static void beiscsi_shutdown(struct pci_dev *pcidev)
4770 {
4771
4772         struct beiscsi_hba *phba = NULL;
4773
4774         phba = (struct beiscsi_hba *)pci_get_drvdata(pcidev);
4775         if (!phba) {
4776                 dev_err(&pcidev->dev, "beiscsi_shutdown called with no phba\n");
4777                 return;
4778         }
4779
4780         beiscsi_quiesce(phba);
4781         pci_disable_device(pcidev);
4782 }
4783
4784 static void beiscsi_msix_enable(struct beiscsi_hba *phba)
4785 {
4786         int i, status;
4787
4788         for (i = 0; i <= phba->num_cpus; i++)
4789                 phba->msix_entries[i].entry = i;
4790
4791         status = pci_enable_msix(phba->pcidev, phba->msix_entries,
4792                                  (phba->num_cpus + 1));
4793         if (!status)
4794                 phba->msix_enabled = true;
4795
4796         return;
4797 }
4798
4799 /*
4800  * beiscsi_hw_health_check()- Check adapter health
4801  * @work: work item to check HW health
4802  *
4803  * Check if adapter in an unrecoverable state or not.
4804  **/
4805 static void
4806 beiscsi_hw_health_check(struct work_struct *work)
4807 {
4808         struct beiscsi_hba *phba =
4809                 container_of(work, struct beiscsi_hba,
4810                              beiscsi_hw_check_task.work);
4811
4812         beiscsi_ue_detect(phba);
4813
4814         schedule_delayed_work(&phba->beiscsi_hw_check_task,
4815                               msecs_to_jiffies(1000));
4816 }
4817
4818 static int beiscsi_dev_probe(struct pci_dev *pcidev,
4819                              const struct pci_device_id *id)
4820 {
4821         struct beiscsi_hba *phba = NULL;
4822         struct hwi_controller *phwi_ctrlr;
4823         struct hwi_context_memory *phwi_context;
4824         struct be_eq_obj *pbe_eq;
4825         int ret, i;
4826
4827         ret = beiscsi_enable_pci(pcidev);
4828         if (ret < 0) {
4829                 dev_err(&pcidev->dev,
4830                         "beiscsi_dev_probe - Failed to enable pci device\n");
4831                 return ret;
4832         }
4833
4834         phba = beiscsi_hba_alloc(pcidev);
4835         if (!phba) {
4836                 dev_err(&pcidev->dev,
4837                         "beiscsi_dev_probe - Failed in beiscsi_hba_alloc\n");
4838                 goto disable_pci;
4839         }
4840
4841         /* Initialize Driver configuration Paramters */
4842         beiscsi_hba_attrs_init(phba);
4843
4844         phba->fw_timeout = false;
4845
4846
4847         switch (pcidev->device) {
4848         case BE_DEVICE_ID1:
4849         case OC_DEVICE_ID1:
4850         case OC_DEVICE_ID2:
4851                 phba->generation = BE_GEN2;
4852                 phba->iotask_fn = beiscsi_iotask;
4853                 break;
4854         case BE_DEVICE_ID2:
4855         case OC_DEVICE_ID3:
4856                 phba->generation = BE_GEN3;
4857                 phba->iotask_fn = beiscsi_iotask;
4858                 break;
4859         case OC_SKH_ID1:
4860                 phba->generation = BE_GEN4;
4861                 phba->iotask_fn = beiscsi_iotask_v2;
4862                 break;
4863         default:
4864                 phba->generation = 0;
4865         }
4866
4867         if (enable_msix)
4868                 find_num_cpus(phba);
4869         else
4870                 phba->num_cpus = 1;
4871
4872         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
4873                     "BM_%d : num_cpus = %d\n",
4874                     phba->num_cpus);
4875
4876         if (enable_msix) {
4877                 beiscsi_msix_enable(phba);
4878                 if (!phba->msix_enabled)
4879                         phba->num_cpus = 1;
4880         }
4881         ret = be_ctrl_init(phba, pcidev);
4882         if (ret) {
4883                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4884                             "BM_%d : beiscsi_dev_probe-"
4885                             "Failed in be_ctrl_init\n");
4886                 goto hba_free;
4887         }
4888
4889         ret = beiscsi_cmd_reset_function(phba);
4890         if (ret) {
4891                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4892                             "BM_%d : Reset Failed. Aborting Crashdump\n");
4893                 goto hba_free;
4894         }
4895         ret = be_chk_reset_complete(phba);
4896         if (ret) {
4897                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4898                             "BM_%d : Failed to get out of reset."
4899                             "Aborting Crashdump\n");
4900                 goto hba_free;
4901         }
4902
4903         spin_lock_init(&phba->io_sgl_lock);
4904         spin_lock_init(&phba->mgmt_sgl_lock);
4905         spin_lock_init(&phba->isr_lock);
4906         ret = mgmt_get_fw_config(&phba->ctrl, phba);
4907         if (ret != 0) {
4908                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4909                             "BM_%d : Error getting fw config\n");
4910                 goto free_port;
4911         }
4912         phba->shost->max_id = phba->fw_config.iscsi_cid_count;
4913         beiscsi_get_params(phba);
4914         phba->shost->can_queue = phba->params.ios_per_ctrl;
4915         ret = beiscsi_init_port(phba);
4916         if (ret < 0) {
4917                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4918                             "BM_%d : beiscsi_dev_probe-"
4919                             "Failed in beiscsi_init_port\n");
4920                 goto free_port;
4921         }
4922
4923         for (i = 0; i < MAX_MCC_CMD ; i++) {
4924                 init_waitqueue_head(&phba->ctrl.mcc_wait[i + 1]);
4925                 phba->ctrl.mcc_tag[i] = i + 1;
4926                 phba->ctrl.mcc_numtag[i + 1] = 0;
4927                 phba->ctrl.mcc_tag_available++;
4928         }
4929
4930         phba->ctrl.mcc_alloc_index = phba->ctrl.mcc_free_index = 0;
4931
4932         snprintf(phba->wq_name, sizeof(phba->wq_name), "beiscsi_%02x_wq",
4933                  phba->shost->host_no);
4934         phba->wq = alloc_workqueue(phba->wq_name, WQ_MEM_RECLAIM, 1);
4935         if (!phba->wq) {
4936                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4937                             "BM_%d : beiscsi_dev_probe-"
4938                             "Failed to allocate work queue\n");
4939                 goto free_twq;
4940         }
4941
4942         INIT_DELAYED_WORK(&phba->beiscsi_hw_check_task,
4943                           beiscsi_hw_health_check);
4944
4945         phwi_ctrlr = phba->phwi_ctrlr;
4946         phwi_context = phwi_ctrlr->phwi_ctxt;
4947
4948         if (blk_iopoll_enabled) {
4949                 for (i = 0; i < phba->num_cpus; i++) {
4950                         pbe_eq = &phwi_context->be_eq[i];
4951                         blk_iopoll_init(&pbe_eq->iopoll, be_iopoll_budget,
4952                                         be_iopoll);
4953                         blk_iopoll_enable(&pbe_eq->iopoll);
4954                 }
4955
4956                 i = (phba->msix_enabled) ? i : 0;
4957                 /* Work item for MCC handling */
4958                 pbe_eq = &phwi_context->be_eq[i];
4959                 INIT_WORK(&pbe_eq->work_cqs, beiscsi_process_all_cqs);
4960         } else {
4961                 if (phba->msix_enabled) {
4962                         for (i = 0; i <= phba->num_cpus; i++) {
4963                                 pbe_eq = &phwi_context->be_eq[i];
4964                                 INIT_WORK(&pbe_eq->work_cqs,
4965                                           beiscsi_process_all_cqs);
4966                         }
4967                 } else {
4968                                 pbe_eq = &phwi_context->be_eq[0];
4969                                 INIT_WORK(&pbe_eq->work_cqs,
4970                                           beiscsi_process_all_cqs);
4971                         }
4972         }
4973
4974         ret = beiscsi_init_irqs(phba);
4975         if (ret < 0) {
4976                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4977                             "BM_%d : beiscsi_dev_probe-"
4978                             "Failed to beiscsi_init_irqs\n");
4979                 goto free_blkenbld;
4980         }
4981         hwi_enable_intr(phba);
4982
4983         if (beiscsi_setup_boot_info(phba))
4984                 /*
4985                  * log error but continue, because we may not be using
4986                  * iscsi boot.
4987                  */
4988                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4989                             "BM_%d : Could not set up "
4990                             "iSCSI boot info.\n");
4991
4992         beiscsi_create_def_ifaces(phba);
4993         schedule_delayed_work(&phba->beiscsi_hw_check_task,
4994                               msecs_to_jiffies(1000));
4995
4996         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
4997                     "\n\n\n BM_%d : SUCCESS - DRIVER LOADED\n\n\n");
4998         return 0;
4999
5000 free_blkenbld:
5001         destroy_workqueue(phba->wq);
5002         if (blk_iopoll_enabled)
5003                 for (i = 0; i < phba->num_cpus; i++) {
5004                         pbe_eq = &phwi_context->be_eq[i];
5005                         blk_iopoll_disable(&pbe_eq->iopoll);
5006                 }
5007 free_twq:
5008         beiscsi_clean_port(phba);
5009         beiscsi_free_mem(phba);
5010 free_port:
5011         pci_free_consistent(phba->pcidev,
5012                             phba->ctrl.mbox_mem_alloced.size,
5013                             phba->ctrl.mbox_mem_alloced.va,
5014                            phba->ctrl.mbox_mem_alloced.dma);
5015         beiscsi_unmap_pci_function(phba);
5016 hba_free:
5017         if (phba->msix_enabled)
5018                 pci_disable_msix(phba->pcidev);
5019         iscsi_host_remove(phba->shost);
5020         pci_dev_put(phba->pcidev);
5021         iscsi_host_free(phba->shost);
5022 disable_pci:
5023         pci_disable_device(pcidev);
5024         return ret;
5025 }
5026
5027 struct iscsi_transport beiscsi_iscsi_transport = {
5028         .owner = THIS_MODULE,
5029         .name = DRV_NAME,
5030         .caps = CAP_RECOVERY_L0 | CAP_HDRDGST | CAP_TEXT_NEGO |
5031                 CAP_MULTI_R2T | CAP_DATADGST | CAP_DATA_PATH_OFFLOAD,
5032         .create_session = beiscsi_session_create,
5033         .destroy_session = beiscsi_session_destroy,
5034         .create_conn = beiscsi_conn_create,
5035         .bind_conn = beiscsi_conn_bind,
5036         .destroy_conn = iscsi_conn_teardown,
5037         .attr_is_visible = be2iscsi_attr_is_visible,
5038         .set_iface_param = be2iscsi_iface_set_param,
5039         .get_iface_param = be2iscsi_iface_get_param,
5040         .set_param = beiscsi_set_param,
5041         .get_conn_param = iscsi_conn_get_param,
5042         .get_session_param = iscsi_session_get_param,
5043         .get_host_param = beiscsi_get_host_param,
5044         .start_conn = beiscsi_conn_start,
5045         .stop_conn = iscsi_conn_stop,
5046         .send_pdu = iscsi_conn_send_pdu,
5047         .xmit_task = beiscsi_task_xmit,
5048         .cleanup_task = beiscsi_cleanup_task,
5049         .alloc_pdu = beiscsi_alloc_pdu,
5050         .parse_pdu_itt = beiscsi_parse_pdu,
5051         .get_stats = beiscsi_conn_get_stats,
5052         .get_ep_param = beiscsi_ep_get_param,
5053         .ep_connect = beiscsi_ep_connect,
5054         .ep_poll = beiscsi_ep_poll,
5055         .ep_disconnect = beiscsi_ep_disconnect,
5056         .session_recovery_timedout = iscsi_session_recovery_timedout,
5057         .bsg_request = beiscsi_bsg_request,
5058 };
5059
5060 static struct pci_driver beiscsi_pci_driver = {
5061         .name = DRV_NAME,
5062         .probe = beiscsi_dev_probe,
5063         .remove = beiscsi_remove,
5064         .shutdown = beiscsi_shutdown,
5065         .id_table = beiscsi_pci_id_table
5066 };
5067
5068
5069 static int __init beiscsi_module_init(void)
5070 {
5071         int ret;
5072
5073         beiscsi_scsi_transport =
5074                         iscsi_register_transport(&beiscsi_iscsi_transport);
5075         if (!beiscsi_scsi_transport) {
5076                 printk(KERN_ERR
5077                        "beiscsi_module_init - Unable to  register beiscsi transport.\n");
5078                 return -ENOMEM;
5079         }
5080         printk(KERN_INFO "In beiscsi_module_init, tt=%p\n",
5081                &beiscsi_iscsi_transport);
5082
5083         ret = pci_register_driver(&beiscsi_pci_driver);
5084         if (ret) {
5085                 printk(KERN_ERR
5086                        "beiscsi_module_init - Unable to  register beiscsi pci driver.\n");
5087                 goto unregister_iscsi_transport;
5088         }
5089         return 0;
5090
5091 unregister_iscsi_transport:
5092         iscsi_unregister_transport(&beiscsi_iscsi_transport);
5093         return ret;
5094 }
5095
5096 static void __exit beiscsi_module_exit(void)
5097 {
5098         pci_unregister_driver(&beiscsi_pci_driver);
5099         iscsi_unregister_transport(&beiscsi_iscsi_transport);
5100 }
5101
5102 module_init(beiscsi_module_init);
5103 module_exit(beiscsi_module_exit);