drivers/scsi/be2iscsi/be_main.c

   1 /**
   2  * Copyright (C) 2005 - 2011 Emulex
   3  * All rights reserved.
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License version 2
   7  * as published by the Free Software Foundation.  The full GNU General
   8  * Public License is included in this distribution in the file called COPYING.
   9  *
  10  * Written by: Jayamohan Kallickal (jayamohan.kallickal@emulex.com)
  11  *
  12  * Contact Information:
  13  * linux-drivers@emulex.com
  14  *
  15  * Emulex
  16  * 3333 Susan Street
  17  * Costa Mesa, CA 92626
  18  */
  19
  20 #include <linux/reboot.h>
  21 #include <linux/delay.h>
  22 #include <linux/slab.h>
  23 #include <linux/interrupt.h>
  24 #include <linux/blkdev.h>
  25 #include <linux/pci.h>
  26 #include <linux/string.h>
  27 #include <linux/kernel.h>
  28 #include <linux/semaphore.h>
  29 #include <linux/iscsi_boot_sysfs.h>
  30 #include <linux/module.h>
  31
  32 #include <scsi/libiscsi.h>
  33 #include <scsi/scsi_transport_iscsi.h>
  34 #include <scsi/scsi_transport.h>
  35 #include <scsi/scsi_cmnd.h>
  36 #include <scsi/scsi_device.h>
  37 #include <scsi/scsi_host.h>
  38 #include <scsi/scsi.h>
  39 #include "be_main.h"
  40 #include "be_iscsi.h"
  41 #include "be_mgmt.h"
  42
  43 static unsigned int be_iopoll_budget = 10;
  44 static unsigned int be_max_phys_size = 64;
  45 static unsigned int enable_msix = 1;
  46 static unsigned int gcrashmode = 0;
  47 static unsigned int num_hba = 0;
  48
  49 MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
  50 MODULE_DESCRIPTION(DRV_DESC " " BUILD_STR);
  51 MODULE_AUTHOR("ServerEngines Corporation");
  52 MODULE_LICENSE("GPL");
  53 module_param(be_iopoll_budget, int, 0);
  54 module_param(enable_msix, int, 0);
  55 module_param(be_max_phys_size, uint, S_IRUGO);
  56 MODULE_PARM_DESC(be_max_phys_size, "Maximum Size (In Kilobytes) of physically"
  57                                    "contiguous memory that can be allocated."
  58                                    "Range is 16 - 128");
  59
  60 static int beiscsi_slave_configure(struct scsi_device *sdev)
  61 {
  62         blk_queue_max_segment_size(sdev->request_queue, 65536);
  63         return 0;
  64 }
  65
  66 static int beiscsi_eh_abort(struct scsi_cmnd *sc)
  67 {
  68         struct iscsi_cls_session *cls_session;
  69         struct iscsi_task *aborted_task = (struct iscsi_task *)sc->SCp.ptr;
  70         struct beiscsi_io_task *aborted_io_task;
  71         struct iscsi_conn *conn;
  72         struct beiscsi_conn *beiscsi_conn;
  73         struct beiscsi_hba *phba;
  74         struct iscsi_session *session;
  75         struct invalidate_command_table *inv_tbl;
  76         struct be_dma_mem nonemb_cmd;
  77         unsigned int cid, tag, num_invalidate;
  78
  79         cls_session = starget_to_session(scsi_target(sc->device));
  80         session = cls_session->dd_data;
  81
  82         spin_lock_bh(&session->lock);
  83         if (!aborted_task || !aborted_task->sc) {
  84                 /* we raced */
  85                 spin_unlock_bh(&session->lock);
  86                 return SUCCESS;
  87         }
  88
  89         aborted_io_task = aborted_task->dd_data;
  90         if (!aborted_io_task->scsi_cmnd) {
  91                 /* raced or invalid command */
  92                 spin_unlock_bh(&session->lock);
  93                 return SUCCESS;
  94         }
  95         spin_unlock_bh(&session->lock);
  96         conn = aborted_task->conn;
  97         beiscsi_conn = conn->dd_data;
  98         phba = beiscsi_conn->phba;
  99
 100         /* invalidate iocb */
 101         cid = beiscsi_conn->beiscsi_conn_cid;
 102         inv_tbl = phba->inv_tbl;
 103         memset(inv_tbl, 0x0, sizeof(*inv_tbl));
 104         inv_tbl->cid = cid;
 105         inv_tbl->icd = aborted_io_task->psgl_handle->sgl_index;
 106         num_invalidate = 1;
 107         nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
 108                                 sizeof(struct invalidate_commands_params_in),
 109                                 &nonemb_cmd.dma);
 110         if (nonemb_cmd.va == NULL) {
 111                 SE_DEBUG(DBG_LVL_1,
 112                          "Failed to allocate memory for"
 113                          "mgmt_invalidate_icds\n");
 114                 return FAILED;
 115         }
 116         nonemb_cmd.size = sizeof(struct invalidate_commands_params_in);
 117
 118         tag = mgmt_invalidate_icds(phba, inv_tbl, num_invalidate,
 119                                    cid, &nonemb_cmd);
 120         if (!tag) {
 121                 shost_printk(KERN_WARNING, phba->shost,
 122                              "mgmt_invalidate_icds could not be"
 123                              " submitted\n");
 124                 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
 125                                     nonemb_cmd.va, nonemb_cmd.dma);
 126
 127                 return FAILED;
 128         } else {
 129                 wait_event_interruptible(phba->ctrl.mcc_wait[tag],
 130                                          phba->ctrl.mcc_numtag[tag]);
 131                 free_mcc_tag(&phba->ctrl, tag);
 132         }
 133         pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
 134                             nonemb_cmd.va, nonemb_cmd.dma);
 135         return iscsi_eh_abort(sc);
 136 }
 137
 138 static int beiscsi_eh_device_reset(struct scsi_cmnd *sc)
 139 {
 140         struct iscsi_task *abrt_task;
 141         struct beiscsi_io_task *abrt_io_task;
 142         struct iscsi_conn *conn;
 143         struct beiscsi_conn *beiscsi_conn;
 144         struct beiscsi_hba *phba;
 145         struct iscsi_session *session;
 146         struct iscsi_cls_session *cls_session;
 147         struct invalidate_command_table *inv_tbl;
 148         struct be_dma_mem nonemb_cmd;
 149         unsigned int cid, tag, i, num_invalidate;
 150         int rc = FAILED;
 151
 152         /* invalidate iocbs */
 153         cls_session = starget_to_session(scsi_target(sc->device));
 154         session = cls_session->dd_data;
 155         spin_lock_bh(&session->lock);
 156         if (!session->leadconn || session->state != ISCSI_STATE_LOGGED_IN)
 157                 goto unlock;
 158
 159         conn = session->leadconn;
 160         beiscsi_conn = conn->dd_data;
 161         phba = beiscsi_conn->phba;
 162         cid = beiscsi_conn->beiscsi_conn_cid;
 163         inv_tbl = phba->inv_tbl;
 164         memset(inv_tbl, 0x0, sizeof(*inv_tbl) * BE2_CMDS_PER_CXN);
 165         num_invalidate = 0;
 166         for (i = 0; i < conn->session->cmds_max; i++) {
 167                 abrt_task = conn->session->cmds[i];
 168                 abrt_io_task = abrt_task->dd_data;
 169                 if (!abrt_task->sc || abrt_task->state == ISCSI_TASK_FREE)
 170                         continue;
 171
 172                 if (abrt_task->sc->device->lun != abrt_task->sc->device->lun)
 173                         continue;
 174
 175                 inv_tbl->cid = cid;
 176                 inv_tbl->icd = abrt_io_task->psgl_handle->sgl_index;
 177                 num_invalidate++;
 178                 inv_tbl++;
 179         }
 180         spin_unlock_bh(&session->lock);
 181         inv_tbl = phba->inv_tbl;
 182
 183         nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
 184                                 sizeof(struct invalidate_commands_params_in),
 185                                 &nonemb_cmd.dma);
 186         if (nonemb_cmd.va == NULL) {
 187                 SE_DEBUG(DBG_LVL_1,
 188                          "Failed to allocate memory for"
 189                          "mgmt_invalidate_icds\n");
 190                 return FAILED;
 191         }
 192         nonemb_cmd.size = sizeof(struct invalidate_commands_params_in);
 193         memset(nonemb_cmd.va, 0, nonemb_cmd.size);
 194         tag = mgmt_invalidate_icds(phba, inv_tbl, num_invalidate,
 195                                    cid, &nonemb_cmd);
 196         if (!tag) {
 197                 shost_printk(KERN_WARNING, phba->shost,
 198                              "mgmt_invalidate_icds could not be"
 199                              " submitted\n");
 200                 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
 201                                     nonemb_cmd.va, nonemb_cmd.dma);
 202                 return FAILED;
 203         } else {
 204                 wait_event_interruptible(phba->ctrl.mcc_wait[tag],
 205                                          phba->ctrl.mcc_numtag[tag]);
 206                 free_mcc_tag(&phba->ctrl, tag);
 207         }
 208         pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
 209                             nonemb_cmd.va, nonemb_cmd.dma);
 210         return iscsi_eh_device_reset(sc);
 211 unlock:
 212         spin_unlock_bh(&session->lock);
 213         return rc;
 214 }
 215
 216 static ssize_t beiscsi_show_boot_tgt_info(void *data, int type, char *buf)
 217 {
 218         struct beiscsi_hba *phba = data;
 219         struct mgmt_session_info *boot_sess = &phba->boot_sess;
 220         struct mgmt_conn_info *boot_conn = &boot_sess->conn_list[0];
 221         char *str = buf;
 222         int rc;
 223
 224         switch (type) {
 225         case ISCSI_BOOT_TGT_NAME:
 226                 rc = sprintf(buf, "%.*s\n",
 227                             (int)strlen(boot_sess->target_name),
 228                             (char *)&boot_sess->target_name);
 229                 break;
 230         case ISCSI_BOOT_TGT_IP_ADDR:
 231                 if (boot_conn->dest_ipaddr.ip_type == 0x1)
 232                         rc = sprintf(buf, "%pI4\n",
 233                                 (char *)&boot_conn->dest_ipaddr.ip_address);
 234                 else
 235                         rc = sprintf(str, "%pI6\n",
 236                                 (char *)&boot_conn->dest_ipaddr.ip_address);
 237                 break;
 238         case ISCSI_BOOT_TGT_PORT:
 239                 rc = sprintf(str, "%d\n", boot_conn->dest_port);
 240                 break;
 241
 242         case ISCSI_BOOT_TGT_CHAP_NAME:
 243                 rc = sprintf(str,  "%.*s\n",
 244                              boot_conn->negotiated_login_options.auth_data.chap.
 245                              target_chap_name_length,
 246                              (char *)&boot_conn->negotiated_login_options.
 247                              auth_data.chap.target_chap_name);
 248                 break;
 249         case ISCSI_BOOT_TGT_CHAP_SECRET:
 250                 rc = sprintf(str,  "%.*s\n",
 251                              boot_conn->negotiated_login_options.auth_data.chap.
 252                              target_secret_length,
 253                              (char *)&boot_conn->negotiated_login_options.
 254                              auth_data.chap.target_secret);
 255                 break;
 256         case ISCSI_BOOT_TGT_REV_CHAP_NAME:
 257                 rc = sprintf(str,  "%.*s\n",
 258                              boot_conn->negotiated_login_options.auth_data.chap.
 259                              intr_chap_name_length,
 260                              (char *)&boot_conn->negotiated_login_options.
 261                              auth_data.chap.intr_chap_name);
 262                 break;
 263         case ISCSI_BOOT_TGT_REV_CHAP_SECRET:
 264                 rc = sprintf(str,  "%.*s\n",
 265                              boot_conn->negotiated_login_options.auth_data.chap.
 266                              intr_secret_length,
 267                              (char *)&boot_conn->negotiated_login_options.
 268                              auth_data.chap.intr_secret);
 269                 break;
 270         case ISCSI_BOOT_TGT_FLAGS:
 271                 rc = sprintf(str, "2\n");
 272                 break;
 273         case ISCSI_BOOT_TGT_NIC_ASSOC:
 274                 rc = sprintf(str, "0\n");
 275                 break;
 276         default:
 277                 rc = -ENOSYS;
 278                 break;
 279         }
 280         return rc;
 281 }
 282
 283 static ssize_t beiscsi_show_boot_ini_info(void *data, int type, char *buf)
 284 {
 285         struct beiscsi_hba *phba = data;
 286         char *str = buf;
 287         int rc;
 288
 289         switch (type) {
 290         case ISCSI_BOOT_INI_INITIATOR_NAME:
 291                 rc = sprintf(str, "%s\n", phba->boot_sess.initiator_iscsiname);
 292                 break;
 293         default:
 294                 rc = -ENOSYS;
 295                 break;
 296         }
 297         return rc;
 298 }
 299
 300 static ssize_t beiscsi_show_boot_eth_info(void *data, int type, char *buf)
 301 {
 302         struct beiscsi_hba *phba = data;
 303         char *str = buf;
 304         int rc;
 305
 306         switch (type) {
 307         case ISCSI_BOOT_ETH_FLAGS:
 308                 rc = sprintf(str, "2\n");
 309                 break;
 310         case ISCSI_BOOT_ETH_INDEX:
 311                 rc = sprintf(str, "0\n");
 312                 break;
 313         case ISCSI_BOOT_ETH_MAC:
 314                 rc  = beiscsi_get_macaddr(buf, phba);
 315                 if (rc < 0) {
 316                         SE_DEBUG(DBG_LVL_1, "beiscsi_get_macaddr Failed\n");
 317                         return rc;
 318                 }
 319         break;
 320         default:
 321                 rc = -ENOSYS;
 322                 break;
 323         }
 324         return rc;
 325 }
 326
 327
 328 static mode_t beiscsi_tgt_get_attr_visibility(void *data, int type)
 329 {
 330         int rc;
 331
 332         switch (type) {
 333         case ISCSI_BOOT_TGT_NAME:
 334         case ISCSI_BOOT_TGT_IP_ADDR:
 335         case ISCSI_BOOT_TGT_PORT:
 336         case ISCSI_BOOT_TGT_CHAP_NAME:
 337         case ISCSI_BOOT_TGT_CHAP_SECRET:
 338         case ISCSI_BOOT_TGT_REV_CHAP_NAME:
 339         case ISCSI_BOOT_TGT_REV_CHAP_SECRET:
 340         case ISCSI_BOOT_TGT_NIC_ASSOC:
 341         case ISCSI_BOOT_TGT_FLAGS:
 342                 rc = S_IRUGO;
 343                 break;
 344         default:
 345                 rc = 0;
 346                 break;
 347         }
 348         return rc;
 349 }
 350
 351 static mode_t beiscsi_ini_get_attr_visibility(void *data, int type)
 352 {
 353         int rc;
 354
 355         switch (type) {
 356         case ISCSI_BOOT_INI_INITIATOR_NAME:
 357                 rc = S_IRUGO;
 358                 break;
 359         default:
 360                 rc = 0;
 361                 break;
 362         }
 363         return rc;
 364 }
 365
 366
 367 static mode_t beiscsi_eth_get_attr_visibility(void *data, int type)
 368 {
 369         int rc;
 370
 371         switch (type) {
 372         case ISCSI_BOOT_ETH_FLAGS:
 373         case ISCSI_BOOT_ETH_MAC:
 374         case ISCSI_BOOT_ETH_INDEX:
 375                 rc = S_IRUGO;
 376                 break;
 377         default:
 378                 rc = 0;
 379                 break;
 380         }
 381         return rc;
 382 }
 383
 384 /*------------------- PCI Driver operations and data ----------------- */
 385 static DEFINE_PCI_DEVICE_TABLE(beiscsi_pci_id_table) = {
 386         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) },
 387         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID2) },
 388         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) },
 389         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) },
 390         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID3) },
 391         { 0 }
 392 };
 393 MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
 394
 395 static struct scsi_host_template beiscsi_sht = {
 396         .module = THIS_MODULE,
 397         .name = "ServerEngines 10Gbe open-iscsi Initiator Driver",
 398         .proc_name = DRV_NAME,
 399         .queuecommand = iscsi_queuecommand,
 400         .change_queue_depth = iscsi_change_queue_depth,
 401         .slave_configure = beiscsi_slave_configure,
 402         .target_alloc = iscsi_target_alloc,
 403         .eh_abort_handler = beiscsi_eh_abort,
 404         .eh_device_reset_handler = beiscsi_eh_device_reset,
 405         .eh_target_reset_handler = iscsi_eh_session_reset,
 406         .sg_tablesize = BEISCSI_SGLIST_ELEMENTS,
 407         .can_queue = BE2_IO_DEPTH,
 408         .this_id = -1,
 409         .max_sectors = BEISCSI_MAX_SECTORS,
 410         .cmd_per_lun = BEISCSI_CMD_PER_LUN,
 411         .use_clustering = ENABLE_CLUSTERING,
 412 };
 413
 414 static struct scsi_transport_template *beiscsi_scsi_transport;
 415
 416 static struct beiscsi_hba *beiscsi_hba_alloc(struct pci_dev *pcidev)
 417 {
 418         struct beiscsi_hba *phba;
 419         struct Scsi_Host *shost;
 420
 421         shost = iscsi_host_alloc(&beiscsi_sht, sizeof(*phba), 0);
 422         if (!shost) {
 423                 dev_err(&pcidev->dev, "beiscsi_hba_alloc -"
 424                         "iscsi_host_alloc failed\n");
 425                 return NULL;
 426         }
 427         shost->dma_boundary = pcidev->dma_mask;
 428         shost->max_id = BE2_MAX_SESSIONS;
 429         shost->max_channel = 0;
 430         shost->max_cmd_len = BEISCSI_MAX_CMD_LEN;
 431         shost->max_lun = BEISCSI_NUM_MAX_LUN;
 432         shost->transportt = beiscsi_scsi_transport;
 433         phba = iscsi_host_priv(shost);
 434         memset(phba, 0, sizeof(*phba));
 435         phba->shost = shost;
 436         phba->pcidev = pci_dev_get(pcidev);
 437         pci_set_drvdata(pcidev, phba);
 438
 439         if (iscsi_host_add(shost, &phba->pcidev->dev))
 440                 goto free_devices;
 441
 442         return phba;
 443
 444 free_devices:
 445         pci_dev_put(phba->pcidev);
 446         iscsi_host_free(phba->shost);
 447         return NULL;
 448 }
 449
 450 static void beiscsi_unmap_pci_function(struct beiscsi_hba *phba)
 451 {
 452         if (phba->csr_va) {
 453                 iounmap(phba->csr_va);
 454                 phba->csr_va = NULL;
 455         }
 456         if (phba->db_va) {
 457                 iounmap(phba->db_va);
 458                 phba->db_va = NULL;
 459         }
 460         if (phba->pci_va) {
 461                 iounmap(phba->pci_va);
 462                 phba->pci_va = NULL;
 463         }
 464 }
 465
 466 static int beiscsi_map_pci_bars(struct beiscsi_hba *phba,
 467                                 struct pci_dev *pcidev)
 468 {
 469         u8 __iomem *addr;
 470         int pcicfg_reg;
 471
 472         addr = ioremap_nocache(pci_resource_start(pcidev, 2),
 473                                pci_resource_len(pcidev, 2));
 474         if (addr == NULL)
 475                 return -ENOMEM;
 476         phba->ctrl.csr = addr;
 477         phba->csr_va = addr;
 478         phba->csr_pa.u.a64.address = pci_resource_start(pcidev, 2);
 479
 480         addr = ioremap_nocache(pci_resource_start(pcidev, 4), 128 * 1024);
 481         if (addr == NULL)
 482                 goto pci_map_err;
 483         phba->ctrl.db = addr;
 484         phba->db_va = addr;
 485         phba->db_pa.u.a64.address =  pci_resource_start(pcidev, 4);
 486
 487         if (phba->generation == BE_GEN2)
 488                 pcicfg_reg = 1;
 489         else
 490                 pcicfg_reg = 0;
 491
 492         addr = ioremap_nocache(pci_resource_start(pcidev, pcicfg_reg),
 493                                pci_resource_len(pcidev, pcicfg_reg));
 494
 495         if (addr == NULL)
 496                 goto pci_map_err;
 497         phba->ctrl.pcicfg = addr;
 498         phba->pci_va = addr;
 499         phba->pci_pa.u.a64.address = pci_resource_start(pcidev, pcicfg_reg);
 500         return 0;
 501
 502 pci_map_err:
 503         beiscsi_unmap_pci_function(phba);
 504         return -ENOMEM;
 505 }
 506
 507 static int beiscsi_enable_pci(struct pci_dev *pcidev)
 508 {
 509         int ret;
 510
 511         ret = pci_enable_device(pcidev);
 512         if (ret) {
 513                 dev_err(&pcidev->dev, "beiscsi_enable_pci - enable device "
 514                         "failed. Returning -ENODEV\n");
 515                 return ret;
 516         }
 517
 518         pci_set_master(pcidev);
 519         if (pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(64))) {
 520                 ret = pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(32));
 521                 if (ret) {
 522                         dev_err(&pcidev->dev, "Could not set PCI DMA Mask\n");
 523                         pci_disable_device(pcidev);
 524                         return ret;
 525                 }
 526         }
 527         return 0;
 528 }
 529
 530 static int be_ctrl_init(struct beiscsi_hba *phba, struct pci_dev *pdev)
 531 {
 532         struct be_ctrl_info *ctrl = &phba->ctrl;
 533         struct be_dma_mem *mbox_mem_alloc = &ctrl->mbox_mem_alloced;
 534         struct be_dma_mem *mbox_mem_align = &ctrl->mbox_mem;
 535         int status = 0;
 536
 537         ctrl->pdev = pdev;
 538         status = beiscsi_map_pci_bars(phba, pdev);
 539         if (status)
 540                 return status;
 541         mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16;
 542         mbox_mem_alloc->va = pci_alloc_consistent(pdev,
 543                                                   mbox_mem_alloc->size,
 544                                                   &mbox_mem_alloc->dma);
 545         if (!mbox_mem_alloc->va) {
 546                 beiscsi_unmap_pci_function(phba);
 547                 status = -ENOMEM;
 548                 return status;
 549         }
 550
 551         mbox_mem_align->size = sizeof(struct be_mcc_mailbox);
 552         mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16);
 553         mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16);
 554         memset(mbox_mem_align->va, 0, sizeof(struct be_mcc_mailbox));
 555         spin_lock_init(&ctrl->mbox_lock);
 556         spin_lock_init(&phba->ctrl.mcc_lock);
 557         spin_lock_init(&phba->ctrl.mcc_cq_lock);
 558
 559         return status;
 560 }
 561
 562 static void beiscsi_get_params(struct beiscsi_hba *phba)
 563 {
 564         phba->params.ios_per_ctrl = (phba->fw_config.iscsi_icd_count
 565                                     - (phba->fw_config.iscsi_cid_count
 566                                     + BE2_TMFS
 567                                     + BE2_NOPOUT_REQ));
 568         phba->params.cxns_per_ctrl = phba->fw_config.iscsi_cid_count;
 569         phba->params.asyncpdus_per_ctrl = phba->fw_config.iscsi_cid_count * 2;
 570         phba->params.icds_per_ctrl = phba->fw_config.iscsi_icd_count;
 571         phba->params.num_sge_per_io = BE2_SGE;
 572         phba->params.defpdu_hdr_sz = BE2_DEFPDU_HDR_SZ;
 573         phba->params.defpdu_data_sz = BE2_DEFPDU_DATA_SZ;
 574         phba->params.eq_timer = 64;
 575         phba->params.num_eq_entries =
 576             (((BE2_CMDS_PER_CXN * 2 + phba->fw_config.iscsi_cid_count * 2
 577                                     + BE2_TMFS) / 512) + 1) * 512;
 578         phba->params.num_eq_entries = (phba->params.num_eq_entries < 1024)
 579                                 ? 1024 : phba->params.num_eq_entries;
 580         SE_DEBUG(DBG_LVL_8, "phba->params.num_eq_entries=%d\n",
 581                              phba->params.num_eq_entries);
 582         phba->params.num_cq_entries =
 583             (((BE2_CMDS_PER_CXN * 2 +  phba->fw_config.iscsi_cid_count * 2
 584                                     + BE2_TMFS) / 512) + 1) * 512;
 585         phba->params.wrbs_per_cxn = 256;
 586 }
 587
 588 static void hwi_ring_eq_db(struct beiscsi_hba *phba,
 589                            unsigned int id, unsigned int clr_interrupt,
 590                            unsigned int num_processed,
 591                            unsigned char rearm, unsigned char event)
 592 {
 593         u32 val = 0;
 594         val |= id & DB_EQ_RING_ID_MASK;
 595         if (rearm)
 596                 val |= 1 << DB_EQ_REARM_SHIFT;
 597         if (clr_interrupt)
 598                 val |= 1 << DB_EQ_CLR_SHIFT;
 599         if (event)
 600                 val |= 1 << DB_EQ_EVNT_SHIFT;
 601         val |= num_processed << DB_EQ_NUM_POPPED_SHIFT;
 602         iowrite32(val, phba->db_va + DB_EQ_OFFSET);
 603 }
 604
 605 /**
 606  * be_isr_mcc - The isr routine of the driver.
 607  * @irq: Not used
 608  * @dev_id: Pointer to host adapter structure
 609  */
 610 static irqreturn_t be_isr_mcc(int irq, void *dev_id)
 611 {
 612         struct beiscsi_hba *phba;
 613         struct be_eq_entry *eqe = NULL;
 614         struct be_queue_info *eq;
 615         struct be_queue_info *mcc;
 616         unsigned int num_eq_processed;
 617         struct be_eq_obj *pbe_eq;
 618         unsigned long flags;
 619
 620         pbe_eq = dev_id;
 621         eq = &pbe_eq->q;
 622         phba =  pbe_eq->phba;
 623         mcc = &phba->ctrl.mcc_obj.cq;
 624         eqe = queue_tail_node(eq);
 625         if (!eqe)
 626                 SE_DEBUG(DBG_LVL_1, "eqe is NULL\n");
 627
 628         num_eq_processed = 0;
 629
 630         while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 631                                 & EQE_VALID_MASK) {
 632                 if (((eqe->dw[offsetof(struct amap_eq_entry,
 633                      resource_id) / 32] &
 634                      EQE_RESID_MASK) >> 16) == mcc->id) {
 635                         spin_lock_irqsave(&phba->isr_lock, flags);
 636                         phba->todo_mcc_cq = 1;
 637                         spin_unlock_irqrestore(&phba->isr_lock, flags);
 638                 }
 639                 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 640                 queue_tail_inc(eq);
 641                 eqe = queue_tail_node(eq);
 642                 num_eq_processed++;
 643         }
 644         if (phba->todo_mcc_cq)
 645                 queue_work(phba->wq, &phba->work_cqs);
 646         if (num_eq_processed)
 647                 hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 1, 1);
 648
 649         return IRQ_HANDLED;
 650 }
 651
 652 /**
 653  * be_isr_msix - The isr routine of the driver.
 654  * @irq: Not used
 655  * @dev_id: Pointer to host adapter structure
 656  */
 657 static irqreturn_t be_isr_msix(int irq, void *dev_id)
 658 {
 659         struct beiscsi_hba *phba;
 660         struct be_eq_entry *eqe = NULL;
 661         struct be_queue_info *eq;
 662         struct be_queue_info *cq;
 663         unsigned int num_eq_processed;
 664         struct be_eq_obj *pbe_eq;
 665         unsigned long flags;
 666
 667         pbe_eq = dev_id;
 668         eq = &pbe_eq->q;
 669         cq = pbe_eq->cq;
 670         eqe = queue_tail_node(eq);
 671         if (!eqe)
 672                 SE_DEBUG(DBG_LVL_1, "eqe is NULL\n");
 673
 674         phba = pbe_eq->phba;
 675         num_eq_processed = 0;
 676         if (blk_iopoll_enabled) {
 677                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 678                                         & EQE_VALID_MASK) {
 679                         if (!blk_iopoll_sched_prep(&pbe_eq->iopoll))
 680                                 blk_iopoll_sched(&pbe_eq->iopoll);
 681
 682                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 683                         queue_tail_inc(eq);
 684                         eqe = queue_tail_node(eq);
 685                         num_eq_processed++;
 686                 }
 687                 if (num_eq_processed)
 688                         hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 0, 1);
 689
 690                 return IRQ_HANDLED;
 691         } else {
 692                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 693                                                 & EQE_VALID_MASK) {
 694                         spin_lock_irqsave(&phba->isr_lock, flags);
 695                         phba->todo_cq = 1;
 696                         spin_unlock_irqrestore(&phba->isr_lock, flags);
 697                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 698                         queue_tail_inc(eq);
 699                         eqe = queue_tail_node(eq);
 700                         num_eq_processed++;
 701                 }
 702                 if (phba->todo_cq)
 703                         queue_work(phba->wq, &phba->work_cqs);
 704
 705                 if (num_eq_processed)
 706                         hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 1, 1);
 707
 708                 return IRQ_HANDLED;
 709         }
 710 }
 711
 712 /**
 713  * be_isr - The isr routine of the driver.
 714  * @irq: Not used
 715  * @dev_id: Pointer to host adapter structure
 716  */
 717 static irqreturn_t be_isr(int irq, void *dev_id)
 718 {
 719         struct beiscsi_hba *phba;
 720         struct hwi_controller *phwi_ctrlr;
 721         struct hwi_context_memory *phwi_context;
 722         struct be_eq_entry *eqe = NULL;
 723         struct be_queue_info *eq;
 724         struct be_queue_info *cq;
 725         struct be_queue_info *mcc;
 726         unsigned long flags, index;
 727         unsigned int num_mcceq_processed, num_ioeq_processed;
 728         struct be_ctrl_info *ctrl;
 729         struct be_eq_obj *pbe_eq;
 730         int isr;
 731
 732         phba = dev_id;
 733         ctrl = &phba->ctrl;
 734         isr = ioread32(ctrl->csr + CEV_ISR0_OFFSET +
 735                        (PCI_FUNC(ctrl->pdev->devfn) * CEV_ISR_SIZE));
 736         if (!isr)
 737                 return IRQ_NONE;
 738
 739         phwi_ctrlr = phba->phwi_ctrlr;
 740         phwi_context = phwi_ctrlr->phwi_ctxt;
 741         pbe_eq = &phwi_context->be_eq[0];
 742
 743         eq = &phwi_context->be_eq[0].q;
 744         mcc = &phba->ctrl.mcc_obj.cq;
 745         index = 0;
 746         eqe = queue_tail_node(eq);
 747         if (!eqe)
 748                 SE_DEBUG(DBG_LVL_1, "eqe is NULL\n");
 749
 750         num_ioeq_processed = 0;
 751         num_mcceq_processed = 0;
 752         if (blk_iopoll_enabled) {
 753                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 754                                         & EQE_VALID_MASK) {
 755                         if (((eqe->dw[offsetof(struct amap_eq_entry,
 756                              resource_id) / 32] &
 757                              EQE_RESID_MASK) >> 16) == mcc->id) {
 758                                 spin_lock_irqsave(&phba->isr_lock, flags);
 759                                 phba->todo_mcc_cq = 1;
 760                                 spin_unlock_irqrestore(&phba->isr_lock, flags);
 761                                 num_mcceq_processed++;
 762                         } else {
 763                                 if (!blk_iopoll_sched_prep(&pbe_eq->iopoll))
 764                                         blk_iopoll_sched(&pbe_eq->iopoll);
 765                                 num_ioeq_processed++;
 766                         }
 767                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 768                         queue_tail_inc(eq);
 769                         eqe = queue_tail_node(eq);
 770                 }
 771                 if (num_ioeq_processed || num_mcceq_processed) {
 772                         if (phba->todo_mcc_cq)
 773                                 queue_work(phba->wq, &phba->work_cqs);
 774
 775                         if ((num_mcceq_processed) && (!num_ioeq_processed))
 776                                 hwi_ring_eq_db(phba, eq->id, 0,
 777                                               (num_ioeq_processed +
 778                                                num_mcceq_processed) , 1, 1);
 779                         else
 780                                 hwi_ring_eq_db(phba, eq->id, 0,
 781                                                (num_ioeq_processed +
 782                                                 num_mcceq_processed), 0, 1);
 783
 784                         return IRQ_HANDLED;
 785                 } else
 786                         return IRQ_NONE;
 787         } else {
 788                 cq = &phwi_context->be_cq[0];
 789                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 790                                                 & EQE_VALID_MASK) {
 791
 792                         if (((eqe->dw[offsetof(struct amap_eq_entry,
 793                              resource_id) / 32] &
 794                              EQE_RESID_MASK) >> 16) != cq->id) {
 795                                 spin_lock_irqsave(&phba->isr_lock, flags);
 796                                 phba->todo_mcc_cq = 1;
 797                                 spin_unlock_irqrestore(&phba->isr_lock, flags);
 798                         } else {
 799                                 spin_lock_irqsave(&phba->isr_lock, flags);
 800                                 phba->todo_cq = 1;
 801                                 spin_unlock_irqrestore(&phba->isr_lock, flags);
 802                         }
 803                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 804                         queue_tail_inc(eq);
 805                         eqe = queue_tail_node(eq);
 806                         num_ioeq_processed++;
 807                 }
 808                 if (phba->todo_cq || phba->todo_mcc_cq)
 809                         queue_work(phba->wq, &phba->work_cqs);
 810
 811                 if (num_ioeq_processed) {
 812                         hwi_ring_eq_db(phba, eq->id, 0,
 813                                        num_ioeq_processed, 1, 1);
 814                         return IRQ_HANDLED;
 815                 } else
 816                         return IRQ_NONE;
 817         }
 818 }
 819
 820 static int beiscsi_init_irqs(struct beiscsi_hba *phba)
 821 {
 822         struct pci_dev *pcidev = phba->pcidev;
 823         struct hwi_controller *phwi_ctrlr;
 824         struct hwi_context_memory *phwi_context;
 825         int ret, msix_vec, i, j;
 826
 827         phwi_ctrlr = phba->phwi_ctrlr;
 828         phwi_context = phwi_ctrlr->phwi_ctxt;
 829
 830         if (phba->msix_enabled) {
 831                 for (i = 0; i < phba->num_cpus; i++) {
 832                         phba->msi_name[i] = kzalloc(BEISCSI_MSI_NAME,
 833                                                     GFP_KERNEL);
 834                         if (!phba->msi_name[i]) {
 835                                 ret = -ENOMEM;
 836                                 goto free_msix_irqs;
 837                         }
 838
 839                         sprintf(phba->msi_name[i], "beiscsi_%02x_%02x",
 840                                 phba->shost->host_no, i);
 841                         msix_vec = phba->msix_entries[i].vector;
 842                         ret = request_irq(msix_vec, be_isr_msix, 0,
 843                                           phba->msi_name[i],
 844                                           &phwi_context->be_eq[i]);
 845                         if (ret) {
 846                                 shost_printk(KERN_ERR, phba->shost,
 847                                              "beiscsi_init_irqs-Failed to"
 848                                              "register msix for i = %d\n", i);
 849                                 kfree(phba->msi_name[i]);
 850                                 goto free_msix_irqs;
 851                         }
 852                 }
 853                 phba->msi_name[i] = kzalloc(BEISCSI_MSI_NAME, GFP_KERNEL);
 854                 if (!phba->msi_name[i]) {
 855                         ret = -ENOMEM;
 856                         goto free_msix_irqs;
 857                 }
 858                 sprintf(phba->msi_name[i], "beiscsi_mcc_%02x",
 859                         phba->shost->host_no);
 860                 msix_vec = phba->msix_entries[i].vector;
 861                 ret = request_irq(msix_vec, be_isr_mcc, 0, phba->msi_name[i],
 862                                   &phwi_context->be_eq[i]);
 863                 if (ret) {
 864                         shost_printk(KERN_ERR, phba->shost, "beiscsi_init_irqs-"
 865                                      "Failed to register beiscsi_msix_mcc\n");
 866                         kfree(phba->msi_name[i]);
 867                         goto free_msix_irqs;
 868                 }
 869
 870         } else {
 871                 ret = request_irq(pcidev->irq, be_isr, IRQF_SHARED,
 872                                   "beiscsi", phba);
 873                 if (ret) {
 874                         shost_printk(KERN_ERR, phba->shost, "beiscsi_init_irqs-"
 875                                      "Failed to register irq\\n");
 876                         return ret;
 877                 }
 878         }
 879         return 0;
 880 free_msix_irqs:
 881         for (j = i - 1; j >= 0; j--) {
 882                 kfree(phba->msi_name[j]);
 883                 msix_vec = phba->msix_entries[j].vector;
 884                 free_irq(msix_vec, &phwi_context->be_eq[j]);
 885         }
 886         return ret;
 887 }
 888
 889 static void hwi_ring_cq_db(struct beiscsi_hba *phba,
 890                            unsigned int id, unsigned int num_processed,
 891                            unsigned char rearm, unsigned char event)
 892 {
 893         u32 val = 0;
 894         val |= id & DB_CQ_RING_ID_MASK;
 895         if (rearm)
 896                 val |= 1 << DB_CQ_REARM_SHIFT;
 897         val |= num_processed << DB_CQ_NUM_POPPED_SHIFT;
 898         iowrite32(val, phba->db_va + DB_CQ_OFFSET);
 899 }
 900
 901 static unsigned int
 902 beiscsi_process_async_pdu(struct beiscsi_conn *beiscsi_conn,
 903                           struct beiscsi_hba *phba,
 904                           unsigned short cid,
 905                           struct pdu_base *ppdu,
 906                           unsigned long pdu_len,
 907                           void *pbuffer, unsigned long buf_len)
 908 {
 909         struct iscsi_conn *conn = beiscsi_conn->conn;
 910         struct iscsi_session *session = conn->session;
 911         struct iscsi_task *task;
 912         struct beiscsi_io_task *io_task;
 913         struct iscsi_hdr *login_hdr;
 914
 915         switch (ppdu->dw[offsetof(struct amap_pdu_base, opcode) / 32] &
 916                                                 PDUBASE_OPCODE_MASK) {
 917         case ISCSI_OP_NOOP_IN:
 918                 pbuffer = NULL;
 919                 buf_len = 0;
 920                 break;
 921         case ISCSI_OP_ASYNC_EVENT:
 922                 break;
 923         case ISCSI_OP_REJECT:
 924                 WARN_ON(!pbuffer);
 925                 WARN_ON(!(buf_len == 48));
 926                 SE_DEBUG(DBG_LVL_1, "In ISCSI_OP_REJECT\n");
 927                 break;
 928         case ISCSI_OP_LOGIN_RSP:
 929         case ISCSI_OP_TEXT_RSP:
 930                 task = conn->login_task;
 931                 io_task = task->dd_data;
 932                 login_hdr = (struct iscsi_hdr *)ppdu;
 933                 login_hdr->itt = io_task->libiscsi_itt;
 934                 break;
 935         default:
 936                 shost_printk(KERN_WARNING, phba->shost,
 937                              "Unrecognized opcode 0x%x in async msg\n",
 938                              (ppdu->
 939                              dw[offsetof(struct amap_pdu_base, opcode) / 32]
 940                                                 & PDUBASE_OPCODE_MASK));
 941                 return 1;
 942         }
 943
 944         spin_lock_bh(&session->lock);
 945         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)ppdu, pbuffer, buf_len);
 946         spin_unlock_bh(&session->lock);
 947         return 0;
 948 }
 949
 950 static struct sgl_handle *alloc_io_sgl_handle(struct beiscsi_hba *phba)
 951 {
 952         struct sgl_handle *psgl_handle;
 953
 954         if (phba->io_sgl_hndl_avbl) {
 955                 SE_DEBUG(DBG_LVL_8,
 956                          "In alloc_io_sgl_handle,io_sgl_alloc_index=%d\n",
 957                          phba->io_sgl_alloc_index);
 958                 psgl_handle = phba->io_sgl_hndl_base[phba->
 959                                                 io_sgl_alloc_index];
 960                 phba->io_sgl_hndl_base[phba->io_sgl_alloc_index] = NULL;
 961                 phba->io_sgl_hndl_avbl--;
 962                 if (phba->io_sgl_alloc_index == (phba->params.
 963                                                  ios_per_ctrl - 1))
 964                         phba->io_sgl_alloc_index = 0;
 965                 else
 966                         phba->io_sgl_alloc_index++;
 967         } else
 968                 psgl_handle = NULL;
 969         return psgl_handle;
 970 }
 971
 972 static void
 973 free_io_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
 974 {
 975         SE_DEBUG(DBG_LVL_8, "In free_,io_sgl_free_index=%d\n",
 976                  phba->io_sgl_free_index);
 977         if (phba->io_sgl_hndl_base[phba->io_sgl_free_index]) {
 978                 /*
 979                  * this can happen if clean_task is called on a task that
 980                  * failed in xmit_task or alloc_pdu.
 981                  */
 982                  SE_DEBUG(DBG_LVL_8,
 983                          "Double Free in IO SGL io_sgl_free_index=%d,"
 984                          "value there=%p\n", phba->io_sgl_free_index,
 985                          phba->io_sgl_hndl_base[phba->io_sgl_free_index]);
 986                 return;
 987         }
 988         phba->io_sgl_hndl_base[phba->io_sgl_free_index] = psgl_handle;
 989         phba->io_sgl_hndl_avbl++;
 990         if (phba->io_sgl_free_index == (phba->params.ios_per_ctrl - 1))
 991                 phba->io_sgl_free_index = 0;
 992         else
 993                 phba->io_sgl_free_index++;
 994 }
 995
 996 /**
 997  * alloc_wrb_handle - To allocate a wrb handle
 998  * @phba: The hba pointer
 999  * @cid: The cid to use for allocation
1000  *
1001  * This happens under session_lock until submission to chip
1002  */
1003 struct wrb_handle *alloc_wrb_handle(struct beiscsi_hba *phba, unsigned int cid)
1004 {
1005         struct hwi_wrb_context *pwrb_context;
1006         struct hwi_controller *phwi_ctrlr;
1007         struct wrb_handle *pwrb_handle, *pwrb_handle_tmp;
1008
1009         phwi_ctrlr = phba->phwi_ctrlr;
1010         pwrb_context = &phwi_ctrlr->wrb_context[cid];
1011         if (pwrb_context->wrb_handles_available >= 2) {
1012                 pwrb_handle = pwrb_context->pwrb_handle_base[
1013                                             pwrb_context->alloc_index];
1014                 pwrb_context->wrb_handles_available--;
1015                 if (pwrb_context->alloc_index ==
1016                                                 (phba->params.wrbs_per_cxn - 1))
1017                         pwrb_context->alloc_index = 0;
1018                 else
1019                         pwrb_context->alloc_index++;
1020                 pwrb_handle_tmp = pwrb_context->pwrb_handle_base[
1021                                                 pwrb_context->alloc_index];
1022                 pwrb_handle->nxt_wrb_index = pwrb_handle_tmp->wrb_index;
1023         } else
1024                 pwrb_handle = NULL;
1025         return pwrb_handle;
1026 }
1027
1028 /**
1029  * free_wrb_handle - To free the wrb handle back to pool
1030  * @phba: The hba pointer
1031  * @pwrb_context: The context to free from
1032  * @pwrb_handle: The wrb_handle to free
1033  *
1034  * This happens under session_lock until submission to chip
1035  */
1036 static void
1037 free_wrb_handle(struct beiscsi_hba *phba, struct hwi_wrb_context *pwrb_context,
1038                 struct wrb_handle *pwrb_handle)
1039 {
1040         pwrb_context->pwrb_handle_base[pwrb_context->free_index] = pwrb_handle;
1041         pwrb_context->wrb_handles_available++;
1042         if (pwrb_context->free_index == (phba->params.wrbs_per_cxn - 1))
1043                 pwrb_context->free_index = 0;
1044         else
1045                 pwrb_context->free_index++;
1046
1047         SE_DEBUG(DBG_LVL_8,
1048                  "FREE WRB: pwrb_handle=%p free_index=0x%x"
1049                  "wrb_handles_available=%d\n",
1050                  pwrb_handle, pwrb_context->free_index,
1051                  pwrb_context->wrb_handles_available);
1052 }
1053
1054 static struct sgl_handle *alloc_mgmt_sgl_handle(struct beiscsi_hba *phba)
1055 {
1056         struct sgl_handle *psgl_handle;
1057
1058         if (phba->eh_sgl_hndl_avbl) {
1059                 psgl_handle = phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index];
1060                 phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index] = NULL;
1061                 SE_DEBUG(DBG_LVL_8, "mgmt_sgl_alloc_index=%d=0x%x\n",
1062                          phba->eh_sgl_alloc_index, phba->eh_sgl_alloc_index);
1063                 phba->eh_sgl_hndl_avbl--;
1064                 if (phba->eh_sgl_alloc_index ==
1065                     (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl -
1066                      1))
1067                         phba->eh_sgl_alloc_index = 0;
1068                 else
1069                         phba->eh_sgl_alloc_index++;
1070         } else
1071                 psgl_handle = NULL;
1072         return psgl_handle;
1073 }
1074
1075 void
1076 free_mgmt_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
1077 {
1078
1079         SE_DEBUG(DBG_LVL_8, "In  free_mgmt_sgl_handle,eh_sgl_free_index=%d\n",
1080                              phba->eh_sgl_free_index);
1081         if (phba->eh_sgl_hndl_base[phba->eh_sgl_free_index]) {
1082                 /*
1083                  * this can happen if clean_task is called on a task that
1084                  * failed in xmit_task or alloc_pdu.
1085                  */
1086                 SE_DEBUG(DBG_LVL_8,
1087                          "Double Free in eh SGL ,eh_sgl_free_index=%d\n",
1088                          phba->eh_sgl_free_index);
1089                 return;
1090         }
1091         phba->eh_sgl_hndl_base[phba->eh_sgl_free_index] = psgl_handle;
1092         phba->eh_sgl_hndl_avbl++;
1093         if (phba->eh_sgl_free_index ==
1094             (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl - 1))
1095                 phba->eh_sgl_free_index = 0;
1096         else
1097                 phba->eh_sgl_free_index++;
1098 }
1099
1100 static void
1101 be_complete_io(struct beiscsi_conn *beiscsi_conn,
1102                struct iscsi_task *task, struct sol_cqe *psol)
1103 {
1104         struct beiscsi_io_task *io_task = task->dd_data;
1105         struct be_status_bhs *sts_bhs =
1106                                 (struct be_status_bhs *)io_task->cmd_bhs;
1107         struct iscsi_conn *conn = beiscsi_conn->conn;
1108         unsigned int sense_len;
1109         unsigned char *sense;
1110         u32 resid = 0, exp_cmdsn, max_cmdsn;
1111         u8 rsp, status, flags;
1112
1113         exp_cmdsn = (psol->
1114                         dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
1115                         & SOL_EXP_CMD_SN_MASK);
1116         max_cmdsn = ((psol->
1117                         dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
1118                         & SOL_EXP_CMD_SN_MASK) +
1119                         ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
1120                                 / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
1121         rsp = ((psol->dw[offsetof(struct amap_sol_cqe, i_resp) / 32]
1122                                                 & SOL_RESP_MASK) >> 16);
1123         status = ((psol->dw[offsetof(struct amap_sol_cqe, i_sts) / 32]
1124                                                 & SOL_STS_MASK) >> 8);
1125         flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
1126                                         & SOL_FLAGS_MASK) >> 24) | 0x80;
1127         if (!task->sc) {
1128                 if (io_task->scsi_cmnd)
1129                         scsi_dma_unmap(io_task->scsi_cmnd);
1130
1131                 return;
1132         }
1133         task->sc->result = (DID_OK << 16) | status;
1134         if (rsp != ISCSI_STATUS_CMD_COMPLETED) {
1135                 task->sc->result = DID_ERROR << 16;
1136                 goto unmap;
1137         }
1138
1139         /* bidi not initially supported */
1140         if (flags & (ISCSI_FLAG_CMD_UNDERFLOW | ISCSI_FLAG_CMD_OVERFLOW)) {
1141                 resid = (psol->dw[offsetof(struct amap_sol_cqe, i_res_cnt) /
1142                                 32] & SOL_RES_CNT_MASK);
1143
1144                 if (!status && (flags & ISCSI_FLAG_CMD_OVERFLOW))
1145                         task->sc->result = DID_ERROR << 16;
1146
1147                 if (flags & ISCSI_FLAG_CMD_UNDERFLOW) {
1148                         scsi_set_resid(task->sc, resid);
1149                         if (!status && (scsi_bufflen(task->sc) - resid <
1150                             task->sc->underflow))
1151                                 task->sc->result = DID_ERROR << 16;
1152                 }
1153         }
1154
1155         if (status == SAM_STAT_CHECK_CONDITION) {
1156                 unsigned short *slen = (unsigned short *)sts_bhs->sense_info;
1157                 sense = sts_bhs->sense_info + sizeof(unsigned short);
1158                 sense_len =  cpu_to_be16(*slen);
1159                 memcpy(task->sc->sense_buffer, sense,
1160                        min_t(u16, sense_len, SCSI_SENSE_BUFFERSIZE));
1161         }
1162
1163         if (io_task->cmd_bhs->iscsi_hdr.flags & ISCSI_FLAG_CMD_READ) {
1164                 if (psol->dw[offsetof(struct amap_sol_cqe, i_res_cnt) / 32]
1165                                                         & SOL_RES_CNT_MASK)
1166                          conn->rxdata_octets += (psol->
1167                              dw[offsetof(struct amap_sol_cqe, i_res_cnt) / 32]
1168                              & SOL_RES_CNT_MASK);
1169         }
1170 unmap:
1171         scsi_dma_unmap(io_task->scsi_cmnd);
1172         iscsi_complete_scsi_task(task, exp_cmdsn, max_cmdsn);
1173 }
1174
1175 static void
1176 be_complete_logout(struct beiscsi_conn *beiscsi_conn,
1177                    struct iscsi_task *task, struct sol_cqe *psol)
1178 {
1179         struct iscsi_logout_rsp *hdr;
1180         struct beiscsi_io_task *io_task = task->dd_data;
1181         struct iscsi_conn *conn = beiscsi_conn->conn;
1182
1183         hdr = (struct iscsi_logout_rsp *)task->hdr;
1184         hdr->opcode = ISCSI_OP_LOGOUT_RSP;
1185         hdr->t2wait = 5;
1186         hdr->t2retain = 0;
1187         hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
1188                                         & SOL_FLAGS_MASK) >> 24) | 0x80;
1189         hdr->response = (psol->dw[offsetof(struct amap_sol_cqe, i_resp) /
1190                                         32] & SOL_RESP_MASK);
1191         hdr->exp_cmdsn = cpu_to_be32(psol->
1192                         dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
1193                                         & SOL_EXP_CMD_SN_MASK);
1194         hdr->max_cmdsn = be32_to_cpu((psol->
1195                          dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
1196                                         & SOL_EXP_CMD_SN_MASK) +
1197                         ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
1198                                         / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
1199         hdr->dlength[0] = 0;
1200         hdr->dlength[1] = 0;
1201         hdr->dlength[2] = 0;
1202         hdr->hlength = 0;
1203         hdr->itt = io_task->libiscsi_itt;
1204         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
1205 }
1206
1207 static void
1208 be_complete_tmf(struct beiscsi_conn *beiscsi_conn,
1209                 struct iscsi_task *task, struct sol_cqe *psol)
1210 {
1211         struct iscsi_tm_rsp *hdr;
1212         struct iscsi_conn *conn = beiscsi_conn->conn;
1213         struct beiscsi_io_task *io_task = task->dd_data;
1214
1215         hdr = (struct iscsi_tm_rsp *)task->hdr;
1216         hdr->opcode = ISCSI_OP_SCSI_TMFUNC_RSP;
1217         hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
1218                                         & SOL_FLAGS_MASK) >> 24) | 0x80;
1219         hdr->response = (psol->dw[offsetof(struct amap_sol_cqe, i_resp) /
1220                                         32] & SOL_RESP_MASK);
1221         hdr->exp_cmdsn = cpu_to_be32(psol->dw[offsetof(struct amap_sol_cqe,
1222                                     i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK);
1223         hdr->max_cmdsn = be32_to_cpu((psol->dw[offsetof(struct amap_sol_cqe,
1224                         i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK) +
1225                         ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
1226                         / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
1227         hdr->itt = io_task->libiscsi_itt;
1228         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
1229 }
1230
1231 static void
1232 hwi_complete_drvr_msgs(struct beiscsi_conn *beiscsi_conn,
1233                        struct beiscsi_hba *phba, struct sol_cqe *psol)
1234 {
1235         struct hwi_wrb_context *pwrb_context;
1236         struct wrb_handle *pwrb_handle = NULL;
1237         struct hwi_controller *phwi_ctrlr;
1238         struct iscsi_task *task;
1239         struct beiscsi_io_task *io_task;
1240         struct iscsi_conn *conn = beiscsi_conn->conn;
1241         struct iscsi_session *session = conn->session;
1242
1243         phwi_ctrlr = phba->phwi_ctrlr;
1244         pwrb_context = &phwi_ctrlr->wrb_context[((psol->
1245                                 dw[offsetof(struct amap_sol_cqe, cid) / 32] &
1246                                 SOL_CID_MASK) >> 6) -
1247                                 phba->fw_config.iscsi_cid_start];
1248         pwrb_handle = pwrb_context->pwrb_handle_basestd[((psol->
1249                                 dw[offsetof(struct amap_sol_cqe, wrb_index) /
1250                                 32] & SOL_WRB_INDEX_MASK) >> 16)];
1251         task = pwrb_handle->pio_handle;
1252
1253         io_task = task->dd_data;
1254         spin_lock(&phba->mgmt_sgl_lock);
1255         free_mgmt_sgl_handle(phba, io_task->psgl_handle);
1256         spin_unlock(&phba->mgmt_sgl_lock);
1257         spin_lock_bh(&session->lock);
1258         free_wrb_handle(phba, pwrb_context, pwrb_handle);
1259         spin_unlock_bh(&session->lock);
1260 }
1261
1262 static void
1263 be_complete_nopin_resp(struct beiscsi_conn *beiscsi_conn,
1264                        struct iscsi_task *task, struct sol_cqe *psol)
1265 {
1266         struct iscsi_nopin *hdr;
1267         struct iscsi_conn *conn = beiscsi_conn->conn;
1268         struct beiscsi_io_task *io_task = task->dd_data;
1269
1270         hdr = (struct iscsi_nopin *)task->hdr;
1271         hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
1272                         & SOL_FLAGS_MASK) >> 24) | 0x80;
1273         hdr->exp_cmdsn = cpu_to_be32(psol->dw[offsetof(struct amap_sol_cqe,
1274                                      i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK);
1275         hdr->max_cmdsn = be32_to_cpu((psol->dw[offsetof(struct amap_sol_cqe,
1276                         i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK) +
1277                         ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
1278                         / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
1279         hdr->opcode = ISCSI_OP_NOOP_IN;
1280         hdr->itt = io_task->libiscsi_itt;
1281         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
1282 }
1283
1284 static void hwi_complete_cmd(struct beiscsi_conn *beiscsi_conn,
1285                              struct beiscsi_hba *phba, struct sol_cqe *psol)
1286 {
1287         struct hwi_wrb_context *pwrb_context;
1288         struct wrb_handle *pwrb_handle;
1289         struct iscsi_wrb *pwrb = NULL;
1290         struct hwi_controller *phwi_ctrlr;
1291         struct iscsi_task *task;
1292         unsigned int type;
1293         struct iscsi_conn *conn = beiscsi_conn->conn;
1294         struct iscsi_session *session = conn->session;
1295
1296         phwi_ctrlr = phba->phwi_ctrlr;
1297         pwrb_context = &phwi_ctrlr->wrb_context[((psol->dw[offsetof
1298                                 (struct amap_sol_cqe, cid) / 32]
1299                                 & SOL_CID_MASK) >> 6) -
1300                                 phba->fw_config.iscsi_cid_start];
1301         pwrb_handle = pwrb_context->pwrb_handle_basestd[((psol->
1302                                 dw[offsetof(struct amap_sol_cqe, wrb_index) /
1303                                 32] & SOL_WRB_INDEX_MASK) >> 16)];
1304         task = pwrb_handle->pio_handle;
1305         pwrb = pwrb_handle->pwrb;
1306         type = (pwrb->dw[offsetof(struct amap_iscsi_wrb, type) / 32] &
1307                                  WRB_TYPE_MASK) >> 28;
1308
1309         spin_lock_bh(&session->lock);
1310         switch (type) {
1311         case HWH_TYPE_IO:
1312         case HWH_TYPE_IO_RD:
1313                 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) ==
1314                      ISCSI_OP_NOOP_OUT)
1315                         be_complete_nopin_resp(beiscsi_conn, task, psol);
1316                 else
1317                         be_complete_io(beiscsi_conn, task, psol);
1318                 break;
1319
1320         case HWH_TYPE_LOGOUT:
1321                 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGOUT)
1322                         be_complete_logout(beiscsi_conn, task, psol);
1323                 else
1324                         be_complete_tmf(beiscsi_conn, task, psol);
1325
1326                 break;
1327
1328         case HWH_TYPE_LOGIN:
1329                 SE_DEBUG(DBG_LVL_1,
1330                          "\t\t No HWH_TYPE_LOGIN Expected in hwi_complete_cmd"
1331                          "- Solicited path\n");
1332                 break;
1333
1334         case HWH_TYPE_NOP:
1335                 be_complete_nopin_resp(beiscsi_conn, task, psol);
1336                 break;
1337
1338         default:
1339                 shost_printk(KERN_WARNING, phba->shost,
1340                                 "In hwi_complete_cmd, unknown type = %d"
1341                                 "wrb_index 0x%x CID 0x%x\n", type,
1342                                 ((psol->dw[offsetof(struct amap_iscsi_wrb,
1343                                 type) / 32] & SOL_WRB_INDEX_MASK) >> 16),
1344                                 ((psol->dw[offsetof(struct amap_sol_cqe,
1345                                 cid) / 32] & SOL_CID_MASK) >> 6));
1346                 break;
1347         }
1348
1349         spin_unlock_bh(&session->lock);
1350 }
1351
1352 static struct list_head *hwi_get_async_busy_list(struct hwi_async_pdu_context
1353                                           *pasync_ctx, unsigned int is_header,
1354                                           unsigned int host_write_ptr)
1355 {
1356         if (is_header)
1357                 return &pasync_ctx->async_entry[host_write_ptr].
1358                     header_busy_list;
1359         else
1360                 return &pasync_ctx->async_entry[host_write_ptr].data_busy_list;
1361 }
1362
1363 static struct async_pdu_handle *
1364 hwi_get_async_handle(struct beiscsi_hba *phba,
1365                      struct beiscsi_conn *beiscsi_conn,
1366                      struct hwi_async_pdu_context *pasync_ctx,
1367                      struct i_t_dpdu_cqe *pdpdu_cqe, unsigned int *pcq_index)
1368 {
1369         struct be_bus_address phys_addr;
1370         struct list_head *pbusy_list;
1371         struct async_pdu_handle *pasync_handle = NULL;
1372         int buffer_len = 0;
1373         unsigned char buffer_index = -1;
1374         unsigned char is_header = 0;
1375
1376         phys_addr.u.a32.address_lo =
1377             pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, db_addr_lo) / 32] -
1378             ((pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, dpl) / 32]
1379                                                 & PDUCQE_DPL_MASK) >> 16);
1380         phys_addr.u.a32.address_hi =
1381             pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, db_addr_hi) / 32];
1382
1383         phys_addr.u.a64.address =
1384                         *((unsigned long long *)(&phys_addr.u.a64.address));
1385
1386         switch (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, code) / 32]
1387                         & PDUCQE_CODE_MASK) {
1388         case UNSOL_HDR_NOTIFY:
1389                 is_header = 1;
1390
1391                 pbusy_list = hwi_get_async_busy_list(pasync_ctx, 1,
1392                         (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1393                         index) / 32] & PDUCQE_INDEX_MASK));
1394
1395                 buffer_len = (unsigned int)(phys_addr.u.a64.address -
1396                                 pasync_ctx->async_header.pa_base.u.a64.address);
1397
1398                 buffer_index = buffer_len /
1399                                 pasync_ctx->async_header.buffer_size;
1400
1401                 break;
1402         case UNSOL_DATA_NOTIFY:
1403                 pbusy_list = hwi_get_async_busy_list(pasync_ctx, 0, (pdpdu_cqe->
1404                                         dw[offsetof(struct amap_i_t_dpdu_cqe,
1405                                         index) / 32] & PDUCQE_INDEX_MASK));
1406                 buffer_len = (unsigned long)(phys_addr.u.a64.address -
1407                                         pasync_ctx->async_data.pa_base.u.
1408                                         a64.address);
1409                 buffer_index = buffer_len / pasync_ctx->async_data.buffer_size;
1410                 break;
1411         default:
1412                 pbusy_list = NULL;
1413                 shost_printk(KERN_WARNING, phba->shost,
1414                         "Unexpected code=%d\n",
1415                          pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1416                                         code) / 32] & PDUCQE_CODE_MASK);
1417                 return NULL;
1418         }
1419
1420         WARN_ON(!(buffer_index <= pasync_ctx->async_data.num_entries));
1421         WARN_ON(list_empty(pbusy_list));
1422         list_for_each_entry(pasync_handle, pbusy_list, link) {
1423                 WARN_ON(pasync_handle->consumed);
1424                 if (pasync_handle->index == buffer_index)
1425                         break;
1426         }
1427
1428         WARN_ON(!pasync_handle);
1429
1430         pasync_handle->cri = (unsigned short)beiscsi_conn->beiscsi_conn_cid -
1431                                              phba->fw_config.iscsi_cid_start;
1432         pasync_handle->is_header = is_header;
1433         pasync_handle->buffer_len = ((pdpdu_cqe->
1434                         dw[offsetof(struct amap_i_t_dpdu_cqe, dpl) / 32]
1435                         & PDUCQE_DPL_MASK) >> 16);
1436
1437         *pcq_index = (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1438                         index) / 32] & PDUCQE_INDEX_MASK);
1439         return pasync_handle;
1440 }
1441
1442 static unsigned int
1443 hwi_update_async_writables(struct hwi_async_pdu_context *pasync_ctx,
1444                            unsigned int is_header, unsigned int cq_index)
1445 {
1446         struct list_head *pbusy_list;
1447         struct async_pdu_handle *pasync_handle;
1448         unsigned int num_entries, writables = 0;
1449         unsigned int *pep_read_ptr, *pwritables;
1450
1451
1452         if (is_header) {
1453                 pep_read_ptr = &pasync_ctx->async_header.ep_read_ptr;
1454                 pwritables = &pasync_ctx->async_header.writables;
1455                 num_entries = pasync_ctx->async_header.num_entries;
1456         } else {
1457                 pep_read_ptr = &pasync_ctx->async_data.ep_read_ptr;
1458                 pwritables = &pasync_ctx->async_data.writables;
1459                 num_entries = pasync_ctx->async_data.num_entries;
1460         }
1461
1462         while ((*pep_read_ptr) != cq_index) {
1463                 (*pep_read_ptr)++;
1464                 *pep_read_ptr = (*pep_read_ptr) % num_entries;
1465
1466                 pbusy_list = hwi_get_async_busy_list(pasync_ctx, is_header,
1467                                                      *pep_read_ptr);
1468                 if (writables == 0)
1469                         WARN_ON(list_empty(pbusy_list));
1470
1471                 if (!list_empty(pbusy_list)) {
1472                         pasync_handle = list_entry(pbusy_list->next,
1473                                                    struct async_pdu_handle,
1474                                                    link);
1475                         WARN_ON(!pasync_handle);
1476                         pasync_handle->consumed = 1;
1477                 }
1478
1479                 writables++;
1480         }
1481
1482         if (!writables) {
1483                 SE_DEBUG(DBG_LVL_1,
1484                          "Duplicate notification received - index 0x%x!!\n",
1485                          cq_index);
1486                 WARN_ON(1);
1487         }
1488
1489         *pwritables = *pwritables + writables;
1490         return 0;
1491 }
1492
1493 static unsigned int hwi_free_async_msg(struct beiscsi_hba *phba,
1494                                        unsigned int cri)
1495 {
1496         struct hwi_controller *phwi_ctrlr;
1497         struct hwi_async_pdu_context *pasync_ctx;
1498         struct async_pdu_handle *pasync_handle, *tmp_handle;
1499         struct list_head *plist;
1500         unsigned int i = 0;
1501
1502         phwi_ctrlr = phba->phwi_ctrlr;
1503         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1504
1505         plist  = &pasync_ctx->async_entry[cri].wait_queue.list;
1506
1507         list_for_each_entry_safe(pasync_handle, tmp_handle, plist, link) {
1508                 list_del(&pasync_handle->link);
1509
1510                 if (i == 0) {
1511                         list_add_tail(&pasync_handle->link,
1512                                       &pasync_ctx->async_header.free_list);
1513                         pasync_ctx->async_header.free_entries++;
1514                         i++;
1515                 } else {
1516                         list_add_tail(&pasync_handle->link,
1517                                       &pasync_ctx->async_data.free_list);
1518                         pasync_ctx->async_data.free_entries++;
1519                         i++;
1520                 }
1521         }
1522
1523         INIT_LIST_HEAD(&pasync_ctx->async_entry[cri].wait_queue.list);
1524         pasync_ctx->async_entry[cri].wait_queue.hdr_received = 0;
1525         pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0;
1526         return 0;
1527 }
1528
1529 static struct phys_addr *
1530 hwi_get_ring_address(struct hwi_async_pdu_context *pasync_ctx,
1531                      unsigned int is_header, unsigned int host_write_ptr)
1532 {
1533         struct phys_addr *pasync_sge = NULL;
1534
1535         if (is_header)
1536                 pasync_sge = pasync_ctx->async_header.ring_base;
1537         else
1538                 pasync_sge = pasync_ctx->async_data.ring_base;
1539
1540         return pasync_sge + host_write_ptr;
1541 }
1542
1543 static void hwi_post_async_buffers(struct beiscsi_hba *phba,
1544                                    unsigned int is_header)
1545 {
1546         struct hwi_controller *phwi_ctrlr;
1547         struct hwi_async_pdu_context *pasync_ctx;
1548         struct async_pdu_handle *pasync_handle;
1549         struct list_head *pfree_link, *pbusy_list;
1550         struct phys_addr *pasync_sge;
1551         unsigned int ring_id, num_entries;
1552         unsigned int host_write_num;
1553         unsigned int writables;
1554         unsigned int i = 0;
1555         u32 doorbell = 0;
1556
1557         phwi_ctrlr = phba->phwi_ctrlr;
1558         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1559
1560         if (is_header) {
1561                 num_entries = pasync_ctx->async_header.num_entries;
1562                 writables = min(pasync_ctx->async_header.writables,
1563                                 pasync_ctx->async_header.free_entries);
1564                 pfree_link = pasync_ctx->async_header.free_list.next;
1565                 host_write_num = pasync_ctx->async_header.host_write_ptr;
1566                 ring_id = phwi_ctrlr->default_pdu_hdr.id;
1567         } else {
1568                 num_entries = pasync_ctx->async_data.num_entries;
1569                 writables = min(pasync_ctx->async_data.writables,
1570                                 pasync_ctx->async_data.free_entries);
1571                 pfree_link = pasync_ctx->async_data.free_list.next;
1572                 host_write_num = pasync_ctx->async_data.host_write_ptr;
1573                 ring_id = phwi_ctrlr->default_pdu_data.id;
1574         }
1575
1576         writables = (writables / 8) * 8;
1577         if (writables) {
1578                 for (i = 0; i < writables; i++) {
1579                         pbusy_list =
1580                             hwi_get_async_busy_list(pasync_ctx, is_header,
1581                                                     host_write_num);
1582                         pasync_handle =
1583                             list_entry(pfree_link, struct async_pdu_handle,
1584                                                                 link);
1585                         WARN_ON(!pasync_handle);
1586                         pasync_handle->consumed = 0;
1587
1588                         pfree_link = pfree_link->next;
1589
1590                         pasync_sge = hwi_get_ring_address(pasync_ctx,
1591                                                 is_header, host_write_num);
1592
1593                         pasync_sge->hi = pasync_handle->pa.u.a32.address_lo;
1594                         pasync_sge->lo = pasync_handle->pa.u.a32.address_hi;
1595
1596                         list_move(&pasync_handle->link, pbusy_list);
1597
1598                         host_write_num++;
1599                         host_write_num = host_write_num % num_entries;
1600                 }
1601
1602                 if (is_header) {
1603                         pasync_ctx->async_header.host_write_ptr =
1604                                                         host_write_num;
1605                         pasync_ctx->async_header.free_entries -= writables;
1606                         pasync_ctx->async_header.writables -= writables;
1607                         pasync_ctx->async_header.busy_entries += writables;
1608                 } else {
1609                         pasync_ctx->async_data.host_write_ptr = host_write_num;
1610                         pasync_ctx->async_data.free_entries -= writables;
1611                         pasync_ctx->async_data.writables -= writables;
1612                         pasync_ctx->async_data.busy_entries += writables;
1613                 }
1614
1615                 doorbell |= ring_id & DB_DEF_PDU_RING_ID_MASK;
1616                 doorbell |= 1 << DB_DEF_PDU_REARM_SHIFT;
1617                 doorbell |= 0 << DB_DEF_PDU_EVENT_SHIFT;
1618                 doorbell |= (writables & DB_DEF_PDU_CQPROC_MASK)
1619                                         << DB_DEF_PDU_CQPROC_SHIFT;
1620
1621                 iowrite32(doorbell, phba->db_va + DB_RXULP0_OFFSET);
1622         }
1623 }
1624
1625 static void hwi_flush_default_pdu_buffer(struct beiscsi_hba *phba,
1626                                          struct beiscsi_conn *beiscsi_conn,
1627                                          struct i_t_dpdu_cqe *pdpdu_cqe)
1628 {
1629         struct hwi_controller *phwi_ctrlr;
1630         struct hwi_async_pdu_context *pasync_ctx;
1631         struct async_pdu_handle *pasync_handle = NULL;
1632         unsigned int cq_index = -1;
1633
1634         phwi_ctrlr = phba->phwi_ctrlr;
1635         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1636
1637         pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx,
1638                                              pdpdu_cqe, &cq_index);
1639         BUG_ON(pasync_handle->is_header != 0);
1640         if (pasync_handle->consumed == 0)
1641                 hwi_update_async_writables(pasync_ctx, pasync_handle->is_header,
1642                                            cq_index);
1643
1644         hwi_free_async_msg(phba, pasync_handle->cri);
1645         hwi_post_async_buffers(phba, pasync_handle->is_header);
1646 }
1647
1648 static unsigned int
1649 hwi_fwd_async_msg(struct beiscsi_conn *beiscsi_conn,
1650                   struct beiscsi_hba *phba,
1651                   struct hwi_async_pdu_context *pasync_ctx, unsigned short cri)
1652 {
1653         struct list_head *plist;
1654         struct async_pdu_handle *pasync_handle;
1655         void *phdr = NULL;
1656         unsigned int hdr_len = 0, buf_len = 0;
1657         unsigned int status, index = 0, offset = 0;
1658         void *pfirst_buffer = NULL;
1659         unsigned int num_buf = 0;
1660
1661         plist = &pasync_ctx->async_entry[cri].wait_queue.list;
1662
1663         list_for_each_entry(pasync_handle, plist, link) {
1664                 if (index == 0) {
1665                         phdr = pasync_handle->pbuffer;
1666                         hdr_len = pasync_handle->buffer_len;
1667                 } else {
1668                         buf_len = pasync_handle->buffer_len;
1669                         if (!num_buf) {
1670                                 pfirst_buffer = pasync_handle->pbuffer;
1671                                 num_buf++;
1672                         }
1673                         memcpy(pfirst_buffer + offset,
1674                                pasync_handle->pbuffer, buf_len);
1675                         offset = buf_len;
1676                 }
1677                 index++;
1678         }
1679
1680         status = beiscsi_process_async_pdu(beiscsi_conn, phba,
1681                                            (beiscsi_conn->beiscsi_conn_cid -
1682                                             phba->fw_config.iscsi_cid_start),
1683                                             phdr, hdr_len, pfirst_buffer,
1684                                             buf_len);
1685
1686         if (status == 0)
1687                 hwi_free_async_msg(phba, cri);
1688         return 0;
1689 }
1690
1691 static unsigned int
1692 hwi_gather_async_pdu(struct beiscsi_conn *beiscsi_conn,
1693                      struct beiscsi_hba *phba,
1694                      struct async_pdu_handle *pasync_handle)
1695 {
1696         struct hwi_async_pdu_context *pasync_ctx;
1697         struct hwi_controller *phwi_ctrlr;
1698         unsigned int bytes_needed = 0, status = 0;
1699         unsigned short cri = pasync_handle->cri;
1700         struct pdu_base *ppdu;
1701
1702         phwi_ctrlr = phba->phwi_ctrlr;
1703         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1704
1705         list_del(&pasync_handle->link);
1706         if (pasync_handle->is_header) {
1707                 pasync_ctx->async_header.busy_entries--;
1708                 if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) {
1709                         hwi_free_async_msg(phba, cri);
1710                         BUG();
1711                 }
1712
1713                 pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0;
1714                 pasync_ctx->async_entry[cri].wait_queue.hdr_received = 1;
1715                 pasync_ctx->async_entry[cri].wait_queue.hdr_len =
1716                                 (unsigned short)pasync_handle->buffer_len;
1717                 list_add_tail(&pasync_handle->link,
1718                               &pasync_ctx->async_entry[cri].wait_queue.list);
1719
1720                 ppdu = pasync_handle->pbuffer;
1721                 bytes_needed = ((((ppdu->dw[offsetof(struct amap_pdu_base,
1722                         data_len_hi) / 32] & PDUBASE_DATALENHI_MASK) << 8) &
1723                         0xFFFF0000) | ((be16_to_cpu((ppdu->
1724                         dw[offsetof(struct amap_pdu_base, data_len_lo) / 32]
1725                         & PDUBASE_DATALENLO_MASK) >> 16)) & 0x0000FFFF));
1726
1727                 if (status == 0) {
1728                         pasync_ctx->async_entry[cri].wait_queue.bytes_needed =
1729                             bytes_needed;
1730
1731                         if (bytes_needed == 0)
1732                                 status = hwi_fwd_async_msg(beiscsi_conn, phba,
1733                                                            pasync_ctx, cri);
1734                 }
1735         } else {
1736                 pasync_ctx->async_data.busy_entries--;
1737                 if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) {
1738                         list_add_tail(&pasync_handle->link,
1739                                       &pasync_ctx->async_entry[cri].wait_queue.
1740                                       list);
1741                         pasync_ctx->async_entry[cri].wait_queue.
1742                                 bytes_received +=
1743                                 (unsigned short)pasync_handle->buffer_len;
1744
1745                         if (pasync_ctx->async_entry[cri].wait_queue.
1746                             bytes_received >=
1747                             pasync_ctx->async_entry[cri].wait_queue.
1748                             bytes_needed)
1749                                 status = hwi_fwd_async_msg(beiscsi_conn, phba,
1750                                                            pasync_ctx, cri);
1751                 }
1752         }
1753         return status;
1754 }
1755
1756 static void hwi_process_default_pdu_ring(struct beiscsi_conn *beiscsi_conn,
1757                                          struct beiscsi_hba *phba,
1758                                          struct i_t_dpdu_cqe *pdpdu_cqe)
1759 {
1760         struct hwi_controller *phwi_ctrlr;
1761         struct hwi_async_pdu_context *pasync_ctx;
1762         struct async_pdu_handle *pasync_handle = NULL;
1763         unsigned int cq_index = -1;
1764
1765         phwi_ctrlr = phba->phwi_ctrlr;
1766         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1767         pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx,
1768                                              pdpdu_cqe, &cq_index);
1769
1770         if (pasync_handle->consumed == 0)
1771                 hwi_update_async_writables(pasync_ctx, pasync_handle->is_header,
1772                                            cq_index);
1773         hwi_gather_async_pdu(beiscsi_conn, phba, pasync_handle);
1774         hwi_post_async_buffers(phba, pasync_handle->is_header);
1775 }
1776
1777 static void  beiscsi_process_mcc_isr(struct beiscsi_hba *phba)
1778 {
1779         struct be_queue_info *mcc_cq;
1780         struct  be_mcc_compl *mcc_compl;
1781         unsigned int num_processed = 0;
1782
1783         mcc_cq = &phba->ctrl.mcc_obj.cq;
1784         mcc_compl = queue_tail_node(mcc_cq);
1785         mcc_compl->flags = le32_to_cpu(mcc_compl->flags);
1786         while (mcc_compl->flags & CQE_FLAGS_VALID_MASK) {
1787
1788                 if (num_processed >= 32) {
1789                         hwi_ring_cq_db(phba, mcc_cq->id,
1790                                         num_processed, 0, 0);
1791                         num_processed = 0;
1792                 }
1793                 if (mcc_compl->flags & CQE_FLAGS_ASYNC_MASK) {
1794                         /* Interpret flags as an async trailer */
1795                         if (is_link_state_evt(mcc_compl->flags))
1796                                 /* Interpret compl as a async link evt */
1797                                 beiscsi_async_link_state_process(phba,
1798                                 (struct be_async_event_link_state *) mcc_compl);
1799                         else
1800                                 SE_DEBUG(DBG_LVL_1,
1801                                         " Unsupported Async Event, flags"
1802                                         " = 0x%08x\n", mcc_compl->flags);
1803                 } else if (mcc_compl->flags & CQE_FLAGS_COMPLETED_MASK) {
1804                         be_mcc_compl_process_isr(&phba->ctrl, mcc_compl);
1805                         atomic_dec(&phba->ctrl.mcc_obj.q.used);
1806                 }
1807
1808                 mcc_compl->flags = 0;
1809                 queue_tail_inc(mcc_cq);
1810                 mcc_compl = queue_tail_node(mcc_cq);
1811                 mcc_compl->flags = le32_to_cpu(mcc_compl->flags);
1812                 num_processed++;
1813         }
1814
1815         if (num_processed > 0)
1816                 hwi_ring_cq_db(phba, mcc_cq->id, num_processed, 1, 0);
1817
1818 }
1819
1820 static unsigned int beiscsi_process_cq(struct be_eq_obj *pbe_eq)
1821 {
1822         struct be_queue_info *cq;
1823         struct sol_cqe *sol;
1824         struct dmsg_cqe *dmsg;
1825         unsigned int num_processed = 0;
1826         unsigned int tot_nump = 0;
1827         struct beiscsi_conn *beiscsi_conn;
1828         struct beiscsi_endpoint *beiscsi_ep;
1829         struct iscsi_endpoint *ep;
1830         struct beiscsi_hba *phba;
1831
1832         cq = pbe_eq->cq;
1833         sol = queue_tail_node(cq);
1834         phba = pbe_eq->phba;
1835
1836         while (sol->dw[offsetof(struct amap_sol_cqe, valid) / 32] &
1837                CQE_VALID_MASK) {
1838                 be_dws_le_to_cpu(sol, sizeof(struct sol_cqe));
1839
1840                 ep = phba->ep_array[(u32) ((sol->
1841                                    dw[offsetof(struct amap_sol_cqe, cid) / 32] &
1842                                    SOL_CID_MASK) >> 6) -
1843                                    phba->fw_config.iscsi_cid_start];
1844
1845                 beiscsi_ep = ep->dd_data;
1846                 beiscsi_conn = beiscsi_ep->conn;
1847
1848                 if (num_processed >= 32) {
1849                         hwi_ring_cq_db(phba, cq->id,
1850                                         num_processed, 0, 0);
1851                         tot_nump += num_processed;
1852                         num_processed = 0;
1853                 }
1854
1855                 switch ((u32) sol->dw[offsetof(struct amap_sol_cqe, code) /
1856                         32] & CQE_CODE_MASK) {
1857                 case SOL_CMD_COMPLETE:
1858                         hwi_complete_cmd(beiscsi_conn, phba, sol);
1859                         break;
1860                 case DRIVERMSG_NOTIFY:
1861                         SE_DEBUG(DBG_LVL_8, "Received DRIVERMSG_NOTIFY\n");
1862                         dmsg = (struct dmsg_cqe *)sol;
1863                         hwi_complete_drvr_msgs(beiscsi_conn, phba, sol);
1864                         break;
1865                 case UNSOL_HDR_NOTIFY:
1866                         SE_DEBUG(DBG_LVL_8, "Received UNSOL_HDR_ NOTIFY\n");
1867                         hwi_process_default_pdu_ring(beiscsi_conn, phba,
1868                                              (struct i_t_dpdu_cqe *)sol);
1869                         break;
1870                 case UNSOL_DATA_NOTIFY:
1871                         SE_DEBUG(DBG_LVL_8, "Received UNSOL_DATA_NOTIFY\n");
1872                         hwi_process_default_pdu_ring(beiscsi_conn, phba,
1873                                              (struct i_t_dpdu_cqe *)sol);
1874                         break;
1875                 case CXN_INVALIDATE_INDEX_NOTIFY:
1876                 case CMD_INVALIDATED_NOTIFY:
1877                 case CXN_INVALIDATE_NOTIFY:
1878                         SE_DEBUG(DBG_LVL_1,
1879                                  "Ignoring CQ Error notification for cmd/cxn"
1880                                  "invalidate\n");
1881                         break;
1882                 case SOL_CMD_KILLED_DATA_DIGEST_ERR:
1883                 case CMD_KILLED_INVALID_STATSN_RCVD:
1884                 case CMD_KILLED_INVALID_R2T_RCVD:
1885                 case CMD_CXN_KILLED_LUN_INVALID:
1886                 case CMD_CXN_KILLED_ICD_INVALID:
1887                 case CMD_CXN_KILLED_ITT_INVALID:
1888                 case CMD_CXN_KILLED_SEQ_OUTOFORDER:
1889                 case CMD_CXN_KILLED_INVALID_DATASN_RCVD:
1890                         SE_DEBUG(DBG_LVL_1,
1891                                  "CQ Error notification for cmd.. "
1892                                  "code %d cid 0x%x\n",
1893                                  sol->dw[offsetof(struct amap_sol_cqe, code) /
1894                                  32] & CQE_CODE_MASK,
1895                                  (sol->dw[offsetof(struct amap_sol_cqe, cid) /
1896                                  32] & SOL_CID_MASK));
1897                         break;
1898                 case UNSOL_DATA_DIGEST_ERROR_NOTIFY:
1899                         SE_DEBUG(DBG_LVL_1,
1900                                  "Digest error on def pdu ring, dropping..\n");
1901                         hwi_flush_default_pdu_buffer(phba, beiscsi_conn,
1902                                              (struct i_t_dpdu_cqe *) sol);
1903                         break;
1904                 case CXN_KILLED_PDU_SIZE_EXCEEDS_DSL:
1905                 case CXN_KILLED_BURST_LEN_MISMATCH:
1906                 case CXN_KILLED_AHS_RCVD:
1907                 case CXN_KILLED_HDR_DIGEST_ERR:
1908                 case CXN_KILLED_UNKNOWN_HDR:
1909                 case CXN_KILLED_STALE_ITT_TTT_RCVD:
1910                 case CXN_KILLED_INVALID_ITT_TTT_RCVD:
1911                 case CXN_KILLED_TIMED_OUT:
1912                 case CXN_KILLED_FIN_RCVD:
1913                 case CXN_KILLED_BAD_UNSOL_PDU_RCVD:
1914                 case CXN_KILLED_BAD_WRB_INDEX_ERROR:
1915                 case CXN_KILLED_OVER_RUN_RESIDUAL:
1916                 case CXN_KILLED_UNDER_RUN_RESIDUAL:
1917                 case CXN_KILLED_CMND_DATA_NOT_ON_SAME_CONN:
1918                         SE_DEBUG(DBG_LVL_1, "CQ Error %d, reset CID "
1919                                  "0x%x...\n",
1920                                  sol->dw[offsetof(struct amap_sol_cqe, code) /
1921                                  32] & CQE_CODE_MASK,
1922                                  (sol->dw[offsetof(struct amap_sol_cqe, cid) /
1923                                  32] & CQE_CID_MASK));
1924                         iscsi_conn_failure(beiscsi_conn->conn,
1925                                            ISCSI_ERR_CONN_FAILED);
1926                         break;
1927                 case CXN_KILLED_RST_SENT:
1928                 case CXN_KILLED_RST_RCVD:
1929                         SE_DEBUG(DBG_LVL_1, "CQ Error %d, reset"
1930                                 "received/sent on CID 0x%x...\n",
1931                                  sol->dw[offsetof(struct amap_sol_cqe, code) /
1932                                  32] & CQE_CODE_MASK,
1933                                  (sol->dw[offsetof(struct amap_sol_cqe, cid) /
1934                                  32] & CQE_CID_MASK));
1935                         iscsi_conn_failure(beiscsi_conn->conn,
1936                                            ISCSI_ERR_CONN_FAILED);
1937                         break;
1938                 default:
1939                         SE_DEBUG(DBG_LVL_1, "CQ Error Invalid code= %d "
1940                                  "received on CID 0x%x...\n",
1941                                  sol->dw[offsetof(struct amap_sol_cqe, code) /
1942                                  32] & CQE_CODE_MASK,
1943                                  (sol->dw[offsetof(struct amap_sol_cqe, cid) /
1944                                  32] & CQE_CID_MASK));
1945                         break;
1946                 }
1947
1948                 AMAP_SET_BITS(struct amap_sol_cqe, valid, sol, 0);
1949                 queue_tail_inc(cq);
1950                 sol = queue_tail_node(cq);
1951                 num_processed++;
1952         }
1953
1954         if (num_processed > 0) {
1955                 tot_nump += num_processed;
1956                 hwi_ring_cq_db(phba, cq->id, num_processed, 1, 0);
1957         }
1958         return tot_nump;
1959 }
1960
1961 void beiscsi_process_all_cqs(struct work_struct *work)
1962 {
1963         unsigned long flags;
1964         struct hwi_controller *phwi_ctrlr;
1965         struct hwi_context_memory *phwi_context;
1966         struct be_eq_obj *pbe_eq;
1967         struct beiscsi_hba *phba =
1968             container_of(work, struct beiscsi_hba, work_cqs);
1969
1970         phwi_ctrlr = phba->phwi_ctrlr;
1971         phwi_context = phwi_ctrlr->phwi_ctxt;
1972         if (phba->msix_enabled)
1973                 pbe_eq = &phwi_context->be_eq[phba->num_cpus];
1974         else
1975                 pbe_eq = &phwi_context->be_eq[0];
1976
1977         if (phba->todo_mcc_cq) {
1978                 spin_lock_irqsave(&phba->isr_lock, flags);
1979                 phba->todo_mcc_cq = 0;
1980                 spin_unlock_irqrestore(&phba->isr_lock, flags);
1981                 beiscsi_process_mcc_isr(phba);
1982         }
1983
1984         if (phba->todo_cq) {
1985                 spin_lock_irqsave(&phba->isr_lock, flags);
1986                 phba->todo_cq = 0;
1987                 spin_unlock_irqrestore(&phba->isr_lock, flags);
1988                 beiscsi_process_cq(pbe_eq);
1989         }
1990 }
1991
1992 static int be_iopoll(struct blk_iopoll *iop, int budget)
1993 {
1994         static unsigned int ret;
1995         struct beiscsi_hba *phba;
1996         struct be_eq_obj *pbe_eq;
1997
1998         pbe_eq = container_of(iop, struct be_eq_obj, iopoll);
1999         ret = beiscsi_process_cq(pbe_eq);
2000         if (ret < budget) {
2001                 phba = pbe_eq->phba;
2002                 blk_iopoll_complete(iop);
2003                 SE_DEBUG(DBG_LVL_8, "rearm pbe_eq->q.id =%d\n", pbe_eq->q.id);
2004                 hwi_ring_eq_db(phba, pbe_eq->q.id, 0, 0, 1, 1);
2005         }
2006         return ret;
2007 }
2008
2009 static void
2010 hwi_write_sgl(struct iscsi_wrb *pwrb, struct scatterlist *sg,
2011               unsigned int num_sg, struct beiscsi_io_task *io_task)
2012 {
2013         struct iscsi_sge *psgl;
2014         unsigned int sg_len, index;
2015         unsigned int sge_len = 0;
2016         unsigned long long addr;
2017         struct scatterlist *l_sg;
2018         unsigned int offset;
2019
2020         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb,
2021                                       io_task->bhs_pa.u.a32.address_lo);
2022         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb,
2023                                       io_task->bhs_pa.u.a32.address_hi);
2024
2025         l_sg = sg;
2026         for (index = 0; (index < num_sg) && (index < 2); index++,
2027                                                          sg = sg_next(sg)) {
2028                 if (index == 0) {
2029                         sg_len = sg_dma_len(sg);
2030                         addr = (u64) sg_dma_address(sg);
2031                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb,
2032                                                 ((u32)(addr & 0xFFFFFFFF)));
2033                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb,
2034                                                         ((u32)(addr >> 32)));
2035                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb,
2036                                                         sg_len);
2037                         sge_len = sg_len;
2038                 } else {
2039                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_r2t_offset,
2040                                                         pwrb, sge_len);
2041                         sg_len = sg_dma_len(sg);
2042                         addr = (u64) sg_dma_address(sg);
2043                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_lo, pwrb,
2044                                                 ((u32)(addr & 0xFFFFFFFF)));
2045                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_hi, pwrb,
2046                                                         ((u32)(addr >> 32)));
2047                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_len, pwrb,
2048                                                         sg_len);
2049                 }
2050         }
2051         psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
2052         memset(psgl, 0, sizeof(*psgl) * BE2_SGE);
2053
2054         AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len - 2);
2055
2056         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2057                         io_task->bhs_pa.u.a32.address_hi);
2058         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2059                         io_task->bhs_pa.u.a32.address_lo);
2060
2061         if (num_sg == 1) {
2062                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
2063                                                                 1);
2064                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
2065                                                                 0);
2066         } else if (num_sg == 2) {
2067                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
2068                                                                 0);
2069                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
2070                                                                 1);
2071         } else {
2072                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
2073                                                                 0);
2074                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
2075                                                                 0);
2076         }
2077         sg = l_sg;
2078         psgl++;
2079         psgl++;
2080         offset = 0;
2081         for (index = 0; index < num_sg; index++, sg = sg_next(sg), psgl++) {
2082                 sg_len = sg_dma_len(sg);
2083                 addr = (u64) sg_dma_address(sg);
2084                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2085                                                 (addr & 0xFFFFFFFF));
2086                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2087                                                 (addr >> 32));
2088                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, sg_len);
2089                 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, offset);
2090                 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
2091                 offset += sg_len;
2092         }
2093         psgl--;
2094         AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
2095 }
2096
2097 static void hwi_write_buffer(struct iscsi_wrb *pwrb, struct iscsi_task *task)
2098 {
2099         struct iscsi_sge *psgl;
2100         unsigned long long addr;
2101         struct beiscsi_io_task *io_task = task->dd_data;
2102         struct beiscsi_conn *beiscsi_conn = io_task->conn;
2103         struct beiscsi_hba *phba = beiscsi_conn->phba;
2104
2105         io_task->bhs_len = sizeof(struct be_nonio_bhs) - 2;
2106         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb,
2107                                 io_task->bhs_pa.u.a32.address_lo);
2108         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb,
2109                                 io_task->bhs_pa.u.a32.address_hi);
2110
2111         if (task->data) {
2112                 if (task->data_count) {
2113                         AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1);
2114                         addr = (u64) pci_map_single(phba->pcidev,
2115                                                     task->data,
2116                                                     task->data_count, 1);
2117                 } else {
2118                         AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
2119                         addr = 0;
2120                 }
2121                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb,
2122                                                 ((u32)(addr & 0xFFFFFFFF)));
2123                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb,
2124                                                 ((u32)(addr >> 32)));
2125                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb,
2126                                                 task->data_count);
2127
2128                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb, 1);
2129         } else {
2130                 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
2131                 addr = 0;
2132         }
2133
2134         psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
2135
2136         AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len);
2137
2138         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2139                       io_task->bhs_pa.u.a32.address_hi);
2140         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2141                       io_task->bhs_pa.u.a32.address_lo);
2142         if (task->data) {
2143                 psgl++;
2144                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl, 0);
2145                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl, 0);
2146                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0);
2147                 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, 0);
2148                 AMAP_SET_BITS(struct amap_iscsi_sge, rsvd0, psgl, 0);
2149                 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
2150
2151                 psgl++;
2152                 if (task->data) {
2153                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2154                                                 ((u32)(addr & 0xFFFFFFFF)));
2155                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2156                                                 ((u32)(addr >> 32)));
2157                 }
2158                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0x106);
2159         }
2160         AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
2161 }
2162
2163 static void beiscsi_find_mem_req(struct beiscsi_hba *phba)
2164 {
2165         unsigned int num_cq_pages, num_async_pdu_buf_pages;
2166         unsigned int num_async_pdu_data_pages, wrb_sz_per_cxn;
2167         unsigned int num_async_pdu_buf_sgl_pages, num_async_pdu_data_sgl_pages;
2168
2169         num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \
2170                                       sizeof(struct sol_cqe));
2171         num_async_pdu_buf_pages =
2172                         PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
2173                                        phba->params.defpdu_hdr_sz);
2174         num_async_pdu_buf_sgl_pages =
2175                         PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
2176                                        sizeof(struct phys_addr));
2177         num_async_pdu_data_pages =
2178                         PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
2179                                        phba->params.defpdu_data_sz);
2180         num_async_pdu_data_sgl_pages =
2181                         PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
2182                                        sizeof(struct phys_addr));
2183
2184         phba->params.hwi_ws_sz = sizeof(struct hwi_controller);
2185
2186         phba->mem_req[ISCSI_MEM_GLOBAL_HEADER] = 2 *
2187                                                  BE_ISCSI_PDU_HEADER_SIZE;
2188         phba->mem_req[HWI_MEM_ADDN_CONTEXT] =
2189                                             sizeof(struct hwi_context_memory);
2190
2191
2192         phba->mem_req[HWI_MEM_WRB] = sizeof(struct iscsi_wrb)
2193             * (phba->params.wrbs_per_cxn)
2194             * phba->params.cxns_per_ctrl;
2195         wrb_sz_per_cxn =  sizeof(struct wrb_handle) *
2196                                  (phba->params.wrbs_per_cxn);
2197         phba->mem_req[HWI_MEM_WRBH] = roundup_pow_of_two((wrb_sz_per_cxn) *
2198                                 phba->params.cxns_per_ctrl);
2199
2200         phba->mem_req[HWI_MEM_SGLH] = sizeof(struct sgl_handle) *
2201                 phba->params.icds_per_ctrl;
2202         phba->mem_req[HWI_MEM_SGE] = sizeof(struct iscsi_sge) *
2203                 phba->params.num_sge_per_io * phba->params.icds_per_ctrl;
2204
2205         phba->mem_req[HWI_MEM_ASYNC_HEADER_BUF] =
2206                 num_async_pdu_buf_pages * PAGE_SIZE;
2207         phba->mem_req[HWI_MEM_ASYNC_DATA_BUF] =
2208                 num_async_pdu_data_pages * PAGE_SIZE;
2209         phba->mem_req[HWI_MEM_ASYNC_HEADER_RING] =
2210                 num_async_pdu_buf_sgl_pages * PAGE_SIZE;
2211         phba->mem_req[HWI_MEM_ASYNC_DATA_RING] =
2212                 num_async_pdu_data_sgl_pages * PAGE_SIZE;
2213         phba->mem_req[HWI_MEM_ASYNC_HEADER_HANDLE] =
2214                 phba->params.asyncpdus_per_ctrl *
2215                 sizeof(struct async_pdu_handle);
2216         phba->mem_req[HWI_MEM_ASYNC_DATA_HANDLE] =
2217                 phba->params.asyncpdus_per_ctrl *
2218                 sizeof(struct async_pdu_handle);
2219         phba->mem_req[HWI_MEM_ASYNC_PDU_CONTEXT] =
2220                 sizeof(struct hwi_async_pdu_context) +
2221                 (phba->params.cxns_per_ctrl * sizeof(struct hwi_async_entry));
2222 }
2223
2224 static int beiscsi_alloc_mem(struct beiscsi_hba *phba)
2225 {
2226         struct be_mem_descriptor *mem_descr;
2227         dma_addr_t bus_add;
2228         struct mem_array *mem_arr, *mem_arr_orig;
2229         unsigned int i, j, alloc_size, curr_alloc_size;
2230
2231         phba->phwi_ctrlr = kmalloc(phba->params.hwi_ws_sz, GFP_KERNEL);
2232         if (!phba->phwi_ctrlr)
2233                 return -ENOMEM;
2234
2235         phba->init_mem = kcalloc(SE_MEM_MAX, sizeof(*mem_descr),
2236                                  GFP_KERNEL);
2237         if (!phba->init_mem) {
2238                 kfree(phba->phwi_ctrlr);
2239                 return -ENOMEM;
2240         }
2241
2242         mem_arr_orig = kmalloc(sizeof(*mem_arr_orig) * BEISCSI_MAX_FRAGS_INIT,
2243                                GFP_KERNEL);
2244         if (!mem_arr_orig) {
2245                 kfree(phba->init_mem);
2246                 kfree(phba->phwi_ctrlr);
2247                 return -ENOMEM;
2248         }
2249
2250         mem_descr = phba->init_mem;
2251         for (i = 0; i < SE_MEM_MAX; i++) {
2252                 j = 0;
2253                 mem_arr = mem_arr_orig;
2254                 alloc_size = phba->mem_req[i];
2255                 memset(mem_arr, 0, sizeof(struct mem_array) *
2256                        BEISCSI_MAX_FRAGS_INIT);
2257                 curr_alloc_size = min(be_max_phys_size * 1024, alloc_size);
2258                 do {
2259                         mem_arr->virtual_address = pci_alloc_consistent(
2260                                                         phba->pcidev,
2261                                                         curr_alloc_size,
2262                                                         &bus_add);
2263                         if (!mem_arr->virtual_address) {
2264                                 if (curr_alloc_size <= BE_MIN_MEM_SIZE)
2265                                         goto free_mem;
2266                                 if (curr_alloc_size -
2267                                         rounddown_pow_of_two(curr_alloc_size))
2268                                         curr_alloc_size = rounddown_pow_of_two
2269                                                              (curr_alloc_size);
2270                                 else
2271                                         curr_alloc_size = curr_alloc_size / 2;
2272                         } else {
2273                                 mem_arr->bus_address.u.
2274                                     a64.address = (__u64) bus_add;
2275                                 mem_arr->size = curr_alloc_size;
2276                                 alloc_size -= curr_alloc_size;
2277                                 curr_alloc_size = min(be_max_phys_size *
2278                                                       1024, alloc_size);
2279                                 j++;
2280                                 mem_arr++;
2281                         }
2282                 } while (alloc_size);
2283                 mem_descr->num_elements = j;
2284                 mem_descr->size_in_bytes = phba->mem_req[i];
2285                 mem_descr->mem_array = kmalloc(sizeof(*mem_arr) * j,
2286                                                GFP_KERNEL);
2287                 if (!mem_descr->mem_array)
2288                         goto free_mem;
2289
2290                 memcpy(mem_descr->mem_array, mem_arr_orig,
2291                        sizeof(struct mem_array) * j);
2292                 mem_descr++;
2293         }
2294         kfree(mem_arr_orig);
2295         return 0;
2296 free_mem:
2297         mem_descr->num_elements = j;
2298         while ((i) || (j)) {
2299                 for (j = mem_descr->num_elements; j > 0; j--) {
2300                         pci_free_consistent(phba->pcidev,
2301                                             mem_descr->mem_array[j - 1].size,
2302                                             mem_descr->mem_array[j - 1].
2303                                             virtual_address,
2304                                             (unsigned long)mem_descr->
2305                                             mem_array[j - 1].
2306                                             bus_address.u.a64.address);
2307                 }
2308                 if (i) {
2309                         i--;
2310                         kfree(mem_descr->mem_array);
2311                         mem_descr--;
2312                 }
2313         }
2314         kfree(mem_arr_orig);
2315         kfree(phba->init_mem);
2316         kfree(phba->phwi_ctrlr);
2317         return -ENOMEM;
2318 }
2319
2320 static int beiscsi_get_memory(struct beiscsi_hba *phba)
2321 {
2322         beiscsi_find_mem_req(phba);
2323         return beiscsi_alloc_mem(phba);
2324 }
2325
2326 static void iscsi_init_global_templates(struct beiscsi_hba *phba)
2327 {
2328         struct pdu_data_out *pdata_out;
2329         struct pdu_nop_out *pnop_out;
2330         struct be_mem_descriptor *mem_descr;
2331
2332         mem_descr = phba->init_mem;
2333         mem_descr += ISCSI_MEM_GLOBAL_HEADER;
2334         pdata_out =
2335             (struct pdu_data_out *)mem_descr->mem_array[0].virtual_address;
2336         memset(pdata_out, 0, BE_ISCSI_PDU_HEADER_SIZE);
2337
2338         AMAP_SET_BITS(struct amap_pdu_data_out, opcode, pdata_out,
2339                       IIOC_SCSI_DATA);
2340
2341         pnop_out =
2342             (struct pdu_nop_out *)((unsigned char *)mem_descr->mem_array[0].
2343                                    virtual_address + BE_ISCSI_PDU_HEADER_SIZE);
2344
2345         memset(pnop_out, 0, BE_ISCSI_PDU_HEADER_SIZE);
2346         AMAP_SET_BITS(struct amap_pdu_nop_out, ttt, pnop_out, 0xFFFFFFFF);
2347         AMAP_SET_BITS(struct amap_pdu_nop_out, f_bit, pnop_out, 1);
2348         AMAP_SET_BITS(struct amap_pdu_nop_out, i_bit, pnop_out, 0);
2349 }
2350
2351 static void beiscsi_init_wrb_handle(struct beiscsi_hba *phba)
2352 {
2353         struct be_mem_descriptor *mem_descr_wrbh, *mem_descr_wrb;
2354         struct wrb_handle *pwrb_handle;
2355         struct hwi_controller *phwi_ctrlr;
2356         struct hwi_wrb_context *pwrb_context;
2357         struct iscsi_wrb *pwrb;
2358         unsigned int num_cxn_wrbh;
2359         unsigned int num_cxn_wrb, j, idx, index;
2360
2361         mem_descr_wrbh = phba->init_mem;
2362         mem_descr_wrbh += HWI_MEM_WRBH;
2363
2364         mem_descr_wrb = phba->init_mem;
2365         mem_descr_wrb += HWI_MEM_WRB;
2366
2367         idx = 0;
2368         pwrb_handle = mem_descr_wrbh->mem_array[idx].virtual_address;
2369         num_cxn_wrbh = ((mem_descr_wrbh->mem_array[idx].size) /
2370                         ((sizeof(struct wrb_handle)) *
2371                          phba->params.wrbs_per_cxn));
2372         phwi_ctrlr = phba->phwi_ctrlr;
2373
2374         for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) {
2375                 pwrb_context = &phwi_ctrlr->wrb_context[index];
2376                 pwrb_context->pwrb_handle_base =
2377                                 kzalloc(sizeof(struct wrb_handle *) *
2378                                         phba->params.wrbs_per_cxn, GFP_KERNEL);
2379                 pwrb_context->pwrb_handle_basestd =
2380                                 kzalloc(sizeof(struct wrb_handle *) *
2381                                         phba->params.wrbs_per_cxn, GFP_KERNEL);
2382                 if (num_cxn_wrbh) {
2383                         pwrb_context->alloc_index = 0;
2384                         pwrb_context->wrb_handles_available = 0;
2385                         for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2386                                 pwrb_context->pwrb_handle_base[j] = pwrb_handle;
2387                                 pwrb_context->pwrb_handle_basestd[j] =
2388                                                                 pwrb_handle;
2389                                 pwrb_context->wrb_handles_available++;
2390                                 pwrb_handle->wrb_index = j;
2391                                 pwrb_handle++;
2392                         }
2393                         pwrb_context->free_index = 0;
2394                         num_cxn_wrbh--;
2395                 } else {
2396                         idx++;
2397                         pwrb_handle =
2398                             mem_descr_wrbh->mem_array[idx].virtual_address;
2399                         num_cxn_wrbh =
2400                             ((mem_descr_wrbh->mem_array[idx].size) /
2401                              ((sizeof(struct wrb_handle)) *
2402                               phba->params.wrbs_per_cxn));
2403                         pwrb_context->alloc_index = 0;
2404                         for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2405                                 pwrb_context->pwrb_handle_base[j] = pwrb_handle;
2406                                 pwrb_context->pwrb_handle_basestd[j] =
2407                                     pwrb_handle;
2408                                 pwrb_context->wrb_handles_available++;
2409                                 pwrb_handle->wrb_index = j;
2410                                 pwrb_handle++;
2411                         }
2412                         pwrb_context->free_index = 0;
2413                         num_cxn_wrbh--;
2414                 }
2415         }
2416         idx = 0;
2417         pwrb = mem_descr_wrb->mem_array[idx].virtual_address;
2418         num_cxn_wrb = (mem_descr_wrb->mem_array[idx].size) /
2419                       ((sizeof(struct iscsi_wrb) *
2420                         phba->params.wrbs_per_cxn));
2421         for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) {
2422                 pwrb_context = &phwi_ctrlr->wrb_context[index];
2423                 if (num_cxn_wrb) {
2424                         for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2425                                 pwrb_handle = pwrb_context->pwrb_handle_base[j];
2426                                 pwrb_handle->pwrb = pwrb;
2427                                 pwrb++;
2428                         }
2429                         num_cxn_wrb--;
2430                 } else {
2431                         idx++;
2432                         pwrb = mem_descr_wrb->mem_array[idx].virtual_address;
2433                         num_cxn_wrb = (mem_descr_wrb->mem_array[idx].size) /
2434                                       ((sizeof(struct iscsi_wrb) *
2435                                         phba->params.wrbs_per_cxn));
2436                         for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2437                                 pwrb_handle = pwrb_context->pwrb_handle_base[j];
2438                                 pwrb_handle->pwrb = pwrb;
2439                                 pwrb++;
2440                         }
2441                         num_cxn_wrb--;
2442                 }
2443         }
2444 }
2445
2446 static void hwi_init_async_pdu_ctx(struct beiscsi_hba *phba)
2447 {
2448         struct hwi_controller *phwi_ctrlr;
2449         struct hba_parameters *p = &phba->params;
2450         struct hwi_async_pdu_context *pasync_ctx;
2451         struct async_pdu_handle *pasync_header_h, *pasync_data_h;
2452         unsigned int index;
2453         struct be_mem_descriptor *mem_descr;
2454
2455         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2456         mem_descr += HWI_MEM_ASYNC_PDU_CONTEXT;
2457
2458         phwi_ctrlr = phba->phwi_ctrlr;
2459         phwi_ctrlr->phwi_ctxt->pasync_ctx = (struct hwi_async_pdu_context *)
2460                                 mem_descr->mem_array[0].virtual_address;
2461         pasync_ctx = phwi_ctrlr->phwi_ctxt->pasync_ctx;
2462         memset(pasync_ctx, 0, sizeof(*pasync_ctx));
2463
2464         pasync_ctx->async_header.num_entries = p->asyncpdus_per_ctrl;
2465         pasync_ctx->async_header.buffer_size = p->defpdu_hdr_sz;
2466         pasync_ctx->async_data.buffer_size = p->defpdu_data_sz;
2467         pasync_ctx->async_data.num_entries = p->asyncpdus_per_ctrl;
2468
2469         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2470         mem_descr += HWI_MEM_ASYNC_HEADER_BUF;
2471         if (mem_descr->mem_array[0].virtual_address) {
2472                 SE_DEBUG(DBG_LVL_8,
2473                          "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_BUF"
2474                          "va=%p\n", mem_descr->mem_array[0].virtual_address);
2475         } else
2476                 shost_printk(KERN_WARNING, phba->shost,
2477                              "No Virtual address\n");
2478
2479         pasync_ctx->async_header.va_base =
2480                         mem_descr->mem_array[0].virtual_address;
2481
2482         pasync_ctx->async_header.pa_base.u.a64.address =
2483                         mem_descr->mem_array[0].bus_address.u.a64.address;
2484
2485         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2486         mem_descr += HWI_MEM_ASYNC_HEADER_RING;
2487         if (mem_descr->mem_array[0].virtual_address) {
2488                 SE_DEBUG(DBG_LVL_8,
2489                          "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_RING"
2490                          "va=%p\n", mem_descr->mem_array[0].virtual_address);
2491         } else
2492                 shost_printk(KERN_WARNING, phba->shost,
2493                             "No Virtual address\n");
2494         pasync_ctx->async_header.ring_base =
2495                         mem_descr->mem_array[0].virtual_address;
2496
2497         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2498         mem_descr += HWI_MEM_ASYNC_HEADER_HANDLE;
2499         if (mem_descr->mem_array[0].virtual_address) {
2500                 SE_DEBUG(DBG_LVL_8,
2501                          "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_HANDLE"
2502                          "va=%p\n", mem_descr->mem_array[0].virtual_address);
2503         } else
2504                 shost_printk(KERN_WARNING, phba->shost,
2505                             "No Virtual address\n");
2506
2507         pasync_ctx->async_header.handle_base =
2508                         mem_descr->mem_array[0].virtual_address;
2509         pasync_ctx->async_header.writables = 0;
2510         INIT_LIST_HEAD(&pasync_ctx->async_header.free_list);
2511
2512         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2513         mem_descr += HWI_MEM_ASYNC_DATA_BUF;
2514         if (mem_descr->mem_array[0].virtual_address) {
2515                 SE_DEBUG(DBG_LVL_8,
2516                          "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_DATA_BUF"
2517                          "va=%p\n", mem_descr->mem_array[0].virtual_address);
2518         } else
2519                 shost_printk(KERN_WARNING, phba->shost,
2520                             "No Virtual address\n");
2521         pasync_ctx->async_data.va_base =
2522                         mem_descr->mem_array[0].virtual_address;
2523         pasync_ctx->async_data.pa_base.u.a64.address =
2524                         mem_descr->mem_array[0].bus_address.u.a64.address;
2525
2526         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2527         mem_descr += HWI_MEM_ASYNC_DATA_RING;
2528         if (mem_descr->mem_array[0].virtual_address) {
2529                 SE_DEBUG(DBG_LVL_8,
2530                          "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_DATA_RING"
2531                          "va=%p\n", mem_descr->mem_array[0].virtual_address);
2532         } else
2533                 shost_printk(KERN_WARNING, phba->shost,
2534                              "No Virtual address\n");
2535
2536         pasync_ctx->async_data.ring_base =
2537                         mem_descr->mem_array[0].virtual_address;
2538
2539         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2540         mem_descr += HWI_MEM_ASYNC_DATA_HANDLE;
2541         if (!mem_descr->mem_array[0].virtual_address)
2542                 shost_printk(KERN_WARNING, phba->shost,
2543                             "No Virtual address\n");
2544
2545         pasync_ctx->async_data.handle_base =
2546                         mem_descr->mem_array[0].virtual_address;
2547         pasync_ctx->async_data.writables = 0;
2548         INIT_LIST_HEAD(&pasync_ctx->async_data.free_list);
2549
2550         pasync_header_h =
2551                 (struct async_pdu_handle *)pasync_ctx->async_header.handle_base;
2552         pasync_data_h =
2553                 (struct async_pdu_handle *)pasync_ctx->async_data.handle_base;
2554
2555         for (index = 0; index < p->asyncpdus_per_ctrl; index++) {
2556                 pasync_header_h->cri = -1;
2557                 pasync_header_h->index = (char)index;
2558                 INIT_LIST_HEAD(&pasync_header_h->link);
2559                 pasync_header_h->pbuffer =
2560                         (void *)((unsigned long)
2561                         (pasync_ctx->async_header.va_base) +
2562                         (p->defpdu_hdr_sz * index));
2563
2564                 pasync_header_h->pa.u.a64.address =
2565                         pasync_ctx->async_header.pa_base.u.a64.address +
2566                         (p->defpdu_hdr_sz * index);
2567
2568                 list_add_tail(&pasync_header_h->link,
2569                                 &pasync_ctx->async_header.free_list);
2570                 pasync_header_h++;
2571                 pasync_ctx->async_header.free_entries++;
2572                 pasync_ctx->async_header.writables++;
2573
2574                 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].wait_queue.list);
2575                 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].
2576                                header_busy_list);
2577                 pasync_data_h->cri = -1;
2578                 pasync_data_h->index = (char)index;
2579                 INIT_LIST_HEAD(&pasync_data_h->link);
2580                 pasync_data_h->pbuffer =
2581                         (void *)((unsigned long)
2582                         (pasync_ctx->async_data.va_base) +
2583                         (p->defpdu_data_sz * index));
2584
2585                 pasync_data_h->pa.u.a64.address =
2586                     pasync_ctx->async_data.pa_base.u.a64.address +
2587                     (p->defpdu_data_sz * index);
2588
2589                 list_add_tail(&pasync_data_h->link,
2590                               &pasync_ctx->async_data.free_list);
2591                 pasync_data_h++;
2592                 pasync_ctx->async_data.free_entries++;
2593                 pasync_ctx->async_data.writables++;
2594
2595                 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].data_busy_list);
2596         }
2597
2598         pasync_ctx->async_header.host_write_ptr = 0;
2599         pasync_ctx->async_header.ep_read_ptr = -1;
2600         pasync_ctx->async_data.host_write_ptr = 0;
2601         pasync_ctx->async_data.ep_read_ptr = -1;
2602 }
2603
2604 static int
2605 be_sgl_create_contiguous(void *virtual_address,
2606                          u64 physical_address, u32 length,
2607                          struct be_dma_mem *sgl)
2608 {
2609         WARN_ON(!virtual_address);
2610         WARN_ON(!physical_address);
2611         WARN_ON(!length > 0);
2612         WARN_ON(!sgl);
2613
2614         sgl->va = virtual_address;
2615         sgl->dma = (unsigned long)physical_address;
2616         sgl->size = length;
2617
2618         return 0;
2619 }
2620
2621 static void be_sgl_destroy_contiguous(struct be_dma_mem *sgl)
2622 {
2623         memset(sgl, 0, sizeof(*sgl));
2624 }
2625
2626 static void
2627 hwi_build_be_sgl_arr(struct beiscsi_hba *phba,
2628                      struct mem_array *pmem, struct be_dma_mem *sgl)
2629 {
2630         if (sgl->va)
2631                 be_sgl_destroy_contiguous(sgl);
2632
2633         be_sgl_create_contiguous(pmem->virtual_address,
2634                                  pmem->bus_address.u.a64.address,
2635                                  pmem->size, sgl);
2636 }
2637
2638 static void
2639 hwi_build_be_sgl_by_offset(struct beiscsi_hba *phba,
2640                            struct mem_array *pmem, struct be_dma_mem *sgl)
2641 {
2642         if (sgl->va)
2643                 be_sgl_destroy_contiguous(sgl);
2644
2645         be_sgl_create_contiguous((unsigned char *)pmem->virtual_address,
2646                                  pmem->bus_address.u.a64.address,
2647                                  pmem->size, sgl);
2648 }
2649
2650 static int be_fill_queue(struct be_queue_info *q,
2651                 u16 len, u16 entry_size, void *vaddress)
2652 {
2653         struct be_dma_mem *mem = &q->dma_mem;
2654
2655         memset(q, 0, sizeof(*q));
2656         q->len = len;
2657         q->entry_size = entry_size;
2658         mem->size = len * entry_size;
2659         mem->va = vaddress;
2660         if (!mem->va)
2661                 return -ENOMEM;
2662         memset(mem->va, 0, mem->size);
2663         return 0;
2664 }
2665
2666 static int beiscsi_create_eqs(struct beiscsi_hba *phba,
2667                              struct hwi_context_memory *phwi_context)
2668 {
2669         unsigned int i, num_eq_pages;
2670         int ret, eq_for_mcc;
2671         struct be_queue_info *eq;
2672         struct be_dma_mem *mem;
2673         void *eq_vaddress;
2674         dma_addr_t paddr;
2675
2676         num_eq_pages = PAGES_REQUIRED(phba->params.num_eq_entries * \
2677                                       sizeof(struct be_eq_entry));
2678
2679         if (phba->msix_enabled)
2680                 eq_for_mcc = 1;
2681         else
2682                 eq_for_mcc = 0;
2683         for (i = 0; i < (phba->num_cpus + eq_for_mcc); i++) {
2684                 eq = &phwi_context->be_eq[i].q;
2685                 mem = &eq->dma_mem;
2686                 phwi_context->be_eq[i].phba = phba;
2687                 eq_vaddress = pci_alloc_consistent(phba->pcidev,
2688                                                      num_eq_pages * PAGE_SIZE,
2689                                                      &paddr);
2690                 if (!eq_vaddress)
2691                         goto create_eq_error;
2692
2693                 mem->va = eq_vaddress;
2694                 ret = be_fill_queue(eq, phba->params.num_eq_entries,
2695                                     sizeof(struct be_eq_entry), eq_vaddress);
2696                 if (ret) {
2697                         shost_printk(KERN_ERR, phba->shost,
2698                                      "be_fill_queue Failed for EQ\n");
2699                         goto create_eq_error;
2700                 }
2701
2702                 mem->dma = paddr;
2703                 ret = beiscsi_cmd_eq_create(&phba->ctrl, eq,
2704                                             phwi_context->cur_eqd);
2705                 if (ret) {
2706                         shost_printk(KERN_ERR, phba->shost,
2707                                      "beiscsi_cmd_eq_create"
2708                                      "Failedfor EQ\n");
2709                         goto create_eq_error;
2710                 }
2711                 SE_DEBUG(DBG_LVL_8, "eqid = %d\n", phwi_context->be_eq[i].q.id);
2712         }
2713         return 0;
2714 create_eq_error:
2715         for (i = 0; i < (phba->num_cpus + 1); i++) {
2716                 eq = &phwi_context->be_eq[i].q;
2717                 mem = &eq->dma_mem;
2718                 if (mem->va)
2719                         pci_free_consistent(phba->pcidev, num_eq_pages
2720                                             * PAGE_SIZE,
2721                                             mem->va, mem->dma);
2722         }
2723         return ret;
2724 }
2725
2726 static int beiscsi_create_cqs(struct beiscsi_hba *phba,
2727                              struct hwi_context_memory *phwi_context)
2728 {
2729         unsigned int i, num_cq_pages;
2730         int ret;
2731         struct be_queue_info *cq, *eq;
2732         struct be_dma_mem *mem;
2733         struct be_eq_obj *pbe_eq;
2734         void *cq_vaddress;
2735         dma_addr_t paddr;
2736
2737         num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \
2738                                       sizeof(struct sol_cqe));
2739
2740         for (i = 0; i < phba->num_cpus; i++) {
2741                 cq = &phwi_context->be_cq[i];
2742                 eq = &phwi_context->be_eq[i].q;
2743                 pbe_eq = &phwi_context->be_eq[i];
2744                 pbe_eq->cq = cq;
2745                 pbe_eq->phba = phba;
2746                 mem = &cq->dma_mem;
2747                 cq_vaddress = pci_alloc_consistent(phba->pcidev,
2748                                                      num_cq_pages * PAGE_SIZE,
2749                                                      &paddr);
2750                 if (!cq_vaddress)
2751                         goto create_cq_error;
2752                 ret = be_fill_queue(cq, phba->params.num_cq_entries,
2753                                     sizeof(struct sol_cqe), cq_vaddress);
2754                 if (ret) {
2755                         shost_printk(KERN_ERR, phba->shost,
2756                                      "be_fill_queue Failed for ISCSI CQ\n");
2757                         goto create_cq_error;
2758                 }
2759
2760                 mem->dma = paddr;
2761                 ret = beiscsi_cmd_cq_create(&phba->ctrl, cq, eq, false,
2762                                             false, 0);
2763                 if (ret) {
2764                         shost_printk(KERN_ERR, phba->shost,
2765                                      "beiscsi_cmd_eq_create"
2766                                      "Failed for ISCSI CQ\n");
2767                         goto create_cq_error;
2768                 }
2769                 SE_DEBUG(DBG_LVL_8, "iscsi cq_id is %d for eq_id %d\n",
2770                                                  cq->id, eq->id);
2771                 SE_DEBUG(DBG_LVL_8, "ISCSI CQ CREATED\n");
2772         }
2773         return 0;
2774
2775 create_cq_error:
2776         for (i = 0; i < phba->num_cpus; i++) {
2777                 cq = &phwi_context->be_cq[i];
2778                 mem = &cq->dma_mem;
2779                 if (mem->va)
2780                         pci_free_consistent(phba->pcidev, num_cq_pages
2781                                             * PAGE_SIZE,
2782                                             mem->va, mem->dma);
2783         }
2784         return ret;
2785
2786 }
2787
2788 static int
2789 beiscsi_create_def_hdr(struct beiscsi_hba *phba,
2790                        struct hwi_context_memory *phwi_context,
2791                        struct hwi_controller *phwi_ctrlr,
2792                        unsigned int def_pdu_ring_sz)
2793 {
2794         unsigned int idx;
2795         int ret;
2796         struct be_queue_info *dq, *cq;
2797         struct be_dma_mem *mem;
2798         struct be_mem_descriptor *mem_descr;
2799         void *dq_vaddress;
2800
2801         idx = 0;
2802         dq = &phwi_context->be_def_hdrq;
2803         cq = &phwi_context->be_cq[0];
2804         mem = &dq->dma_mem;
2805         mem_descr = phba->init_mem;
2806         mem_descr += HWI_MEM_ASYNC_HEADER_RING;
2807         dq_vaddress = mem_descr->mem_array[idx].virtual_address;
2808         ret = be_fill_queue(dq, mem_descr->mem_array[0].size /
2809                             sizeof(struct phys_addr),
2810                             sizeof(struct phys_addr), dq_vaddress);
2811         if (ret) {
2812                 shost_printk(KERN_ERR, phba->shost,
2813                              "be_fill_queue Failed for DEF PDU HDR\n");
2814                 return ret;
2815         }
2816         mem->dma = (unsigned long)mem_descr->mem_array[idx].
2817                                   bus_address.u.a64.address;
2818         ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dq,
2819                                               def_pdu_ring_sz,
2820                                               phba->params.defpdu_hdr_sz);
2821         if (ret) {
2822                 shost_printk(KERN_ERR, phba->shost,
2823                              "be_cmd_create_default_pdu_queue Failed DEFHDR\n");
2824                 return ret;
2825         }
2826         phwi_ctrlr->default_pdu_hdr.id = phwi_context->be_def_hdrq.id;
2827         SE_DEBUG(DBG_LVL_8, "iscsi def pdu id is %d\n",
2828                  phwi_context->be_def_hdrq.id);
2829         hwi_post_async_buffers(phba, 1);
2830         return 0;
2831 }
2832
2833 static int
2834 beiscsi_create_def_data(struct beiscsi_hba *phba,
2835                         struct hwi_context_memory *phwi_context,
2836                         struct hwi_controller *phwi_ctrlr,
2837                         unsigned int def_pdu_ring_sz)
2838 {
2839         unsigned int idx;
2840         int ret;
2841         struct be_queue_info *dataq, *cq;
2842         struct be_dma_mem *mem;
2843         struct be_mem_descriptor *mem_descr;
2844         void *dq_vaddress;
2845
2846         idx = 0;
2847         dataq = &phwi_context->be_def_dataq;
2848         cq = &phwi_context->be_cq[0];
2849         mem = &dataq->dma_mem;
2850         mem_descr = phba->init_mem;
2851         mem_descr += HWI_MEM_ASYNC_DATA_RING;
2852         dq_vaddress = mem_descr->mem_array[idx].virtual_address;
2853         ret = be_fill_queue(dataq, mem_descr->mem_array[0].size /
2854                             sizeof(struct phys_addr),
2855                             sizeof(struct phys_addr), dq_vaddress);
2856         if (ret) {
2857                 shost_printk(KERN_ERR, phba->shost,
2858                              "be_fill_queue Failed for DEF PDU DATA\n");
2859                 return ret;
2860         }
2861         mem->dma = (unsigned long)mem_descr->mem_array[idx].
2862                                   bus_address.u.a64.address;
2863         ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dataq,
2864                                               def_pdu_ring_sz,
2865                                               phba->params.defpdu_data_sz);
2866         if (ret) {
2867                 shost_printk(KERN_ERR, phba->shost,
2868                              "be_cmd_create_default_pdu_queue Failed"
2869                              " for DEF PDU DATA\n");
2870                 return ret;
2871         }
2872         phwi_ctrlr->default_pdu_data.id = phwi_context->be_def_dataq.id;
2873         SE_DEBUG(DBG_LVL_8, "iscsi def data id is %d\n",
2874                  phwi_context->be_def_dataq.id);
2875         hwi_post_async_buffers(phba, 0);
2876         SE_DEBUG(DBG_LVL_8, "DEFAULT PDU DATA RING CREATED\n");
2877         return 0;
2878 }
2879
2880 static int
2881 beiscsi_post_pages(struct beiscsi_hba *phba)
2882 {
2883         struct be_mem_descriptor *mem_descr;
2884         struct mem_array *pm_arr;
2885         unsigned int page_offset, i;
2886         struct be_dma_mem sgl;
2887         int status;
2888
2889         mem_descr = phba->init_mem;
2890         mem_descr += HWI_MEM_SGE;
2891         pm_arr = mem_descr->mem_array;
2892
2893         page_offset = (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io *
2894                         phba->fw_config.iscsi_icd_start) / PAGE_SIZE;
2895         for (i = 0; i < mem_descr->num_elements; i++) {
2896                 hwi_build_be_sgl_arr(phba, pm_arr, &sgl);
2897                 status = be_cmd_iscsi_post_sgl_pages(&phba->ctrl, &sgl,
2898                                                 page_offset,
2899                                                 (pm_arr->size / PAGE_SIZE));
2900                 page_offset += pm_arr->size / PAGE_SIZE;
2901                 if (status != 0) {
2902                         shost_printk(KERN_ERR, phba->shost,
2903                                      "post sgl failed.\n");
2904                         return status;
2905                 }
2906                 pm_arr++;
2907         }
2908         SE_DEBUG(DBG_LVL_8, "POSTED PAGES\n");
2909         return 0;
2910 }
2911
2912 static void be_queue_free(struct beiscsi_hba *phba, struct be_queue_info *q)
2913 {
2914         struct be_dma_mem *mem = &q->dma_mem;
2915         if (mem->va)
2916                 pci_free_consistent(phba->pcidev, mem->size,
2917                         mem->va, mem->dma);
2918 }
2919
2920 static int be_queue_alloc(struct beiscsi_hba *phba, struct be_queue_info *q,
2921                 u16 len, u16 entry_size)
2922 {
2923         struct be_dma_mem *mem = &q->dma_mem;
2924
2925         memset(q, 0, sizeof(*q));
2926         q->len = len;
2927         q->entry_size = entry_size;
2928         mem->size = len * entry_size;
2929         mem->va = pci_alloc_consistent(phba->pcidev, mem->size, &mem->dma);
2930         if (!mem->va)
2931                 return -ENOMEM;
2932         memset(mem->va, 0, mem->size);
2933         return 0;
2934 }
2935
2936 static int
2937 beiscsi_create_wrb_rings(struct beiscsi_hba *phba,
2938                          struct hwi_context_memory *phwi_context,
2939                          struct hwi_controller *phwi_ctrlr)
2940 {
2941         unsigned int wrb_mem_index, offset, size, num_wrb_rings;
2942         u64 pa_addr_lo;
2943         unsigned int idx, num, i;
2944         struct mem_array *pwrb_arr;
2945         void *wrb_vaddr;
2946         struct be_dma_mem sgl;
2947         struct be_mem_descriptor *mem_descr;
2948         int status;
2949
2950         idx = 0;
2951         mem_descr = phba->init_mem;
2952         mem_descr += HWI_MEM_WRB;
2953         pwrb_arr = kmalloc(sizeof(*pwrb_arr) * phba->params.cxns_per_ctrl,
2954                            GFP_KERNEL);
2955         if (!pwrb_arr) {
2956                 shost_printk(KERN_ERR, phba->shost,
2957                              "Memory alloc failed in create wrb ring.\n");
2958                 return -ENOMEM;
2959         }
2960         wrb_vaddr = mem_descr->mem_array[idx].virtual_address;
2961         pa_addr_lo = mem_descr->mem_array[idx].bus_address.u.a64.address;
2962         num_wrb_rings = mem_descr->mem_array[idx].size /
2963                 (phba->params.wrbs_per_cxn * sizeof(struct iscsi_wrb));
2964
2965         for (num = 0; num < phba->params.cxns_per_ctrl; num++) {
2966                 if (num_wrb_rings) {
2967                         pwrb_arr[num].virtual_address = wrb_vaddr;
2968                         pwrb_arr[num].bus_address.u.a64.address = pa_addr_lo;
2969                         pwrb_arr[num].size = phba->params.wrbs_per_cxn *
2970                                             sizeof(struct iscsi_wrb);
2971                         wrb_vaddr += pwrb_arr[num].size;
2972                         pa_addr_lo += pwrb_arr[num].size;
2973                         num_wrb_rings--;
2974                 } else {
2975                         idx++;
2976                         wrb_vaddr = mem_descr->mem_array[idx].virtual_address;
2977                         pa_addr_lo = mem_descr->mem_array[idx].\
2978                                         bus_address.u.a64.address;
2979                         num_wrb_rings = mem_descr->mem_array[idx].size /
2980                                         (phba->params.wrbs_per_cxn *
2981                                         sizeof(struct iscsi_wrb));
2982                         pwrb_arr[num].virtual_address = wrb_vaddr;
2983                         pwrb_arr[num].bus_address.u.a64.address\
2984                                                 = pa_addr_lo;
2985                         pwrb_arr[num].size = phba->params.wrbs_per_cxn *
2986                                                  sizeof(struct iscsi_wrb);
2987                         wrb_vaddr += pwrb_arr[num].size;
2988                         pa_addr_lo   += pwrb_arr[num].size;
2989                         num_wrb_rings--;
2990                 }
2991         }
2992         for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
2993                 wrb_mem_index = 0;
2994                 offset = 0;
2995                 size = 0;
2996
2997                 hwi_build_be_sgl_by_offset(phba, &pwrb_arr[i], &sgl);
2998                 status = be_cmd_wrbq_create(&phba->ctrl, &sgl,
2999                                             &phwi_context->be_wrbq[i]);
3000                 if (status != 0) {
3001                         shost_printk(KERN_ERR, phba->shost,
3002                                      "wrbq create failed.");
3003                         kfree(pwrb_arr);
3004                         return status;
3005                 }
3006                 phwi_ctrlr->wrb_context[i * 2].cid = phwi_context->be_wrbq[i].
3007                                                                    id;
3008         }
3009         kfree(pwrb_arr);
3010         return 0;
3011 }
3012
3013 static void free_wrb_handles(struct beiscsi_hba *phba)
3014 {
3015         unsigned int index;
3016         struct hwi_controller *phwi_ctrlr;
3017         struct hwi_wrb_context *pwrb_context;
3018
3019         phwi_ctrlr = phba->phwi_ctrlr;
3020         for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) {
3021                 pwrb_context = &phwi_ctrlr->wrb_context[index];
3022                 kfree(pwrb_context->pwrb_handle_base);
3023                 kfree(pwrb_context->pwrb_handle_basestd);
3024         }
3025 }
3026
3027 static void be_mcc_queues_destroy(struct beiscsi_hba *phba)
3028 {
3029         struct be_queue_info *q;
3030         struct be_ctrl_info *ctrl = &phba->ctrl;
3031
3032         q = &phba->ctrl.mcc_obj.q;
3033         if (q->created)
3034                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_MCCQ);
3035         be_queue_free(phba, q);
3036
3037         q = &phba->ctrl.mcc_obj.cq;
3038         if (q->created)
3039                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ);
3040         be_queue_free(phba, q);
3041 }
3042
3043 static void hwi_cleanup(struct beiscsi_hba *phba)
3044 {
3045         struct be_queue_info *q;
3046         struct be_ctrl_info *ctrl = &phba->ctrl;
3047         struct hwi_controller *phwi_ctrlr;
3048         struct hwi_context_memory *phwi_context;
3049         int i, eq_num;
3050
3051         phwi_ctrlr = phba->phwi_ctrlr;
3052         phwi_context = phwi_ctrlr->phwi_ctxt;
3053         for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
3054                 q = &phwi_context->be_wrbq[i];
3055                 if (q->created)
3056                         beiscsi_cmd_q_destroy(ctrl, q, QTYPE_WRBQ);
3057         }
3058         free_wrb_handles(phba);
3059
3060         q = &phwi_context->be_def_hdrq;
3061         if (q->created)
3062                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ);
3063
3064         q = &phwi_context->be_def_dataq;
3065         if (q->created)
3066                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ);
3067
3068         beiscsi_cmd_q_destroy(ctrl, NULL, QTYPE_SGL);
3069
3070         for (i = 0; i < (phba->num_cpus); i++) {
3071                 q = &phwi_context->be_cq[i];
3072                 if (q->created)
3073                         beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ);
3074         }
3075         if (phba->msix_enabled)
3076                 eq_num = 1;
3077         else
3078                 eq_num = 0;
3079         for (i = 0; i < (phba->num_cpus + eq_num); i++) {
3080                 q = &phwi_context->be_eq[i].q;
3081                 if (q->created)
3082                         beiscsi_cmd_q_destroy(ctrl, q, QTYPE_EQ);
3083         }
3084         be_mcc_queues_destroy(phba);
3085 }
3086
3087 static int be_mcc_queues_create(struct beiscsi_hba *phba,
3088                                 struct hwi_context_memory *phwi_context)
3089 {
3090         struct be_queue_info *q, *cq;
3091         struct be_ctrl_info *ctrl = &phba->ctrl;
3092
3093         /* Alloc MCC compl queue */
3094         cq = &phba->ctrl.mcc_obj.cq;
3095         if (be_queue_alloc(phba, cq, MCC_CQ_LEN,
3096                         sizeof(struct be_mcc_compl)))
3097                 goto err;
3098         /* Ask BE to create MCC compl queue; */
3099         if (phba->msix_enabled) {
3100                 if (beiscsi_cmd_cq_create(ctrl, cq, &phwi_context->be_eq
3101                                          [phba->num_cpus].q, false, true, 0))
3102                 goto mcc_cq_free;
3103         } else {
3104                 if (beiscsi_cmd_cq_create(ctrl, cq, &phwi_context->be_eq[0].q,
3105                                           false, true, 0))
3106                 goto mcc_cq_free;
3107         }
3108
3109         /* Alloc MCC queue */
3110         q = &phba->ctrl.mcc_obj.q;
3111         if (be_queue_alloc(phba, q, MCC_Q_LEN, sizeof(struct be_mcc_wrb)))
3112                 goto mcc_cq_destroy;
3113
3114         /* Ask BE to create MCC queue */
3115         if (beiscsi_cmd_mccq_create(phba, q, cq))
3116                 goto mcc_q_free;
3117
3118         return 0;
3119
3120 mcc_q_free:
3121         be_queue_free(phba, q);
3122 mcc_cq_destroy:
3123         beiscsi_cmd_q_destroy(ctrl, cq, QTYPE_CQ);
3124 mcc_cq_free:
3125         be_queue_free(phba, cq);
3126 err:
3127         return -ENOMEM;
3128 }
3129
3130 static int find_num_cpus(void)
3131 {
3132         int  num_cpus = 0;
3133
3134         num_cpus = num_online_cpus();
3135         if (num_cpus >= MAX_CPUS)
3136                 num_cpus = MAX_CPUS - 1;
3137
3138         SE_DEBUG(DBG_LVL_8, "num_cpus = %d\n", num_cpus);
3139         return num_cpus;
3140 }
3141
3142 static int hwi_init_port(struct beiscsi_hba *phba)
3143 {
3144         struct hwi_controller *phwi_ctrlr;
3145         struct hwi_context_memory *phwi_context;
3146         unsigned int def_pdu_ring_sz;
3147         struct be_ctrl_info *ctrl = &phba->ctrl;
3148         int status;
3149
3150         def_pdu_ring_sz =
3151                 phba->params.asyncpdus_per_ctrl * sizeof(struct phys_addr);
3152         phwi_ctrlr = phba->phwi_ctrlr;
3153         phwi_context = phwi_ctrlr->phwi_ctxt;
3154         phwi_context->max_eqd = 0;
3155         phwi_context->min_eqd = 0;
3156         phwi_context->cur_eqd = 64;
3157         be_cmd_fw_initialize(&phba->ctrl);
3158
3159         status = beiscsi_create_eqs(phba, phwi_context);
3160         if (status != 0) {
3161                 shost_printk(KERN_ERR, phba->shost, "EQ not created\n");
3162                 goto error;
3163         }
3164
3165         status = be_mcc_queues_create(phba, phwi_context);
3166         if (status != 0)
3167                 goto error;
3168
3169         status = mgmt_check_supported_fw(ctrl, phba);
3170         if (status != 0) {
3171                 shost_printk(KERN_ERR, phba->shost,
3172                              "Unsupported fw version\n");
3173                 goto error;
3174         }
3175
3176         status = beiscsi_create_cqs(phba, phwi_context);
3177         if (status != 0) {
3178                 shost_printk(KERN_ERR, phba->shost, "CQ not created\n");
3179                 goto error;
3180         }
3181
3182         status = beiscsi_create_def_hdr(phba, phwi_context, phwi_ctrlr,
3183                                         def_pdu_ring_sz);
3184         if (status != 0) {
3185                 shost_printk(KERN_ERR, phba->shost,
3186                              "Default Header not created\n");
3187                 goto error;
3188         }
3189
3190         status = beiscsi_create_def_data(phba, phwi_context,
3191                                          phwi_ctrlr, def_pdu_ring_sz);
3192         if (status != 0) {
3193                 shost_printk(KERN_ERR, phba->shost,
3194                              "Default Data not created\n");
3195                 goto error;
3196         }
3197
3198         status = beiscsi_post_pages(phba);
3199         if (status != 0) {
3200                 shost_printk(KERN_ERR, phba->shost, "Post SGL Pages Failed\n");
3201                 goto error;
3202         }
3203
3204         status = beiscsi_create_wrb_rings(phba, phwi_context, phwi_ctrlr);
3205         if (status != 0) {
3206                 shost_printk(KERN_ERR, phba->shost,
3207                              "WRB Rings not created\n");
3208                 goto error;
3209         }
3210
3211         SE_DEBUG(DBG_LVL_8, "hwi_init_port success\n");
3212         return 0;
3213
3214 error:
3215         shost_printk(KERN_ERR, phba->shost, "hwi_init_port failed");
3216         hwi_cleanup(phba);
3217         return -ENOMEM;
3218 }
3219
3220 static int hwi_init_controller(struct beiscsi_hba *phba)
3221 {
3222         struct hwi_controller *phwi_ctrlr;
3223
3224         phwi_ctrlr = phba->phwi_ctrlr;
3225         if (1 == phba->init_mem[HWI_MEM_ADDN_CONTEXT].num_elements) {
3226                 phwi_ctrlr->phwi_ctxt = (struct hwi_context_memory *)phba->
3227                     init_mem[HWI_MEM_ADDN_CONTEXT].mem_array[0].virtual_address;
3228                 SE_DEBUG(DBG_LVL_8, " phwi_ctrlr->phwi_ctxt=%p\n",
3229                          phwi_ctrlr->phwi_ctxt);
3230         } else {
3231                 shost_printk(KERN_ERR, phba->shost,
3232                              "HWI_MEM_ADDN_CONTEXT is more than one element."
3233                              "Failing to load\n");
3234                 return -ENOMEM;
3235         }
3236
3237         iscsi_init_global_templates(phba);
3238         beiscsi_init_wrb_handle(phba);
3239         hwi_init_async_pdu_ctx(phba);
3240         if (hwi_init_port(phba) != 0) {
3241                 shost_printk(KERN_ERR, phba->shost,
3242                              "hwi_init_controller failed\n");
3243                 return -ENOMEM;
3244         }
3245         return 0;
3246 }
3247
3248 static void beiscsi_free_mem(struct beiscsi_hba *phba)
3249 {
3250         struct be_mem_descriptor *mem_descr;
3251         int i, j;
3252
3253         mem_descr = phba->init_mem;
3254         i = 0;
3255         j = 0;
3256         for (i = 0; i < SE_MEM_MAX; i++) {
3257                 for (j = mem_descr->num_elements; j > 0; j--) {
3258                         pci_free_consistent(phba->pcidev,
3259                           mem_descr->mem_array[j - 1].size,
3260                           mem_descr->mem_array[j - 1].virtual_address,
3261                           (unsigned long)mem_descr->mem_array[j - 1].
3262                           bus_address.u.a64.address);
3263                 }
3264                 kfree(mem_descr->mem_array);
3265                 mem_descr++;
3266         }
3267         kfree(phba->init_mem);
3268         kfree(phba->phwi_ctrlr);
3269 }
3270
3271 static int beiscsi_init_controller(struct beiscsi_hba *phba)
3272 {
3273         int ret = -ENOMEM;
3274
3275         ret = beiscsi_get_memory(phba);
3276         if (ret < 0) {
3277                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe -"
3278                              "Failed in beiscsi_alloc_memory\n");
3279                 return ret;
3280         }
3281
3282         ret = hwi_init_controller(phba);
3283         if (ret)
3284                 goto free_init;
3285         SE_DEBUG(DBG_LVL_8, "Return success from beiscsi_init_controller");
3286         return 0;
3287
3288 free_init:
3289         beiscsi_free_mem(phba);
3290         return -ENOMEM;
3291 }
3292
3293 static int beiscsi_init_sgl_handle(struct beiscsi_hba *phba)
3294 {
3295         struct be_mem_descriptor *mem_descr_sglh, *mem_descr_sg;
3296         struct sgl_handle *psgl_handle;
3297         struct iscsi_sge *pfrag;
3298         unsigned int arr_index, i, idx;
3299
3300         phba->io_sgl_hndl_avbl = 0;
3301         phba->eh_sgl_hndl_avbl = 0;
3302
3303         mem_descr_sglh = phba->init_mem;
3304         mem_descr_sglh += HWI_MEM_SGLH;
3305         if (1 == mem_descr_sglh->num_elements) {
3306                 phba->io_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) *
3307                                                  phba->params.ios_per_ctrl,
3308                                                  GFP_KERNEL);
3309                 if (!phba->io_sgl_hndl_base) {
3310                         shost_printk(KERN_ERR, phba->shost,
3311                                      "Mem Alloc Failed. Failing to load\n");
3312                         return -ENOMEM;
3313                 }
3314                 phba->eh_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) *
3315                                                  (phba->params.icds_per_ctrl -
3316                                                  phba->params.ios_per_ctrl),
3317                                                  GFP_KERNEL);
3318                 if (!phba->eh_sgl_hndl_base) {
3319                         kfree(phba->io_sgl_hndl_base);
3320                         shost_printk(KERN_ERR, phba->shost,
3321                                      "Mem Alloc Failed. Failing to load\n");
3322                         return -ENOMEM;
3323                 }
3324         } else {
3325                 shost_printk(KERN_ERR, phba->shost,
3326                              "HWI_MEM_SGLH is more than one element."
3327                              "Failing to load\n");
3328                 return -ENOMEM;
3329         }
3330
3331         arr_index = 0;
3332         idx = 0;
3333         while (idx < mem_descr_sglh->num_elements) {
3334                 psgl_handle = mem_descr_sglh->mem_array[idx].virtual_address;
3335
3336                 for (i = 0; i < (mem_descr_sglh->mem_array[idx].size /
3337                       sizeof(struct sgl_handle)); i++) {
3338                         if (arr_index < phba->params.ios_per_ctrl) {
3339                                 phba->io_sgl_hndl_base[arr_index] = psgl_handle;
3340                                 phba->io_sgl_hndl_avbl++;
3341                                 arr_index++;
3342                         } else {
3343                                 phba->eh_sgl_hndl_base[arr_index -
3344                                         phba->params.ios_per_ctrl] =
3345                                                                 psgl_handle;
3346                                 arr_index++;
3347                                 phba->eh_sgl_hndl_avbl++;
3348                         }
3349                         psgl_handle++;
3350                 }
3351                 idx++;
3352         }
3353         SE_DEBUG(DBG_LVL_8,
3354                  "phba->io_sgl_hndl_avbl=%d"
3355                  "phba->eh_sgl_hndl_avbl=%d\n",
3356                  phba->io_sgl_hndl_avbl,
3357                  phba->eh_sgl_hndl_avbl);
3358         mem_descr_sg = phba->init_mem;
3359         mem_descr_sg += HWI_MEM_SGE;
3360         SE_DEBUG(DBG_LVL_8, "\n mem_descr_sg->num_elements=%d\n",
3361                  mem_descr_sg->num_elements);
3362         arr_index = 0;
3363         idx = 0;
3364         while (idx < mem_descr_sg->num_elements) {
3365                 pfrag = mem_descr_sg->mem_array[idx].virtual_address;
3366
3367                 for (i = 0;
3368                      i < (mem_descr_sg->mem_array[idx].size) /
3369                      (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io);
3370                      i++) {
3371                         if (arr_index < phba->params.ios_per_ctrl)
3372                                 psgl_handle = phba->io_sgl_hndl_base[arr_index];
3373                         else
3374                                 psgl_handle = phba->eh_sgl_hndl_base[arr_index -
3375                                                 phba->params.ios_per_ctrl];
3376                         psgl_handle->pfrag = pfrag;
3377                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, pfrag, 0);
3378                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, pfrag, 0);
3379                         pfrag += phba->params.num_sge_per_io;
3380                         psgl_handle->sgl_index =
3381                                 phba->fw_config.iscsi_icd_start + arr_index++;
3382                 }
3383                 idx++;
3384         }
3385         phba->io_sgl_free_index = 0;
3386         phba->io_sgl_alloc_index = 0;
3387         phba->eh_sgl_free_index = 0;
3388         phba->eh_sgl_alloc_index = 0;
3389         return 0;
3390 }
3391
3392 static int hba_setup_cid_tbls(struct beiscsi_hba *phba)
3393 {
3394         int i, new_cid;
3395
3396         phba->cid_array = kzalloc(sizeof(void *) * phba->params.cxns_per_ctrl,
3397                                   GFP_KERNEL);
3398         if (!phba->cid_array) {
3399                 shost_printk(KERN_ERR, phba->shost,
3400                              "Failed to allocate memory in "
3401                              "hba_setup_cid_tbls\n");
3402                 return -ENOMEM;
3403         }
3404         phba->ep_array = kzalloc(sizeof(struct iscsi_endpoint *) *
3405                                  phba->params.cxns_per_ctrl * 2, GFP_KERNEL);
3406         if (!phba->ep_array) {
3407                 shost_printk(KERN_ERR, phba->shost,
3408                              "Failed to allocate memory in "
3409                              "hba_setup_cid_tbls\n");
3410                 kfree(phba->cid_array);
3411                 return -ENOMEM;
3412         }
3413         new_cid = phba->fw_config.iscsi_cid_start;
3414         for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
3415                 phba->cid_array[i] = new_cid;
3416                 new_cid += 2;
3417         }
3418         phba->avlbl_cids = phba->params.cxns_per_ctrl;
3419         return 0;
3420 }
3421
3422 static void hwi_enable_intr(struct beiscsi_hba *phba)
3423 {
3424         struct be_ctrl_info *ctrl = &phba->ctrl;
3425         struct hwi_controller *phwi_ctrlr;
3426         struct hwi_context_memory *phwi_context;
3427         struct be_queue_info *eq;
3428         u8 __iomem *addr;
3429         u32 reg, i;
3430         u32 enabled;
3431
3432         phwi_ctrlr = phba->phwi_ctrlr;
3433         phwi_context = phwi_ctrlr->phwi_ctxt;
3434
3435         addr = (u8 __iomem *) ((u8 __iomem *) ctrl->pcicfg +
3436                         PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET);
3437         reg = ioread32(addr);
3438
3439         enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3440         if (!enabled) {
3441                 reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3442                 SE_DEBUG(DBG_LVL_8, "reg =x%08x addr=%p\n", reg, addr);
3443                 iowrite32(reg, addr);
3444         }
3445
3446         if (!phba->msix_enabled) {
3447                 eq = &phwi_context->be_eq[0].q;
3448                 SE_DEBUG(DBG_LVL_8, "eq->id=%d\n", eq->id);
3449                 hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1);
3450         } else {
3451                 for (i = 0; i <= phba->num_cpus; i++) {
3452                         eq = &phwi_context->be_eq[i].q;
3453                         SE_DEBUG(DBG_LVL_8, "eq->id=%d\n", eq->id);
3454                         hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1);
3455                 }
3456         }
3457 }
3458
3459 static void hwi_disable_intr(struct beiscsi_hba *phba)
3460 {
3461         struct be_ctrl_info *ctrl = &phba->ctrl;
3462
3463         u8 __iomem *addr = ctrl->pcicfg + PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET;
3464         u32 reg = ioread32(addr);
3465
3466         u32 enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3467         if (enabled) {
3468                 reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3469                 iowrite32(reg, addr);
3470         } else
3471                 shost_printk(KERN_WARNING, phba->shost,
3472                              "In hwi_disable_intr, Already Disabled\n");
3473 }
3474
3475 static int beiscsi_get_boot_info(struct beiscsi_hba *phba)
3476 {
3477         struct be_cmd_resp_get_boot_target *boot_resp;
3478         struct be_cmd_resp_get_session *session_resp;
3479         struct be_mcc_wrb *wrb;
3480         struct be_dma_mem nonemb_cmd;
3481         unsigned int tag, wrb_num;
3482         unsigned short status, extd_status;
3483         struct be_queue_info *mccq = &phba->ctrl.mcc_obj.q;
3484         int ret = -ENOMEM;
3485
3486         tag = beiscsi_get_boot_target(phba);
3487         if (!tag) {
3488                 SE_DEBUG(DBG_LVL_1, "be_cmd_get_mac_addr Failed\n");
3489                 return -EAGAIN;
3490         } else
3491                 wait_event_interruptible(phba->ctrl.mcc_wait[tag],
3492                                          phba->ctrl.mcc_numtag[tag]);
3493
3494         wrb_num = (phba->ctrl.mcc_numtag[tag] & 0x00FF0000) >> 16;
3495         extd_status = (phba->ctrl.mcc_numtag[tag] & 0x0000FF00) >> 8;
3496         status = phba->ctrl.mcc_numtag[tag] & 0x000000FF;
3497         if (status || extd_status) {
3498                 SE_DEBUG(DBG_LVL_1, "be_cmd_get_mac_addr Failed"
3499                                     " status = %d extd_status = %d\n",
3500                                     status, extd_status);
3501                 free_mcc_tag(&phba->ctrl, tag);
3502                 return -EBUSY;
3503         }
3504         wrb = queue_get_wrb(mccq, wrb_num);
3505         free_mcc_tag(&phba->ctrl, tag);
3506         boot_resp = embedded_payload(wrb);
3507
3508         if (boot_resp->boot_session_handle < 0) {
3509                 shost_printk(KERN_INFO, phba->shost, "No Boot Session.\n");
3510                 return -ENXIO;
3511         }
3512
3513         nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
3514                                 sizeof(*session_resp),
3515                                 &nonemb_cmd.dma);
3516         if (nonemb_cmd.va == NULL) {
3517                 SE_DEBUG(DBG_LVL_1,
3518                          "Failed to allocate memory for"
3519                          "beiscsi_get_session_info\n");
3520                 return -ENOMEM;
3521         }
3522
3523         memset(nonemb_cmd.va, 0, sizeof(*session_resp));
3524         tag = beiscsi_get_session_info(phba,
3525                 boot_resp->boot_session_handle, &nonemb_cmd);
3526         if (!tag) {
3527                 SE_DEBUG(DBG_LVL_1, "beiscsi_get_session_info"
3528                         " Failed\n");
3529                 goto boot_freemem;
3530         } else
3531                 wait_event_interruptible(phba->ctrl.mcc_wait[tag],
3532                                          phba->ctrl.mcc_numtag[tag]);
3533
3534         wrb_num = (phba->ctrl.mcc_numtag[tag] & 0x00FF0000) >> 16;
3535         extd_status = (phba->ctrl.mcc_numtag[tag] & 0x0000FF00) >> 8;
3536         status = phba->ctrl.mcc_numtag[tag] & 0x000000FF;
3537         if (status || extd_status) {
3538                 SE_DEBUG(DBG_LVL_1, "beiscsi_get_session_info Failed"
3539                                     " status = %d extd_status = %d\n",
3540                                     status, extd_status);
3541                 free_mcc_tag(&phba->ctrl, tag);
3542                 goto boot_freemem;
3543         }
3544         wrb = queue_get_wrb(mccq, wrb_num);
3545         free_mcc_tag(&phba->ctrl, tag);
3546         session_resp = nonemb_cmd.va ;
3547
3548         memcpy(&phba->boot_sess, &session_resp->session_info,
3549                sizeof(struct mgmt_session_info));
3550         ret = 0;
3551
3552 boot_freemem:
3553         pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
3554                     nonemb_cmd.va, nonemb_cmd.dma);
3555         return ret;
3556 }
3557
3558 static void beiscsi_boot_release(void *data)
3559 {
3560         struct beiscsi_hba *phba = data;
3561
3562         scsi_host_put(phba->shost);
3563 }
3564
3565 static int beiscsi_setup_boot_info(struct beiscsi_hba *phba)
3566 {
3567         struct iscsi_boot_kobj *boot_kobj;
3568
3569         /* get boot info using mgmt cmd */
3570         if (beiscsi_get_boot_info(phba))
3571                 /* Try to see if we can carry on without this */
3572                 return 0;
3573
3574         phba->boot_kset = iscsi_boot_create_host_kset(phba->shost->host_no);
3575         if (!phba->boot_kset)
3576                 return -ENOMEM;
3577
3578         /* get a ref because the show function will ref the phba */
3579         if (!scsi_host_get(phba->shost))
3580                 goto free_kset;
3581         boot_kobj = iscsi_boot_create_target(phba->boot_kset, 0, phba,
3582                                              beiscsi_show_boot_tgt_info,
3583                                              beiscsi_tgt_get_attr_visibility,
3584                                              beiscsi_boot_release);
3585         if (!boot_kobj)
3586                 goto put_shost;
3587
3588         if (!scsi_host_get(phba->shost))
3589                 goto free_kset;
3590         boot_kobj = iscsi_boot_create_initiator(phba->boot_kset, 0, phba,
3591                                                 beiscsi_show_boot_ini_info,
3592                                                 beiscsi_ini_get_attr_visibility,
3593                                                 beiscsi_boot_release);
3594         if (!boot_kobj)
3595                 goto put_shost;
3596
3597         if (!scsi_host_get(phba->shost))
3598                 goto free_kset;
3599         boot_kobj = iscsi_boot_create_ethernet(phba->boot_kset, 0, phba,
3600                                                beiscsi_show_boot_eth_info,
3601                                                beiscsi_eth_get_attr_visibility,
3602                                                beiscsi_boot_release);
3603         if (!boot_kobj)
3604                 goto put_shost;
3605         return 0;
3606
3607 put_shost:
3608         scsi_host_put(phba->shost);
3609 free_kset:
3610         iscsi_boot_destroy_kset(phba->boot_kset);
3611         return -ENOMEM;
3612 }
3613
3614 static int beiscsi_init_port(struct beiscsi_hba *phba)
3615 {
3616         int ret;
3617
3618         ret = beiscsi_init_controller(phba);
3619         if (ret < 0) {
3620                 shost_printk(KERN_ERR, phba->shost,
3621                              "beiscsi_dev_probe - Failed in"
3622                              "beiscsi_init_controller\n");
3623                 return ret;
3624         }
3625         ret = beiscsi_init_sgl_handle(phba);
3626         if (ret < 0) {
3627                 shost_printk(KERN_ERR, phba->shost,
3628                              "beiscsi_dev_probe - Failed in"
3629                              "beiscsi_init_sgl_handle\n");
3630                 goto do_cleanup_ctrlr;
3631         }
3632
3633         if (hba_setup_cid_tbls(phba)) {
3634                 shost_printk(KERN_ERR, phba->shost,
3635                              "Failed in hba_setup_cid_tbls\n");
3636                 kfree(phba->io_sgl_hndl_base);
3637                 kfree(phba->eh_sgl_hndl_base);
3638                 goto do_cleanup_ctrlr;
3639         }
3640
3641         return ret;
3642
3643 do_cleanup_ctrlr:
3644         hwi_cleanup(phba);
3645         return ret;
3646 }
3647
3648 static void hwi_purge_eq(struct beiscsi_hba *phba)
3649 {
3650         struct hwi_controller *phwi_ctrlr;
3651         struct hwi_context_memory *phwi_context;
3652         struct be_queue_info *eq;
3653         struct be_eq_entry *eqe = NULL;
3654         int i, eq_msix;
3655         unsigned int num_processed;
3656
3657         phwi_ctrlr = phba->phwi_ctrlr;
3658         phwi_context = phwi_ctrlr->phwi_ctxt;
3659         if (phba->msix_enabled)
3660                 eq_msix = 1;
3661         else
3662                 eq_msix = 0;
3663
3664         for (i = 0; i < (phba->num_cpus + eq_msix); i++) {
3665                 eq = &phwi_context->be_eq[i].q;
3666                 eqe = queue_tail_node(eq);
3667                 num_processed = 0;
3668                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
3669                                         & EQE_VALID_MASK) {
3670                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
3671                         queue_tail_inc(eq);
3672                         eqe = queue_tail_node(eq);
3673                         num_processed++;
3674                 }
3675
3676                 if (num_processed)
3677                         hwi_ring_eq_db(phba, eq->id, 1, num_processed, 1, 1);
3678         }
3679 }
3680
3681 static void beiscsi_clean_port(struct beiscsi_hba *phba)
3682 {
3683         int mgmt_status;
3684
3685         mgmt_status = mgmt_epfw_cleanup(phba, CMD_CONNECTION_CHUTE_0);
3686         if (mgmt_status)
3687                 shost_printk(KERN_WARNING, phba->shost,
3688                              "mgmt_epfw_cleanup FAILED\n");
3689
3690         hwi_purge_eq(phba);
3691         hwi_cleanup(phba);
3692         kfree(phba->io_sgl_hndl_base);
3693         kfree(phba->eh_sgl_hndl_base);
3694         kfree(phba->cid_array);
3695         kfree(phba->ep_array);
3696 }
3697
3698 void
3699 beiscsi_offload_connection(struct beiscsi_conn *beiscsi_conn,
3700                            struct beiscsi_offload_params *params)
3701 {
3702         struct wrb_handle *pwrb_handle;
3703         struct iscsi_target_context_update_wrb *pwrb = NULL;
3704         struct be_mem_descriptor *mem_descr;
3705         struct beiscsi_hba *phba = beiscsi_conn->phba;
3706         u32 doorbell = 0;
3707
3708         /*
3709          * We can always use 0 here because it is reserved by libiscsi for
3710          * login/startup related tasks.
3711          */
3712         pwrb_handle = alloc_wrb_handle(phba, (beiscsi_conn->beiscsi_conn_cid -
3713                                        phba->fw_config.iscsi_cid_start));
3714         pwrb = (struct iscsi_target_context_update_wrb *)pwrb_handle->pwrb;
3715         memset(pwrb, 0, sizeof(*pwrb));
3716         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3717                       max_burst_length, pwrb, params->dw[offsetof
3718                       (struct amap_beiscsi_offload_params,
3719                       max_burst_length) / 32]);
3720         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3721                       max_send_data_segment_length, pwrb,
3722                       params->dw[offsetof(struct amap_beiscsi_offload_params,
3723                       max_send_data_segment_length) / 32]);
3724         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3725                       first_burst_length,
3726                       pwrb,
3727                       params->dw[offsetof(struct amap_beiscsi_offload_params,
3728                       first_burst_length) / 32]);
3729
3730         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, erl, pwrb,
3731                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
3732                       erl) / 32] & OFFLD_PARAMS_ERL));
3733         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, dde, pwrb,
3734                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
3735                       dde) / 32] & OFFLD_PARAMS_DDE) >> 2);
3736         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, hde, pwrb,
3737                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
3738                       hde) / 32] & OFFLD_PARAMS_HDE) >> 3);
3739         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, ir2t, pwrb,
3740                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
3741                       ir2t) / 32] & OFFLD_PARAMS_IR2T) >> 4);
3742         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, imd, pwrb,
3743                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
3744                        imd) / 32] & OFFLD_PARAMS_IMD) >> 5);
3745         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, stat_sn,
3746                       pwrb,
3747                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
3748                       exp_statsn) / 32] + 1));
3749         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, type, pwrb,
3750                       0x7);
3751         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, wrb_idx,
3752                       pwrb, pwrb_handle->wrb_index);
3753         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, ptr2nextwrb,
3754                       pwrb, pwrb_handle->nxt_wrb_index);
3755         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3756                         session_state, pwrb, 0);
3757         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, compltonack,
3758                       pwrb, 1);
3759         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, notpredblq,
3760                       pwrb, 0);
3761         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, mode, pwrb,
3762                       0);
3763
3764         mem_descr = phba->init_mem;
3765         mem_descr += ISCSI_MEM_GLOBAL_HEADER;
3766
3767         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3768                         pad_buffer_addr_hi, pwrb,
3769                       mem_descr->mem_array[0].bus_address.u.a32.address_hi);
3770         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
3771                         pad_buffer_addr_lo, pwrb,
3772                       mem_descr->mem_array[0].bus_address.u.a32.address_lo);
3773
3774         be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_target_context_update_wrb));
3775
3776         doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
3777         doorbell |= (pwrb_handle->wrb_index & DB_DEF_PDU_WRB_INDEX_MASK)
3778                              << DB_DEF_PDU_WRB_INDEX_SHIFT;
3779         doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
3780
3781         iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
3782 }
3783
3784 static void beiscsi_parse_pdu(struct iscsi_conn *conn, itt_t itt,
3785                               int *index, int *age)
3786 {
3787         *index = (int)itt;
3788         if (age)
3789                 *age = conn->session->age;
3790 }
3791
3792 /**
3793  * beiscsi_alloc_pdu - allocates pdu and related resources
3794  * @task: libiscsi task
3795  * @opcode: opcode of pdu for task
3796  *
3797  * This is called with the session lock held. It will allocate
3798  * the wrb and sgl if needed for the command. And it will prep
3799  * the pdu's itt. beiscsi_parse_pdu will later translate
3800  * the pdu itt to the libiscsi task itt.
3801  */
3802 static int beiscsi_alloc_pdu(struct iscsi_task *task, uint8_t opcode)
3803 {
3804         struct beiscsi_io_task *io_task = task->dd_data;
3805         struct iscsi_conn *conn = task->conn;
3806         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3807         struct beiscsi_hba *phba = beiscsi_conn->phba;
3808         struct hwi_wrb_context *pwrb_context;
3809         struct hwi_controller *phwi_ctrlr;
3810         itt_t itt;
3811         struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess;
3812         dma_addr_t paddr;
3813
3814         io_task->cmd_bhs = pci_pool_alloc(beiscsi_sess->bhs_pool,
3815                                           GFP_ATOMIC, &paddr);
3816         if (!io_task->cmd_bhs)
3817                 return -ENOMEM;
3818         io_task->bhs_pa.u.a64.address = paddr;
3819         io_task->libiscsi_itt = (itt_t)task->itt;
3820         io_task->conn = beiscsi_conn;
3821
3822         task->hdr = (struct iscsi_hdr *)&io_task->cmd_bhs->iscsi_hdr;
3823         task->hdr_max = sizeof(struct be_cmd_bhs);
3824         io_task->psgl_handle = NULL;
3825         io_task->psgl_handle = NULL;
3826
3827         if (task->sc) {
3828                 spin_lock(&phba->io_sgl_lock);
3829                 io_task->psgl_handle = alloc_io_sgl_handle(phba);
3830                 spin_unlock(&phba->io_sgl_lock);
3831                 if (!io_task->psgl_handle)
3832                         goto free_hndls;
3833                 io_task->pwrb_handle = alloc_wrb_handle(phba,
3834                                         beiscsi_conn->beiscsi_conn_cid -
3835                                         phba->fw_config.iscsi_cid_start);
3836                 if (!io_task->pwrb_handle)
3837                         goto free_io_hndls;
3838         } else {
3839                 io_task->scsi_cmnd = NULL;
3840                 if ((opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGIN) {
3841                         if (!beiscsi_conn->login_in_progress) {
3842                                 spin_lock(&phba->mgmt_sgl_lock);
3843                                 io_task->psgl_handle = (struct sgl_handle *)
3844                                                 alloc_mgmt_sgl_handle(phba);
3845                                 spin_unlock(&phba->mgmt_sgl_lock);
3846                                 if (!io_task->psgl_handle)
3847                                         goto free_hndls;
3848
3849                                 beiscsi_conn->login_in_progress = 1;
3850                                 beiscsi_conn->plogin_sgl_handle =
3851                                                         io_task->psgl_handle;
3852                                 io_task->pwrb_handle =
3853                                         alloc_wrb_handle(phba,
3854                                         beiscsi_conn->beiscsi_conn_cid -
3855                                         phba->fw_config.iscsi_cid_start);
3856                                 if (!io_task->pwrb_handle)
3857                                         goto free_io_hndls;
3858                                 beiscsi_conn->plogin_wrb_handle =
3859                                                         io_task->pwrb_handle;
3860
3861                         } else {
3862                                 io_task->psgl_handle =
3863                                                 beiscsi_conn->plogin_sgl_handle;
3864                                 io_task->pwrb_handle =
3865                                                 beiscsi_conn->plogin_wrb_handle;
3866                         }
3867                 } else {
3868                         spin_lock(&phba->mgmt_sgl_lock);
3869                         io_task->psgl_handle = alloc_mgmt_sgl_handle(phba);
3870                         spin_unlock(&phba->mgmt_sgl_lock);
3871                         if (!io_task->psgl_handle)
3872                                 goto free_hndls;
3873                         io_task->pwrb_handle =
3874                                         alloc_wrb_handle(phba,
3875                                         beiscsi_conn->beiscsi_conn_cid -
3876                                         phba->fw_config.iscsi_cid_start);
3877                         if (!io_task->pwrb_handle)
3878                                 goto free_mgmt_hndls;
3879
3880                 }
3881         }
3882         itt = (itt_t) cpu_to_be32(((unsigned int)io_task->pwrb_handle->
3883                                  wrb_index << 16) | (unsigned int)
3884                                 (io_task->psgl_handle->sgl_index));
3885         io_task->pwrb_handle->pio_handle = task;
3886
3887         io_task->cmd_bhs->iscsi_hdr.itt = itt;
3888         return 0;
3889
3890 free_io_hndls:
3891         spin_lock(&phba->io_sgl_lock);
3892         free_io_sgl_handle(phba, io_task->psgl_handle);
3893         spin_unlock(&phba->io_sgl_lock);
3894         goto free_hndls;
3895 free_mgmt_hndls:
3896         spin_lock(&phba->mgmt_sgl_lock);
3897         free_mgmt_sgl_handle(phba, io_task->psgl_handle);
3898         spin_unlock(&phba->mgmt_sgl_lock);
3899 free_hndls:
3900         phwi_ctrlr = phba->phwi_ctrlr;
3901         pwrb_context = &phwi_ctrlr->wrb_context[
3902                         beiscsi_conn->beiscsi_conn_cid -
3903                         phba->fw_config.iscsi_cid_start];
3904         if (io_task->pwrb_handle)
3905                 free_wrb_handle(phba, pwrb_context, io_task->pwrb_handle);
3906         io_task->pwrb_handle = NULL;
3907         pci_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs,
3908                       io_task->bhs_pa.u.a64.address);
3909         SE_DEBUG(DBG_LVL_1, "Alloc of SGL_ICD Failed\n");
3910         return -ENOMEM;
3911 }
3912
3913 static void beiscsi_cleanup_task(struct iscsi_task *task)
3914 {
3915         struct beiscsi_io_task *io_task = task->dd_data;
3916         struct iscsi_conn *conn = task->conn;
3917         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3918         struct beiscsi_hba *phba = beiscsi_conn->phba;
3919         struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess;
3920         struct hwi_wrb_context *pwrb_context;
3921         struct hwi_controller *phwi_ctrlr;
3922
3923         phwi_ctrlr = phba->phwi_ctrlr;
3924         pwrb_context = &phwi_ctrlr->wrb_context[beiscsi_conn->beiscsi_conn_cid
3925                         - phba->fw_config.iscsi_cid_start];
3926         if (io_task->pwrb_handle) {
3927                 free_wrb_handle(phba, pwrb_context, io_task->pwrb_handle);
3928                 io_task->pwrb_handle = NULL;
3929         }
3930
3931         if (io_task->cmd_bhs) {
3932                 pci_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs,
3933                               io_task->bhs_pa.u.a64.address);
3934         }
3935
3936         if (task->sc) {
3937                 if (io_task->psgl_handle) {
3938                         spin_lock(&phba->io_sgl_lock);
3939                         free_io_sgl_handle(phba, io_task->psgl_handle);
3940                         spin_unlock(&phba->io_sgl_lock);
3941                         io_task->psgl_handle = NULL;
3942                 }
3943         } else {
3944                 if (task->hdr &&
3945                    ((task->hdr->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGIN))
3946                         return;
3947                 if (io_task->psgl_handle) {
3948                         spin_lock(&phba->mgmt_sgl_lock);
3949                         free_mgmt_sgl_handle(phba, io_task->psgl_handle);
3950                         spin_unlock(&phba->mgmt_sgl_lock);
3951                         io_task->psgl_handle = NULL;
3952                 }
3953         }
3954 }
3955
3956 static int beiscsi_iotask(struct iscsi_task *task, struct scatterlist *sg,
3957                           unsigned int num_sg, unsigned int xferlen,
3958                           unsigned int writedir)
3959 {
3960
3961         struct beiscsi_io_task *io_task = task->dd_data;
3962         struct iscsi_conn *conn = task->conn;
3963         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3964         struct beiscsi_hba *phba = beiscsi_conn->phba;
3965         struct iscsi_wrb *pwrb = NULL;
3966         unsigned int doorbell = 0;
3967
3968         pwrb = io_task->pwrb_handle->pwrb;
3969         io_task->cmd_bhs->iscsi_hdr.exp_statsn = 0;
3970         io_task->bhs_len = sizeof(struct be_cmd_bhs);
3971
3972         if (writedir) {
3973                 memset(&io_task->cmd_bhs->iscsi_data_pdu, 0, 48);
3974                 AMAP_SET_BITS(struct amap_pdu_data_out, itt,
3975                               &io_task->cmd_bhs->iscsi_data_pdu,
3976                               (unsigned int)io_task->cmd_bhs->iscsi_hdr.itt);
3977                 AMAP_SET_BITS(struct amap_pdu_data_out, opcode,
3978                               &io_task->cmd_bhs->iscsi_data_pdu,
3979                               ISCSI_OPCODE_SCSI_DATA_OUT);
3980                 AMAP_SET_BITS(struct amap_pdu_data_out, final_bit,
3981                               &io_task->cmd_bhs->iscsi_data_pdu, 1);
3982                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
3983                               INI_WR_CMD);
3984                 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1);
3985         } else {
3986                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
3987                               INI_RD_CMD);
3988                 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
3989         }
3990         memcpy(&io_task->cmd_bhs->iscsi_data_pdu.
3991                dw[offsetof(struct amap_pdu_data_out, lun) / 32],
3992                &io_task->cmd_bhs->iscsi_hdr.lun, sizeof(struct scsi_lun));
3993
3994         AMAP_SET_BITS(struct amap_iscsi_wrb, lun, pwrb,
3995                       cpu_to_be16(*(unsigned short *)&io_task->cmd_bhs->iscsi_hdr.lun));
3996         AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb, xferlen);
3997         AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb,
3998                       io_task->pwrb_handle->wrb_index);
3999         AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb,
4000                       be32_to_cpu(task->cmdsn));
4001         AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb,
4002                       io_task->psgl_handle->sgl_index);
4003
4004         hwi_write_sgl(pwrb, sg, num_sg, io_task);
4005
4006         AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb,
4007                       io_task->pwrb_handle->nxt_wrb_index);
4008         be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb));
4009
4010         doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
4011         doorbell |= (io_task->pwrb_handle->wrb_index &
4012                      DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT;
4013         doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
4014
4015         iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
4016         return 0;
4017 }
4018
4019 static int beiscsi_mtask(struct iscsi_task *task)
4020 {
4021         struct beiscsi_io_task *io_task = task->dd_data;
4022         struct iscsi_conn *conn = task->conn;
4023         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
4024         struct beiscsi_hba *phba = beiscsi_conn->phba;
4025         struct iscsi_wrb *pwrb = NULL;
4026         unsigned int doorbell = 0;
4027         unsigned int cid;
4028
4029         cid = beiscsi_conn->beiscsi_conn_cid;
4030         pwrb = io_task->pwrb_handle->pwrb;
4031         memset(pwrb, 0, sizeof(*pwrb));
4032         AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb,
4033                       be32_to_cpu(task->cmdsn));
4034         AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb,
4035                       io_task->pwrb_handle->wrb_index);
4036         AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb,
4037                       io_task->psgl_handle->sgl_index);
4038
4039         switch (task->hdr->opcode & ISCSI_OPCODE_MASK) {
4040         case ISCSI_OP_LOGIN:
4041                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4042                               TGT_DM_CMD);
4043                 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
4044                 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb, 1);
4045                 hwi_write_buffer(pwrb, task);
4046                 break;
4047         case ISCSI_OP_NOOP_OUT:
4048                 if (task->hdr->ttt != ISCSI_RESERVED_TAG) {
4049                         AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4050                                       TGT_DM_CMD);
4051                         AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt,
4052                                       pwrb, 0);
4053                         AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 1);
4054                 } else {
4055                         AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4056                                       INI_RD_CMD);
4057                         AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
4058                 }
4059                 hwi_write_buffer(pwrb, task);
4060                 break;
4061         case ISCSI_OP_TEXT:
4062                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4063                               TGT_DM_CMD);
4064                 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
4065                 hwi_write_buffer(pwrb, task);
4066                 break;
4067         case ISCSI_OP_SCSI_TMFUNC:
4068                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4069                               INI_TMF_CMD);
4070                 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
4071                 hwi_write_buffer(pwrb, task);
4072                 break;
4073         case ISCSI_OP_LOGOUT:
4074                 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
4075                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4076                               HWH_TYPE_LOGOUT);
4077                 hwi_write_buffer(pwrb, task);
4078                 break;
4079
4080         default:
4081                 SE_DEBUG(DBG_LVL_1, "opcode =%d Not supported\n",
4082                          task->hdr->opcode & ISCSI_OPCODE_MASK);
4083                 return -EINVAL;
4084         }
4085
4086         AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb,
4087                       task->data_count);
4088         AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb,
4089                       io_task->pwrb_handle->nxt_wrb_index);
4090         be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb));
4091
4092         doorbell |= cid & DB_WRB_POST_CID_MASK;
4093         doorbell |= (io_task->pwrb_handle->wrb_index &
4094                      DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT;
4095         doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
4096         iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
4097         return 0;
4098 }
4099
4100 static int beiscsi_task_xmit(struct iscsi_task *task)
4101 {
4102         struct beiscsi_io_task *io_task = task->dd_data;
4103         struct scsi_cmnd *sc = task->sc;
4104         struct scatterlist *sg;
4105         int num_sg;
4106         unsigned int  writedir = 0, xferlen = 0;
4107
4108         if (!sc)
4109                 return beiscsi_mtask(task);
4110
4111         io_task->scsi_cmnd = sc;
4112         num_sg = scsi_dma_map(sc);
4113         if (num_sg < 0) {
4114                 SE_DEBUG(DBG_LVL_1, " scsi_dma_map Failed\n")
4115                 return num_sg;
4116         }
4117         xferlen = scsi_bufflen(sc);
4118         sg = scsi_sglist(sc);
4119         if (sc->sc_data_direction == DMA_TO_DEVICE) {
4120                 writedir = 1;
4121                 SE_DEBUG(DBG_LVL_4, "task->imm_count=0x%08x\n",
4122                          task->imm_count);
4123         } else
4124                 writedir = 0;
4125         return beiscsi_iotask(task, sg, num_sg, xferlen, writedir);
4126 }
4127
4128 static void beiscsi_quiesce(struct beiscsi_hba *phba)
4129 {
4130         struct hwi_controller *phwi_ctrlr;
4131         struct hwi_context_memory *phwi_context;
4132         struct be_eq_obj *pbe_eq;
4133         unsigned int i, msix_vec;
4134         u8 *real_offset = 0;
4135         u32 value = 0;
4136
4137         phwi_ctrlr = phba->phwi_ctrlr;
4138         phwi_context = phwi_ctrlr->phwi_ctxt;
4139         hwi_disable_intr(phba);
4140         if (phba->msix_enabled) {
4141                 for (i = 0; i <= phba->num_cpus; i++) {
4142                         msix_vec = phba->msix_entries[i].vector;
4143                         free_irq(msix_vec, &phwi_context->be_eq[i]);
4144                         kfree(phba->msi_name[i]);
4145                 }
4146         } else
4147                 if (phba->pcidev->irq)
4148                         free_irq(phba->pcidev->irq, phba);
4149         pci_disable_msix(phba->pcidev);
4150         destroy_workqueue(phba->wq);
4151         if (blk_iopoll_enabled)
4152                 for (i = 0; i < phba->num_cpus; i++) {
4153                         pbe_eq = &phwi_context->be_eq[i];
4154                         blk_iopoll_disable(&pbe_eq->iopoll);
4155                 }
4156
4157         beiscsi_clean_port(phba);
4158         beiscsi_free_mem(phba);
4159         real_offset = (u8 *)phba->csr_va + MPU_EP_SEMAPHORE;
4160
4161         value = readl((void *)real_offset);
4162
4163         if (value & 0x00010000) {
4164                 value &= 0xfffeffff;
4165                 writel(value, (void *)real_offset);
4166         }
4167         beiscsi_unmap_pci_function(phba);
4168         pci_free_consistent(phba->pcidev,
4169                             phba->ctrl.mbox_mem_alloced.size,
4170                             phba->ctrl.mbox_mem_alloced.va,
4171                             phba->ctrl.mbox_mem_alloced.dma);
4172 }
4173
4174 static void beiscsi_remove(struct pci_dev *pcidev)
4175 {
4176
4177         struct beiscsi_hba *phba = NULL;
4178
4179         phba = pci_get_drvdata(pcidev);
4180         if (!phba) {
4181                 dev_err(&pcidev->dev, "beiscsi_remove called with no phba\n");
4182                 return;
4183         }
4184
4185         beiscsi_quiesce(phba);
4186         iscsi_boot_destroy_kset(phba->boot_kset);
4187         iscsi_host_remove(phba->shost);
4188         pci_dev_put(phba->pcidev);
4189         iscsi_host_free(phba->shost);
4190         pci_disable_device(pcidev);
4191 }
4192
4193 static void beiscsi_shutdown(struct pci_dev *pcidev)
4194 {
4195
4196         struct beiscsi_hba *phba = NULL;
4197
4198         phba = (struct beiscsi_hba *)pci_get_drvdata(pcidev);
4199         if (!phba) {
4200                 dev_err(&pcidev->dev, "beiscsi_shutdown called with no phba\n");
4201                 return;
4202         }
4203
4204         beiscsi_quiesce(phba);
4205         pci_disable_device(pcidev);
4206 }
4207
4208 static void beiscsi_msix_enable(struct beiscsi_hba *phba)
4209 {
4210         int i, status;
4211
4212         for (i = 0; i <= phba->num_cpus; i++)
4213                 phba->msix_entries[i].entry = i;
4214
4215         status = pci_enable_msix(phba->pcidev, phba->msix_entries,
4216                                  (phba->num_cpus + 1));
4217         if (!status)
4218                 phba->msix_enabled = true;
4219
4220         return;
4221 }
4222
4223 static int __devinit beiscsi_dev_probe(struct pci_dev *pcidev,
4224                                 const struct pci_device_id *id)
4225 {
4226         struct beiscsi_hba *phba = NULL;
4227         struct hwi_controller *phwi_ctrlr;
4228         struct hwi_context_memory *phwi_context;
4229         struct be_eq_obj *pbe_eq;
4230         int ret, num_cpus, i;
4231         u8 *real_offset = 0;
4232         u32 value = 0;
4233
4234         ret = beiscsi_enable_pci(pcidev);
4235         if (ret < 0) {
4236                 dev_err(&pcidev->dev, "beiscsi_dev_probe-"
4237                         " Failed to enable pci device\n");
4238                 return ret;
4239         }
4240
4241         phba = beiscsi_hba_alloc(pcidev);
4242         if (!phba) {
4243                 dev_err(&pcidev->dev, "beiscsi_dev_probe-"
4244                         " Failed in beiscsi_hba_alloc\n");
4245                 goto disable_pci;
4246         }
4247
4248         switch (pcidev->device) {
4249         case BE_DEVICE_ID1:
4250         case OC_DEVICE_ID1:
4251         case OC_DEVICE_ID2:
4252                 phba->generation = BE_GEN2;
4253                 break;
4254         case BE_DEVICE_ID2:
4255         case OC_DEVICE_ID3:
4256                 phba->generation = BE_GEN3;
4257                 break;
4258         default:
4259                 phba->generation = 0;
4260         }
4261
4262         if (enable_msix)
4263                 num_cpus = find_num_cpus();
4264         else
4265                 num_cpus = 1;
4266         phba->num_cpus = num_cpus;
4267         SE_DEBUG(DBG_LVL_8, "num_cpus = %d\n", phba->num_cpus);
4268
4269         if (enable_msix)
4270                 beiscsi_msix_enable(phba);
4271         ret = be_ctrl_init(phba, pcidev);
4272         if (ret) {
4273                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
4274                                 "Failed in be_ctrl_init\n");
4275                 goto hba_free;
4276         }
4277
4278         if (!num_hba) {
4279                 real_offset = (u8 *)phba->csr_va + MPU_EP_SEMAPHORE;
4280                 value = readl((void *)real_offset);
4281                 if (value & 0x00010000) {
4282                         gcrashmode++;
4283                         shost_printk(KERN_ERR, phba->shost,
4284                                 "Loading Driver in crashdump mode\n");
4285                         ret = beiscsi_cmd_reset_function(phba);
4286                         if (ret) {
4287                                 shost_printk(KERN_ERR, phba->shost,
4288                                         "Reset Failed. Aborting Crashdump\n");
4289                                 goto hba_free;
4290                         }
4291                         ret = be_chk_reset_complete(phba);
4292                         if (ret) {
4293                                 shost_printk(KERN_ERR, phba->shost,
4294                                         "Failed to get out of reset."
4295                                         "Aborting Crashdump\n");
4296                                 goto hba_free;
4297                         }
4298                 } else {
4299                         value |= 0x00010000;
4300                         writel(value, (void *)real_offset);
4301                         num_hba++;
4302                 }
4303         }
4304
4305         spin_lock_init(&phba->io_sgl_lock);
4306         spin_lock_init(&phba->mgmt_sgl_lock);
4307         spin_lock_init(&phba->isr_lock);
4308         ret = mgmt_get_fw_config(&phba->ctrl, phba);
4309         if (ret != 0) {
4310                 shost_printk(KERN_ERR, phba->shost,
4311                              "Error getting fw config\n");
4312                 goto free_port;
4313         }
4314         phba->shost->max_id = phba->fw_config.iscsi_cid_count;
4315         beiscsi_get_params(phba);
4316         phba->shost->can_queue = phba->params.ios_per_ctrl;
4317         ret = beiscsi_init_port(phba);
4318         if (ret < 0) {
4319                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
4320                              "Failed in beiscsi_init_port\n");
4321                 goto free_port;
4322         }
4323
4324         for (i = 0; i < MAX_MCC_CMD ; i++) {
4325                 init_waitqueue_head(&phba->ctrl.mcc_wait[i + 1]);
4326                 phba->ctrl.mcc_tag[i] = i + 1;
4327                 phba->ctrl.mcc_numtag[i + 1] = 0;
4328                 phba->ctrl.mcc_tag_available++;
4329         }
4330
4331         phba->ctrl.mcc_alloc_index = phba->ctrl.mcc_free_index = 0;
4332
4333         snprintf(phba->wq_name, sizeof(phba->wq_name), "beiscsi_q_irq%u",
4334                  phba->shost->host_no);
4335         phba->wq = alloc_workqueue(phba->wq_name, WQ_MEM_RECLAIM, 1);
4336         if (!phba->wq) {
4337                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
4338                                 "Failed to allocate work queue\n");
4339                 goto free_twq;
4340         }
4341
4342         INIT_WORK(&phba->work_cqs, beiscsi_process_all_cqs);
4343
4344         phwi_ctrlr = phba->phwi_ctrlr;
4345         phwi_context = phwi_ctrlr->phwi_ctxt;
4346         if (blk_iopoll_enabled) {
4347                 for (i = 0; i < phba->num_cpus; i++) {
4348                         pbe_eq = &phwi_context->be_eq[i];
4349                         blk_iopoll_init(&pbe_eq->iopoll, be_iopoll_budget,
4350                                         be_iopoll);
4351                         blk_iopoll_enable(&pbe_eq->iopoll);
4352                 }
4353         }
4354         ret = beiscsi_init_irqs(phba);
4355         if (ret < 0) {
4356                 shost_printk(KERN_ERR, phba->shost, "beiscsi_dev_probe-"
4357                              "Failed to beiscsi_init_irqs\n");
4358                 goto free_blkenbld;
4359         }
4360         hwi_enable_intr(phba);
4361
4362         if (beiscsi_setup_boot_info(phba))
4363                 /*
4364                  * log error but continue, because we may not be using
4365                  * iscsi boot.
4366                  */
4367                 shost_printk(KERN_ERR, phba->shost, "Could not set up "
4368                              "iSCSI boot info.");
4369
4370         SE_DEBUG(DBG_LVL_8, "\n\n\n SUCCESS - DRIVER LOADED\n\n\n");
4371         return 0;
4372
4373 free_blkenbld:
4374         destroy_workqueue(phba->wq);
4375         if (blk_iopoll_enabled)
4376                 for (i = 0; i < phba->num_cpus; i++) {
4377                         pbe_eq = &phwi_context->be_eq[i];
4378                         blk_iopoll_disable(&pbe_eq->iopoll);
4379                 }
4380 free_twq:
4381         beiscsi_clean_port(phba);
4382         beiscsi_free_mem(phba);
4383 free_port:
4384         real_offset = (u8 *)phba->csr_va + MPU_EP_SEMAPHORE;
4385
4386         value = readl((void *)real_offset);
4387
4388         if (value & 0x00010000) {
4389                 value &= 0xfffeffff;
4390                 writel(value, (void *)real_offset);
4391         }
4392
4393         pci_free_consistent(phba->pcidev,
4394                             phba->ctrl.mbox_mem_alloced.size,
4395                             phba->ctrl.mbox_mem_alloced.va,
4396                            phba->ctrl.mbox_mem_alloced.dma);
4397         beiscsi_unmap_pci_function(phba);
4398 hba_free:
4399         if (phba->msix_enabled)
4400                 pci_disable_msix(phba->pcidev);
4401         iscsi_host_remove(phba->shost);
4402         pci_dev_put(phba->pcidev);
4403         iscsi_host_free(phba->shost);
4404 disable_pci:
4405         pci_disable_device(pcidev);
4406         return ret;
4407 }
4408
4409 struct iscsi_transport beiscsi_iscsi_transport = {
4410         .owner = THIS_MODULE,
4411         .name = DRV_NAME,
4412         .caps = CAP_RECOVERY_L0 | CAP_HDRDGST | CAP_TEXT_NEGO |
4413                 CAP_MULTI_R2T | CAP_DATADGST | CAP_DATA_PATH_OFFLOAD,
4414         .create_session = beiscsi_session_create,
4415         .destroy_session = beiscsi_session_destroy,
4416         .create_conn = beiscsi_conn_create,
4417         .bind_conn = beiscsi_conn_bind,
4418         .destroy_conn = iscsi_conn_teardown,
4419         .attr_is_visible = be2iscsi_attr_is_visible,
4420         .set_param = beiscsi_set_param,
4421         .get_conn_param = iscsi_conn_get_param,
4422         .get_session_param = iscsi_session_get_param,
4423         .get_host_param = beiscsi_get_host_param,
4424         .start_conn = beiscsi_conn_start,
4425         .stop_conn = iscsi_conn_stop,
4426         .send_pdu = iscsi_conn_send_pdu,
4427         .xmit_task = beiscsi_task_xmit,
4428         .cleanup_task = beiscsi_cleanup_task,
4429         .alloc_pdu = beiscsi_alloc_pdu,
4430         .parse_pdu_itt = beiscsi_parse_pdu,
4431         .get_stats = beiscsi_conn_get_stats,
4432         .get_ep_param = beiscsi_ep_get_param,
4433         .ep_connect = beiscsi_ep_connect,
4434         .ep_poll = beiscsi_ep_poll,
4435         .ep_disconnect = beiscsi_ep_disconnect,
4436         .session_recovery_timedout = iscsi_session_recovery_timedout,
4437 };
4438
4439 static struct pci_driver beiscsi_pci_driver = {
4440         .name = DRV_NAME,
4441         .probe = beiscsi_dev_probe,
4442         .remove = beiscsi_remove,
4443         .shutdown = beiscsi_shutdown,
4444         .id_table = beiscsi_pci_id_table
4445 };
4446
4447
4448 static int __init beiscsi_module_init(void)
4449 {
4450         int ret;
4451
4452         beiscsi_scsi_transport =
4453                         iscsi_register_transport(&beiscsi_iscsi_transport);
4454         if (!beiscsi_scsi_transport) {
4455                 SE_DEBUG(DBG_LVL_1,
4456                          "beiscsi_module_init - Unable to  register beiscsi"
4457                          "transport.\n");
4458                 return -ENOMEM;
4459         }
4460         SE_DEBUG(DBG_LVL_8, "In beiscsi_module_init, tt=%p\n",
4461                  &beiscsi_iscsi_transport);
4462
4463         ret = pci_register_driver(&beiscsi_pci_driver);
4464         if (ret) {
4465                 SE_DEBUG(DBG_LVL_1,
4466                          "beiscsi_module_init - Unable to  register"
4467                          "beiscsi pci driver.\n");
4468                 goto unregister_iscsi_transport;
4469         }
4470         return 0;
4471
4472 unregister_iscsi_transport:
4473         iscsi_unregister_transport(&beiscsi_iscsi_transport);
4474         return ret;
4475 }
4476
4477 static void __exit beiscsi_module_exit(void)
4478 {
4479         pci_unregister_driver(&beiscsi_pci_driver);
4480         iscsi_unregister_transport(&beiscsi_iscsi_transport);
4481 }
4482
4483 module_init(beiscsi_module_init);
4484 module_exit(beiscsi_module_exit);