drivers/infiniband/hw/nes/nes_hw.c

   1 /*
   2  * Copyright (c) 2006 - 2011 Intel Corporation.  All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  *
  32  */
  33
  34 #include <linux/module.h>
  35 #include <linux/moduleparam.h>
  36 #include <linux/netdevice.h>
  37 #include <linux/etherdevice.h>
  38 #include <linux/ip.h>
  39 #include <linux/tcp.h>
  40 #include <linux/if_vlan.h>
  41 #include <linux/inet_lro.h>
  42 #include <linux/slab.h>
  43
  44 #include "nes.h"
  45
  46 static unsigned int nes_lro_max_aggr = NES_LRO_MAX_AGGR;
  47 module_param(nes_lro_max_aggr, uint, 0444);
  48 MODULE_PARM_DESC(nes_lro_max_aggr, "NIC LRO max packet aggregation");
  49
  50 static int wide_ppm_offset;
  51 module_param(wide_ppm_offset, int, 0644);
  52 MODULE_PARM_DESC(wide_ppm_offset, "Increase CX4 interface clock ppm offset, 0=100ppm (default), 1=300ppm");
  53
  54 static u32 crit_err_count;
  55 u32 int_mod_timer_init;
  56 u32 int_mod_cq_depth_256;
  57 u32 int_mod_cq_depth_128;
  58 u32 int_mod_cq_depth_32;
  59 u32 int_mod_cq_depth_24;
  60 u32 int_mod_cq_depth_16;
  61 u32 int_mod_cq_depth_4;
  62 u32 int_mod_cq_depth_1;
  63 static const u8 nes_max_critical_error_count = 100;
  64 #include "nes_cm.h"
  65
  66 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq);
  67 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count);
  68 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
  69                                 struct nes_adapter *nesadapter, u8  OneG_Mode);
  70 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq);
  71 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq);
  72 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq);
  73 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
  74                                    struct nes_hw_aeqe *aeqe);
  75 static void process_critical_error(struct nes_device *nesdev);
  76 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number);
  77 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode);
  78 static void nes_terminate_timeout(unsigned long context);
  79 static void nes_terminate_start_timer(struct nes_qp *nesqp);
  80
  81 #ifdef CONFIG_INFINIBAND_NES_DEBUG
  82 static unsigned char *nes_iwarp_state_str[] = {
  83         "Non-Existent",
  84         "Idle",
  85         "RTS",
  86         "Closing",
  87         "RSVD1",
  88         "Terminate",
  89         "Error",
  90         "RSVD2",
  91 };
  92
  93 static unsigned char *nes_tcp_state_str[] = {
  94         "Non-Existent",
  95         "Closed",
  96         "Listen",
  97         "SYN Sent",
  98         "SYN Rcvd",
  99         "Established",
 100         "Close Wait",
 101         "FIN Wait 1",
 102         "Closing",
 103         "Last Ack",
 104         "FIN Wait 2",
 105         "Time Wait",
 106         "RSVD1",
 107         "RSVD2",
 108         "RSVD3",
 109         "RSVD4",
 110 };
 111 #endif
 112
 113 static inline void print_ip(struct nes_cm_node *cm_node)
 114 {
 115         unsigned char *rem_addr;
 116         if (cm_node) {
 117                 rem_addr = (unsigned char *)&cm_node->rem_addr;
 118                 printk(KERN_ERR PFX "Remote IP addr: %pI4\n", rem_addr);
 119         }
 120 }
 121
 122 /**
 123  * nes_nic_init_timer_defaults
 124  */
 125 void  nes_nic_init_timer_defaults(struct nes_device *nesdev, u8 jumbomode)
 126 {
 127         unsigned long flags;
 128         struct nes_adapter *nesadapter = nesdev->nesadapter;
 129         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 130
 131         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 132
 133         shared_timer->timer_in_use_min = NES_NIC_FAST_TIMER_LOW;
 134         shared_timer->timer_in_use_max = NES_NIC_FAST_TIMER_HIGH;
 135         if (jumbomode) {
 136                 shared_timer->threshold_low    = DEFAULT_JUMBO_NES_QL_LOW;
 137                 shared_timer->threshold_target = DEFAULT_JUMBO_NES_QL_TARGET;
 138                 shared_timer->threshold_high   = DEFAULT_JUMBO_NES_QL_HIGH;
 139         } else {
 140                 shared_timer->threshold_low    = DEFAULT_NES_QL_LOW;
 141                 shared_timer->threshold_target = DEFAULT_NES_QL_TARGET;
 142                 shared_timer->threshold_high   = DEFAULT_NES_QL_HIGH;
 143         }
 144
 145         /* todo use netdev->mtu to set thresholds */
 146         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 147 }
 148
 149
 150 /**
 151  * nes_nic_init_timer
 152  */
 153 static void  nes_nic_init_timer(struct nes_device *nesdev)
 154 {
 155         unsigned long flags;
 156         struct nes_adapter *nesadapter = nesdev->nesadapter;
 157         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 158
 159         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 160
 161         if (shared_timer->timer_in_use_old == 0) {
 162                 nesdev->deepcq_count = 0;
 163                 shared_timer->timer_direction_upward = 0;
 164                 shared_timer->timer_direction_downward = 0;
 165                 shared_timer->timer_in_use = NES_NIC_FAST_TIMER;
 166                 shared_timer->timer_in_use_old = 0;
 167
 168         }
 169         if (shared_timer->timer_in_use != shared_timer->timer_in_use_old) {
 170                 shared_timer->timer_in_use_old = shared_timer->timer_in_use;
 171                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL,
 172                         0x80000000 | ((u32)(shared_timer->timer_in_use*8)));
 173         }
 174         /* todo use netdev->mtu to set thresholds */
 175         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 176 }
 177
 178
 179 /**
 180  * nes_nic_tune_timer
 181  */
 182 static void nes_nic_tune_timer(struct nes_device *nesdev)
 183 {
 184         unsigned long flags;
 185         struct nes_adapter *nesadapter = nesdev->nesadapter;
 186         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 187         u16 cq_count = nesdev->currcq_count;
 188
 189         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 190
 191         if (shared_timer->cq_count_old <= cq_count)
 192                 shared_timer->cq_direction_downward = 0;
 193         else
 194                 shared_timer->cq_direction_downward++;
 195         shared_timer->cq_count_old = cq_count;
 196         if (shared_timer->cq_direction_downward > NES_NIC_CQ_DOWNWARD_TREND) {
 197                 if (cq_count <= shared_timer->threshold_low &&
 198                     shared_timer->threshold_low > 4) {
 199                         shared_timer->threshold_low = shared_timer->threshold_low/2;
 200                         shared_timer->cq_direction_downward=0;
 201                         nesdev->currcq_count = 0;
 202                         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 203                         return;
 204                 }
 205         }
 206
 207         if (cq_count > 1) {
 208                 nesdev->deepcq_count += cq_count;
 209                 if (cq_count <= shared_timer->threshold_low) {       /* increase timer gently */
 210                         shared_timer->timer_direction_upward++;
 211                         shared_timer->timer_direction_downward = 0;
 212                 } else if (cq_count <= shared_timer->threshold_target) { /* balanced */
 213                         shared_timer->timer_direction_upward = 0;
 214                         shared_timer->timer_direction_downward = 0;
 215                 } else if (cq_count <= shared_timer->threshold_high) {  /* decrease timer gently */
 216                         shared_timer->timer_direction_downward++;
 217                         shared_timer->timer_direction_upward = 0;
 218                 } else if (cq_count <= (shared_timer->threshold_high) * 2) {
 219                         shared_timer->timer_in_use -= 2;
 220                         shared_timer->timer_direction_upward = 0;
 221                         shared_timer->timer_direction_downward++;
 222                 } else {
 223                         shared_timer->timer_in_use -= 4;
 224                         shared_timer->timer_direction_upward = 0;
 225                         shared_timer->timer_direction_downward++;
 226                 }
 227
 228                 if (shared_timer->timer_direction_upward > 3 ) {  /* using history */
 229                         shared_timer->timer_in_use += 3;
 230                         shared_timer->timer_direction_upward = 0;
 231                         shared_timer->timer_direction_downward = 0;
 232                 }
 233                 if (shared_timer->timer_direction_downward > 5) { /* using history */
 234                         shared_timer->timer_in_use -= 4 ;
 235                         shared_timer->timer_direction_downward = 0;
 236                         shared_timer->timer_direction_upward = 0;
 237                 }
 238         }
 239
 240         /* boundary checking */
 241         if (shared_timer->timer_in_use > shared_timer->threshold_high)
 242                 shared_timer->timer_in_use = shared_timer->threshold_high;
 243         else if (shared_timer->timer_in_use < shared_timer->threshold_low)
 244                 shared_timer->timer_in_use = shared_timer->threshold_low;
 245
 246         nesdev->currcq_count = 0;
 247
 248         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 249 }
 250
 251
 252 /**
 253  * nes_init_adapter - initialize adapter
 254  */
 255 struct nes_adapter *nes_init_adapter(struct nes_device *nesdev, u8 hw_rev) {
 256         struct nes_adapter *nesadapter = NULL;
 257         unsigned long num_pds;
 258         u32 u32temp;
 259         u32 port_count;
 260         u16 max_rq_wrs;
 261         u16 max_sq_wrs;
 262         u32 max_mr;
 263         u32 max_256pbl;
 264         u32 max_4kpbl;
 265         u32 max_qp;
 266         u32 max_irrq;
 267         u32 max_cq;
 268         u32 hte_index_mask;
 269         u32 adapter_size;
 270         u32 arp_table_size;
 271         u16 vendor_id;
 272         u16 device_id;
 273         u8  OneG_Mode;
 274         u8  func_index;
 275
 276         /* search the list of existing adapters */
 277         list_for_each_entry(nesadapter, &nes_adapter_list, list) {
 278                 nes_debug(NES_DBG_INIT, "Searching Adapter list for PCI devfn = 0x%X,"
 279                                 " adapter PCI slot/bus = %u/%u, pci devices PCI slot/bus = %u/%u, .\n",
 280                                 nesdev->pcidev->devfn,
 281                                 PCI_SLOT(nesadapter->devfn),
 282                                 nesadapter->bus_number,
 283                                 PCI_SLOT(nesdev->pcidev->devfn),
 284                                 nesdev->pcidev->bus->number );
 285                 if ((PCI_SLOT(nesadapter->devfn) == PCI_SLOT(nesdev->pcidev->devfn)) &&
 286                                 (nesadapter->bus_number == nesdev->pcidev->bus->number)) {
 287                         nesadapter->ref_count++;
 288                         return nesadapter;
 289                 }
 290         }
 291
 292         /* no adapter found */
 293         num_pds = pci_resource_len(nesdev->pcidev, BAR_1) >> PAGE_SHIFT;
 294         if ((hw_rev != NE020_REV) && (hw_rev != NE020_REV1)) {
 295                 nes_debug(NES_DBG_INIT, "NE020 driver detected unknown hardware revision 0x%x\n",
 296                                 hw_rev);
 297                 return NULL;
 298         }
 299
 300         nes_debug(NES_DBG_INIT, "Determine Soft Reset, QP_control=0x%x, CPU0=0x%x, CPU1=0x%x, CPU2=0x%x\n",
 301                         nes_read_indexed(nesdev, NES_IDX_QP_CONTROL + PCI_FUNC(nesdev->pcidev->devfn) * 8),
 302                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS),
 303                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 4),
 304                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 8));
 305
 306         nes_debug(NES_DBG_INIT, "Reset and init NE020\n");
 307
 308
 309         if ((port_count = nes_reset_adapter_ne020(nesdev, &OneG_Mode)) == 0)
 310                 return NULL;
 311
 312         max_qp = nes_read_indexed(nesdev, NES_IDX_QP_CTX_SIZE);
 313         nes_debug(NES_DBG_INIT, "QP_CTX_SIZE=%u\n", max_qp);
 314
 315         u32temp = nes_read_indexed(nesdev, NES_IDX_QUAD_HASH_TABLE_SIZE);
 316         if (max_qp > ((u32)1 << (u32temp & 0x001f))) {
 317                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to hash table size = 0x%08X\n",
 318                                 max_qp, u32temp);
 319                 max_qp = (u32)1 << (u32temp & 0x001f);
 320         }
 321
 322         hte_index_mask = ((u32)1 << ((u32temp & 0x001f)+1))-1;
 323         nes_debug(NES_DBG_INIT, "Max QP = %u, hte_index_mask = 0x%08X.\n",
 324                         max_qp, hte_index_mask);
 325
 326         u32temp = nes_read_indexed(nesdev, NES_IDX_IRRQ_COUNT);
 327
 328         max_irrq = 1 << (u32temp & 0x001f);
 329
 330         if (max_qp > max_irrq) {
 331                 max_qp = max_irrq;
 332                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to Available Q1s.\n",
 333                                 max_qp);
 334         }
 335
 336         /* there should be no reason to allocate more pds than qps */
 337         if (num_pds > max_qp)
 338                 num_pds = max_qp;
 339
 340         u32temp = nes_read_indexed(nesdev, NES_IDX_MRT_SIZE);
 341         max_mr = (u32)8192 << (u32temp & 0x7);
 342
 343         u32temp = nes_read_indexed(nesdev, NES_IDX_PBL_REGION_SIZE);
 344         max_256pbl = (u32)1 << (u32temp & 0x0000001f);
 345         max_4kpbl = (u32)1 << ((u32temp >> 16) & 0x0000001f);
 346         max_cq = nes_read_indexed(nesdev, NES_IDX_CQ_CTX_SIZE);
 347
 348         u32temp = nes_read_indexed(nesdev, NES_IDX_ARP_CACHE_SIZE);
 349         arp_table_size = 1 << u32temp;
 350
 351         adapter_size = (sizeof(struct nes_adapter) +
 352                         (sizeof(unsigned long)-1)) & (~(sizeof(unsigned long)-1));
 353         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_qp);
 354         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_mr);
 355         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_cq);
 356         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(num_pds);
 357         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(arp_table_size);
 358         adapter_size += sizeof(struct nes_qp **) * max_qp;
 359
 360         /* allocate a new adapter struct */
 361         nesadapter = kzalloc(adapter_size, GFP_KERNEL);
 362         if (nesadapter == NULL) {
 363                 return NULL;
 364         }
 365
 366         nes_debug(NES_DBG_INIT, "Allocating new nesadapter @ %p, size = %u (actual size = %u).\n",
 367                         nesadapter, (u32)sizeof(struct nes_adapter), adapter_size);
 368
 369         if (nes_read_eeprom_values(nesdev, nesadapter)) {
 370                 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
 371                 kfree(nesadapter);
 372                 return NULL;
 373         }
 374
 375         nesadapter->vendor_id = (((u32) nesadapter->mac_addr_high) << 8) |
 376                                 (nesadapter->mac_addr_low >> 24);
 377
 378         pci_bus_read_config_word(nesdev->pcidev->bus, nesdev->pcidev->devfn,
 379                                  PCI_DEVICE_ID, &device_id);
 380         nesadapter->vendor_part_id = device_id;
 381
 382         if (nes_init_serdes(nesdev, hw_rev, port_count, nesadapter,
 383                                                         OneG_Mode)) {
 384                 kfree(nesadapter);
 385                 return NULL;
 386         }
 387         nes_init_csr_ne020(nesdev, hw_rev, port_count);
 388
 389         memset(nesadapter->pft_mcast_map, 255,
 390                sizeof nesadapter->pft_mcast_map);
 391
 392         /* populate the new nesadapter */
 393         nesadapter->devfn = nesdev->pcidev->devfn;
 394         nesadapter->bus_number = nesdev->pcidev->bus->number;
 395         nesadapter->ref_count = 1;
 396         nesadapter->timer_int_req = 0xffff0000;
 397         nesadapter->OneG_Mode = OneG_Mode;
 398         nesadapter->doorbell_start = nesdev->doorbell_region;
 399
 400         /* nesadapter->tick_delta = clk_divisor; */
 401         nesadapter->hw_rev = hw_rev;
 402         nesadapter->port_count = port_count;
 403
 404         nesadapter->max_qp = max_qp;
 405         nesadapter->hte_index_mask = hte_index_mask;
 406         nesadapter->max_irrq = max_irrq;
 407         nesadapter->max_mr = max_mr;
 408         nesadapter->max_256pbl = max_256pbl - 1;
 409         nesadapter->max_4kpbl = max_4kpbl - 1;
 410         nesadapter->max_cq = max_cq;
 411         nesadapter->free_256pbl = max_256pbl - 1;
 412         nesadapter->free_4kpbl = max_4kpbl - 1;
 413         nesadapter->max_pd = num_pds;
 414         nesadapter->arp_table_size = arp_table_size;
 415
 416         nesadapter->et_pkt_rate_low = NES_TIMER_ENABLE_LIMIT;
 417         if (nes_drv_opt & NES_DRV_OPT_DISABLE_INT_MOD) {
 418                 nesadapter->et_use_adaptive_rx_coalesce = 0;
 419                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT;
 420                 nesadapter->et_rx_coalesce_usecs_irq = interrupt_mod_interval;
 421         } else {
 422                 nesadapter->et_use_adaptive_rx_coalesce = 1;
 423                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT_DYNAMIC;
 424                 nesadapter->et_rx_coalesce_usecs_irq = 0;
 425                 printk(PFX "%s: Using Adaptive Interrupt Moderation\n", __func__);
 426         }
 427         /* Setup and enable the periodic timer */
 428         if (nesadapter->et_rx_coalesce_usecs_irq)
 429                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x80000000 |
 430                                 ((u32)(nesadapter->et_rx_coalesce_usecs_irq * 8)));
 431         else
 432                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x00000000);
 433
 434         nesadapter->base_pd = 1;
 435
 436         nesadapter->device_cap_flags = IB_DEVICE_LOCAL_DMA_LKEY |
 437                                        IB_DEVICE_MEM_WINDOW |
 438                                        IB_DEVICE_MEM_MGT_EXTENSIONS;
 439
 440         nesadapter->allocated_qps = (unsigned long *)&(((unsigned char *)nesadapter)
 441                         [(sizeof(struct nes_adapter)+(sizeof(unsigned long)-1))&(~(sizeof(unsigned long)-1))]);
 442         nesadapter->allocated_cqs = &nesadapter->allocated_qps[BITS_TO_LONGS(max_qp)];
 443         nesadapter->allocated_mrs = &nesadapter->allocated_cqs[BITS_TO_LONGS(max_cq)];
 444         nesadapter->allocated_pds = &nesadapter->allocated_mrs[BITS_TO_LONGS(max_mr)];
 445         nesadapter->allocated_arps = &nesadapter->allocated_pds[BITS_TO_LONGS(num_pds)];
 446         nesadapter->qp_table = (struct nes_qp **)(&nesadapter->allocated_arps[BITS_TO_LONGS(arp_table_size)]);
 447
 448
 449         /* mark the usual suspect QPs, MR and CQs as in use */
 450         for (u32temp = 0; u32temp < NES_FIRST_QPN; u32temp++) {
 451                 set_bit(u32temp, nesadapter->allocated_qps);
 452                 set_bit(u32temp, nesadapter->allocated_cqs);
 453         }
 454         set_bit(0, nesadapter->allocated_mrs);
 455
 456         for (u32temp = 0; u32temp < 20; u32temp++)
 457                 set_bit(u32temp, nesadapter->allocated_pds);
 458         u32temp = nes_read_indexed(nesdev, NES_IDX_QP_MAX_CFG_SIZES);
 459
 460         max_rq_wrs = ((u32temp >> 8) & 3);
 461         switch (max_rq_wrs) {
 462                 case 0:
 463                         max_rq_wrs = 4;
 464                         break;
 465                 case 1:
 466                         max_rq_wrs = 16;
 467                         break;
 468                 case 2:
 469                         max_rq_wrs = 32;
 470                         break;
 471                 case 3:
 472                         max_rq_wrs = 512;
 473                         break;
 474         }
 475
 476         max_sq_wrs = (u32temp & 3);
 477         switch (max_sq_wrs) {
 478                 case 0:
 479                         max_sq_wrs = 4;
 480                         break;
 481                 case 1:
 482                         max_sq_wrs = 16;
 483                         break;
 484                 case 2:
 485                         max_sq_wrs = 32;
 486                         break;
 487                 case 3:
 488                         max_sq_wrs = 512;
 489                         break;
 490         }
 491         nesadapter->max_qp_wr = min(max_rq_wrs, max_sq_wrs);
 492         nesadapter->max_irrq_wr = (u32temp >> 16) & 3;
 493
 494         nesadapter->max_sge = 4;
 495         nesadapter->max_cqe = 32766;
 496
 497         if (nes_read_eeprom_values(nesdev, nesadapter)) {
 498                 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
 499                 kfree(nesadapter);
 500                 return NULL;
 501         }
 502
 503         u32temp = nes_read_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG);
 504         nes_write_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG,
 505                         (u32temp & 0xff000000) | (nesadapter->tcp_timer_core_clk_divisor & 0x00ffffff));
 506
 507         /* setup port configuration */
 508         if (nesadapter->port_count == 1) {
 509                 nesadapter->log_port = 0x00000000;
 510                 if (nes_drv_opt & NES_DRV_OPT_DUAL_LOGICAL_PORT)
 511                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000002);
 512                 else
 513                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 514         } else {
 515                 if (nesadapter->phy_type[0] == NES_PHY_TYPE_PUMA_1G) {
 516                         nesadapter->log_port = 0x000000D8;
 517                 } else {
 518                         if (nesadapter->port_count == 2)
 519                                 nesadapter->log_port = 0x00000044;
 520                         else
 521                                 nesadapter->log_port = 0x000000e4;
 522                 }
 523                 nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 524         }
 525
 526         nes_write_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT,
 527                                                 nesadapter->log_port);
 528         nes_debug(NES_DBG_INIT, "Probe time, LOG2PHY=%u\n",
 529                         nes_read_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT));
 530
 531         spin_lock_init(&nesadapter->resource_lock);
 532         spin_lock_init(&nesadapter->phy_lock);
 533         spin_lock_init(&nesadapter->pbl_lock);
 534         spin_lock_init(&nesadapter->periodic_timer_lock);
 535
 536         INIT_LIST_HEAD(&nesadapter->nesvnic_list[0]);
 537         INIT_LIST_HEAD(&nesadapter->nesvnic_list[1]);
 538         INIT_LIST_HEAD(&nesadapter->nesvnic_list[2]);
 539         INIT_LIST_HEAD(&nesadapter->nesvnic_list[3]);
 540
 541         if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
 542                 u32 pcs_control_status0, pcs_control_status1;
 543                 u32 reset_value;
 544                 u32 i = 0;
 545                 u32 int_cnt = 0;
 546                 u32 ext_cnt = 0;
 547                 unsigned long flags;
 548                 u32 j = 0;
 549
 550                 pcs_control_status0 = nes_read_indexed(nesdev,
 551                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 552                 pcs_control_status1 = nes_read_indexed(nesdev,
 553                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 554
 555                 for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 556                         pcs_control_status0 = nes_read_indexed(nesdev,
 557                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 558                         pcs_control_status1 = nes_read_indexed(nesdev,
 559                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 560                         if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 561                             || (0x0F000100 == (pcs_control_status1 & 0x0F000100)))
 562                                 int_cnt++;
 563                         msleep(1);
 564                 }
 565                 if (int_cnt > 1) {
 566                         spin_lock_irqsave(&nesadapter->phy_lock, flags);
 567                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
 568                         mh_detected++;
 569                         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 570                         reset_value |= 0x0000003d;
 571                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 572
 573                         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 574                                 & 0x00000040) != 0x00000040) && (j++ < 5000));
 575                         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 576
 577                         pcs_control_status0 = nes_read_indexed(nesdev,
 578                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 579                         pcs_control_status1 = nes_read_indexed(nesdev,
 580                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 581
 582                         for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 583                                 pcs_control_status0 = nes_read_indexed(nesdev,
 584                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 585                                 pcs_control_status1 = nes_read_indexed(nesdev,
 586                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 587                                 if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 588                                         || (0x0F000100 == (pcs_control_status1 & 0x0F000100))) {
 589                                         if (++ext_cnt > int_cnt) {
 590                                                 spin_lock_irqsave(&nesadapter->phy_lock, flags);
 591                                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1,
 592                                                                 0x0000F088);
 593                                                 mh_detected++;
 594                                                 reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 595                                                 reset_value |= 0x0000003d;
 596                                                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 597
 598                                                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 599                                                         & 0x00000040) != 0x00000040) && (j++ < 5000));
 600                                                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 601                                                 break;
 602                                         }
 603                                 }
 604                                 msleep(1);
 605                         }
 606                 }
 607         }
 608
 609         if (nesadapter->hw_rev == NE020_REV) {
 610                 init_timer(&nesadapter->mh_timer);
 611                 nesadapter->mh_timer.function = nes_mh_fix;
 612                 nesadapter->mh_timer.expires = jiffies + (HZ/5);  /* 1 second */
 613                 nesadapter->mh_timer.data = (unsigned long)nesdev;
 614                 add_timer(&nesadapter->mh_timer);
 615         } else {
 616                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, 0x0f000000);
 617         }
 618
 619         init_timer(&nesadapter->lc_timer);
 620         nesadapter->lc_timer.function = nes_clc;
 621         nesadapter->lc_timer.expires = jiffies + 3600 * HZ;  /* 1 hour */
 622         nesadapter->lc_timer.data = (unsigned long)nesdev;
 623         add_timer(&nesadapter->lc_timer);
 624
 625         list_add_tail(&nesadapter->list, &nes_adapter_list);
 626
 627         for (func_index = 0; func_index < 8; func_index++) {
 628                 pci_bus_read_config_word(nesdev->pcidev->bus,
 629                                         PCI_DEVFN(PCI_SLOT(nesdev->pcidev->devfn),
 630                                         func_index), 0, &vendor_id);
 631                 if (vendor_id == 0xffff)
 632                         break;
 633         }
 634         nes_debug(NES_DBG_INIT, "%s %d functions found for %s.\n", __func__,
 635                 func_index, pci_name(nesdev->pcidev));
 636         nesadapter->adapter_fcn_count = func_index;
 637
 638         return nesadapter;
 639 }
 640
 641
 642 /**
 643  * nes_reset_adapter_ne020
 644  */
 645 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode)
 646 {
 647         u32 port_count;
 648         u32 u32temp;
 649         u32 i;
 650
 651         u32temp = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 652         port_count = ((u32temp & 0x00000300) >> 8) + 1;
 653         /* TODO: assuming that both SERDES are set the same for now */
 654         *OneG_Mode = (u32temp & 0x00003c00) ? 0 : 1;
 655         nes_debug(NES_DBG_INIT, "Initial Software Reset = 0x%08X, port_count=%u\n",
 656                         u32temp, port_count);
 657         if (*OneG_Mode)
 658                 nes_debug(NES_DBG_INIT, "Running in 1G mode.\n");
 659         u32temp &= 0xff00ffc0;
 660         switch (port_count) {
 661                 case 1:
 662                         u32temp |= 0x00ee0000;
 663                         break;
 664                 case 2:
 665                         u32temp |= 0x00cc0000;
 666                         break;
 667                 case 4:
 668                         u32temp |= 0x00000000;
 669                         break;
 670                 default:
 671                         return 0;
 672                         break;
 673         }
 674
 675         /* check and do full reset if needed */
 676         if (nes_read_indexed(nesdev, NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8))) {
 677                 nes_debug(NES_DBG_INIT, "Issuing Full Soft reset = 0x%08X\n", u32temp | 0xd);
 678                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 679
 680                 i = 0;
 681                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 682                         mdelay(1);
 683                 if (i > 10000) {
 684                         nes_debug(NES_DBG_INIT, "Did not see full soft reset done.\n");
 685                         return 0;
 686                 }
 687
 688                 i = 0;
 689                 while ((nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS) != 0x80) && i++ < 10000)
 690                         mdelay(1);
 691                 if (i > 10000) {
 692                         printk(KERN_ERR PFX "Internal CPU not ready, status = %02X\n",
 693                                nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS));
 694                         return 0;
 695                 }
 696         }
 697
 698         /* port reset */
 699         switch (port_count) {
 700                 case 1:
 701                         u32temp |= 0x00ee0010;
 702                         break;
 703                 case 2:
 704                         u32temp |= 0x00cc0030;
 705                         break;
 706                 case 4:
 707                         u32temp |= 0x00000030;
 708                         break;
 709         }
 710
 711         nes_debug(NES_DBG_INIT, "Issuing Port Soft reset = 0x%08X\n", u32temp | 0xd);
 712         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 713
 714         i = 0;
 715         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 716                 mdelay(1);
 717         if (i > 10000) {
 718                 nes_debug(NES_DBG_INIT, "Did not see port soft reset done.\n");
 719                 return 0;
 720         }
 721
 722         /* serdes 0 */
 723         i = 0;
 724         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 725                         & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 726                 mdelay(1);
 727         if (i > 5000) {
 728                 nes_debug(NES_DBG_INIT, "Serdes 0 not ready, status=%x\n", u32temp);
 729                 return 0;
 730         }
 731
 732         /* serdes 1 */
 733         if (port_count > 1) {
 734                 i = 0;
 735                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 736                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 737                         mdelay(1);
 738                 if (i > 5000) {
 739                         nes_debug(NES_DBG_INIT, "Serdes 1 not ready, status=%x\n", u32temp);
 740                         return 0;
 741                 }
 742         }
 743
 744         return port_count;
 745 }
 746
 747
 748 /**
 749  * nes_init_serdes
 750  */
 751 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
 752                                 struct nes_adapter *nesadapter, u8  OneG_Mode)
 753 {
 754         int i;
 755         u32 u32temp;
 756         u32 sds;
 757
 758         if (hw_rev != NE020_REV) {
 759                 /* init serdes 0 */
 760                 switch (nesadapter->phy_type[0]) {
 761                 case NES_PHY_TYPE_CX4:
 762                         if (wide_ppm_offset)
 763                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000FFFAA);
 764                         else
 765                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 766                         break;
 767                 case NES_PHY_TYPE_KR:
 768                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 769                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x00000000);
 770                         break;
 771                 case NES_PHY_TYPE_PUMA_1G:
 772                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 773                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0);
 774                         sds |= 0x00000100;
 775                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, sds);
 776                         break;
 777                 default:
 778                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 779                         break;
 780                 }
 781
 782                 if (!OneG_Mode)
 783                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE0, 0x11110000);
 784
 785                 if (port_count < 2)
 786                         return 0;
 787
 788                 /* init serdes 1 */
 789                 if (!(OneG_Mode && (nesadapter->phy_type[1] != NES_PHY_TYPE_PUMA_1G)))
 790                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000FF);
 791
 792                 switch (nesadapter->phy_type[1]) {
 793                 case NES_PHY_TYPE_ARGUS:
 794                 case NES_PHY_TYPE_SFP_D:
 795                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x00000000);
 796                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x00000000);
 797                         break;
 798                 case NES_PHY_TYPE_CX4:
 799                         if (wide_ppm_offset)
 800                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000FFFAA);
 801                         break;
 802                 case NES_PHY_TYPE_KR:
 803                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x00000000);
 804                         break;
 805                 case NES_PHY_TYPE_PUMA_1G:
 806                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
 807                         sds |= 0x000000100;
 808                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, sds);
 809                 }
 810                 if (!OneG_Mode) {
 811                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE1, 0x11110000);
 812                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
 813                         sds &= 0xFFFFFFBF;
 814                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, sds);
 815                 }
 816         } else {
 817                 /* init serdes 0 */
 818                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, 0x00000008);
 819                 i = 0;
 820                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 821                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 822                         mdelay(1);
 823                 if (i > 5000) {
 824                         nes_debug(NES_DBG_PHY, "Init: serdes 0 not ready, status=%x\n", u32temp);
 825                         return 1;
 826                 }
 827                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x000bdef7);
 828                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE0, 0x9ce73000);
 829                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE0, 0x0ff00000);
 830                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET0, 0x00000000);
 831                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS0, 0x00000000);
 832                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL0, 0x00000000);
 833                 if (OneG_Mode)
 834                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0182222);
 835                 else
 836                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0042222);
 837
 838                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000ff);
 839                 if (port_count > 1) {
 840                         /* init serdes 1 */
 841                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x00000048);
 842                         i = 0;
 843                         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 844                                 & 0x0000000f)) != 0x0000000f) && (i++ < 5000))
 845                                 mdelay(1);
 846                         if (i > 5000) {
 847                                 printk("%s: Init: serdes 1 not ready, status=%x\n", __func__, u32temp);
 848                                 /* return 1; */
 849                         }
 850                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x000bdef7);
 851                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE1, 0x9ce73000);
 852                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE1, 0x0ff00000);
 853                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET1, 0x00000000);
 854                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS1, 0x00000000);
 855                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL1, 0x00000000);
 856                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL1, 0xf0002222);
 857                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000ff);
 858                 }
 859         }
 860         return 0;
 861 }
 862
 863
 864 /**
 865  * nes_init_csr_ne020
 866  * Initialize registers for ne020 hardware
 867  */
 868 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count)
 869 {
 870         u32 u32temp;
 871
 872         nes_debug(NES_DBG_INIT, "port_count=%d\n", port_count);
 873
 874         nes_write_indexed(nesdev, 0x000001E4, 0x00000007);
 875         /* nes_write_indexed(nesdev, 0x000001E8, 0x000208C4); */
 876         nes_write_indexed(nesdev, 0x000001E8, 0x00020874);
 877         nes_write_indexed(nesdev, 0x000001D8, 0x00048002);
 878         /* nes_write_indexed(nesdev, 0x000001D8, 0x0004B002); */
 879         nes_write_indexed(nesdev, 0x000001FC, 0x00050005);
 880         nes_write_indexed(nesdev, 0x00000600, 0x55555555);
 881         nes_write_indexed(nesdev, 0x00000604, 0x55555555);
 882
 883         /* TODO: move these MAC register settings to NIC bringup */
 884         nes_write_indexed(nesdev, 0x00002000, 0x00000001);
 885         nes_write_indexed(nesdev, 0x00002004, 0x00000001);
 886         nes_write_indexed(nesdev, 0x00002008, 0x0000FFFF);
 887         nes_write_indexed(nesdev, 0x0000200C, 0x00000001);
 888         nes_write_indexed(nesdev, 0x00002010, 0x000003c1);
 889         nes_write_indexed(nesdev, 0x0000201C, 0x75345678);
 890         if (port_count > 1) {
 891                 nes_write_indexed(nesdev, 0x00002200, 0x00000001);
 892                 nes_write_indexed(nesdev, 0x00002204, 0x00000001);
 893                 nes_write_indexed(nesdev, 0x00002208, 0x0000FFFF);
 894                 nes_write_indexed(nesdev, 0x0000220C, 0x00000001);
 895                 nes_write_indexed(nesdev, 0x00002210, 0x000003c1);
 896                 nes_write_indexed(nesdev, 0x0000221C, 0x75345678);
 897                 nes_write_indexed(nesdev, 0x00000908, 0x20000001);
 898         }
 899         if (port_count > 2) {
 900                 nes_write_indexed(nesdev, 0x00002400, 0x00000001);
 901                 nes_write_indexed(nesdev, 0x00002404, 0x00000001);
 902                 nes_write_indexed(nesdev, 0x00002408, 0x0000FFFF);
 903                 nes_write_indexed(nesdev, 0x0000240C, 0x00000001);
 904                 nes_write_indexed(nesdev, 0x00002410, 0x000003c1);
 905                 nes_write_indexed(nesdev, 0x0000241C, 0x75345678);
 906                 nes_write_indexed(nesdev, 0x00000910, 0x20000001);
 907
 908                 nes_write_indexed(nesdev, 0x00002600, 0x00000001);
 909                 nes_write_indexed(nesdev, 0x00002604, 0x00000001);
 910                 nes_write_indexed(nesdev, 0x00002608, 0x0000FFFF);
 911                 nes_write_indexed(nesdev, 0x0000260C, 0x00000001);
 912                 nes_write_indexed(nesdev, 0x00002610, 0x000003c1);
 913                 nes_write_indexed(nesdev, 0x0000261C, 0x75345678);
 914                 nes_write_indexed(nesdev, 0x00000918, 0x20000001);
 915         }
 916
 917         nes_write_indexed(nesdev, 0x00005000, 0x00018000);
 918         /* nes_write_indexed(nesdev, 0x00005000, 0x00010000); */
 919         nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG1, (wqm_quanta << 1) |
 920                                                          0x00000001);
 921         nes_write_indexed(nesdev, 0x00005008, 0x1F1F1F1F);
 922         nes_write_indexed(nesdev, 0x00005010, 0x1F1F1F1F);
 923         nes_write_indexed(nesdev, 0x00005018, 0x1F1F1F1F);
 924         nes_write_indexed(nesdev, 0x00005020, 0x1F1F1F1F);
 925         nes_write_indexed(nesdev, 0x00006090, 0xFFFFFFFF);
 926
 927         /* TODO: move this to code, get from EEPROM */
 928         nes_write_indexed(nesdev, 0x00000900, 0x20000001);
 929         nes_write_indexed(nesdev, 0x000060C0, 0x0000028e);
 930         nes_write_indexed(nesdev, 0x000060C8, 0x00000020);
 931
 932         nes_write_indexed(nesdev, 0x000001EC, 0x7b2625a0);
 933         /* nes_write_indexed(nesdev, 0x000001EC, 0x5f2625a0); */
 934
 935         if (hw_rev != NE020_REV) {
 936                 u32temp = nes_read_indexed(nesdev, 0x000008e8);
 937                 u32temp |= 0x80000000;
 938                 nes_write_indexed(nesdev, 0x000008e8, u32temp);
 939                 u32temp = nes_read_indexed(nesdev, 0x000021f8);
 940                 u32temp &= 0x7fffffff;
 941                 u32temp |= 0x7fff0010;
 942                 nes_write_indexed(nesdev, 0x000021f8, u32temp);
 943                 if (port_count > 1) {
 944                         u32temp = nes_read_indexed(nesdev, 0x000023f8);
 945                         u32temp &= 0x7fffffff;
 946                         u32temp |= 0x7fff0010;
 947                         nes_write_indexed(nesdev, 0x000023f8, u32temp);
 948                 }
 949         }
 950 }
 951
 952
 953 /**
 954  * nes_destroy_adapter - destroy the adapter structure
 955  */
 956 void nes_destroy_adapter(struct nes_adapter *nesadapter)
 957 {
 958         struct nes_adapter *tmp_adapter;
 959
 960         list_for_each_entry(tmp_adapter, &nes_adapter_list, list) {
 961                 nes_debug(NES_DBG_SHUTDOWN, "Nes Adapter list entry = 0x%p.\n",
 962                                 tmp_adapter);
 963         }
 964
 965         nesadapter->ref_count--;
 966         if (!nesadapter->ref_count) {
 967                 if (nesadapter->hw_rev == NE020_REV) {
 968                         del_timer(&nesadapter->mh_timer);
 969                 }
 970                 del_timer(&nesadapter->lc_timer);
 971
 972                 list_del(&nesadapter->list);
 973                 kfree(nesadapter);
 974         }
 975 }
 976
 977
 978 /**
 979  * nes_init_cqp
 980  */
 981 int nes_init_cqp(struct nes_device *nesdev)
 982 {
 983         struct nes_adapter *nesadapter = nesdev->nesadapter;
 984         struct nes_hw_cqp_qp_context *cqp_qp_context;
 985         struct nes_hw_cqp_wqe *cqp_wqe;
 986         struct nes_hw_ceq *ceq;
 987         struct nes_hw_ceq *nic_ceq;
 988         struct nes_hw_aeq *aeq;
 989         void *vmem;
 990         dma_addr_t pmem;
 991         u32 count=0;
 992         u32 cqp_head;
 993         u64 u64temp;
 994         u32 u32temp;
 995
 996         /* allocate CQP memory */
 997         /* Need to add max_cq to the aeq size once cq overflow checking is added back */
 998         /* SQ is 512 byte aligned, others are 256 byte aligned */
 999         nesdev->cqp_mem_size = 512 +
1000                         (sizeof(struct nes_hw_cqp_wqe) * NES_CQP_SQ_SIZE) +
1001                         (sizeof(struct nes_hw_cqe) * NES_CCQ_SIZE) +
1002                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_CCEQ_SIZE), (u32)256) +
1003                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_NIC_CEQ_SIZE), (u32)256) +
1004                         (sizeof(struct nes_hw_aeqe) * nesadapter->max_qp) +
1005                         sizeof(struct nes_hw_cqp_qp_context);
1006
1007         nesdev->cqp_vbase = pci_alloc_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1008                         &nesdev->cqp_pbase);
1009         if (!nesdev->cqp_vbase) {
1010                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for host descriptor rings\n");
1011                 return -ENOMEM;
1012         }
1013         memset(nesdev->cqp_vbase, 0, nesdev->cqp_mem_size);
1014
1015         /* Allocate a twice the number of CQP requests as the SQ size */
1016         nesdev->nes_cqp_requests = kzalloc(sizeof(struct nes_cqp_request) *
1017                         2 * NES_CQP_SQ_SIZE, GFP_KERNEL);
1018         if (nesdev->nes_cqp_requests == NULL) {
1019                 nes_debug(NES_DBG_INIT, "Unable to allocate memory CQP request entries.\n");
1020                 pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
1021                                 nesdev->cqp.sq_pbase);
1022                 return -ENOMEM;
1023         }
1024
1025         nes_debug(NES_DBG_INIT, "Allocated CQP structures at %p (phys = %016lX), size = %u.\n",
1026                         nesdev->cqp_vbase, (unsigned long)nesdev->cqp_pbase, nesdev->cqp_mem_size);
1027
1028         spin_lock_init(&nesdev->cqp.lock);
1029         init_waitqueue_head(&nesdev->cqp.waitq);
1030
1031         /* Setup Various Structures */
1032         vmem = (void *)(((unsigned long)nesdev->cqp_vbase + (512 - 1)) &
1033                         ~(unsigned long)(512 - 1));
1034         pmem = (dma_addr_t)(((unsigned long long)nesdev->cqp_pbase + (512 - 1)) &
1035                         ~(unsigned long long)(512 - 1));
1036
1037         nesdev->cqp.sq_vbase = vmem;
1038         nesdev->cqp.sq_pbase = pmem;
1039         nesdev->cqp.sq_size = NES_CQP_SQ_SIZE;
1040         nesdev->cqp.sq_head = 0;
1041         nesdev->cqp.sq_tail = 0;
1042         nesdev->cqp.qp_id = PCI_FUNC(nesdev->pcidev->devfn);
1043
1044         vmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
1045         pmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
1046
1047         nesdev->ccq.cq_vbase = vmem;
1048         nesdev->ccq.cq_pbase = pmem;
1049         nesdev->ccq.cq_size = NES_CCQ_SIZE;
1050         nesdev->ccq.cq_head = 0;
1051         nesdev->ccq.ce_handler = nes_cqp_ce_handler;
1052         nesdev->ccq.cq_number = PCI_FUNC(nesdev->pcidev->devfn);
1053
1054         vmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
1055         pmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
1056
1057         nesdev->ceq_index = PCI_FUNC(nesdev->pcidev->devfn);
1058         ceq = &nesadapter->ceq[nesdev->ceq_index];
1059         ceq->ceq_vbase = vmem;
1060         ceq->ceq_pbase = pmem;
1061         ceq->ceq_size = NES_CCEQ_SIZE;
1062         ceq->ceq_head = 0;
1063
1064         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
1065         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
1066
1067         nesdev->nic_ceq_index = PCI_FUNC(nesdev->pcidev->devfn) + 8;
1068         nic_ceq = &nesadapter->ceq[nesdev->nic_ceq_index];
1069         nic_ceq->ceq_vbase = vmem;
1070         nic_ceq->ceq_pbase = pmem;
1071         nic_ceq->ceq_size = NES_NIC_CEQ_SIZE;
1072         nic_ceq->ceq_head = 0;
1073
1074         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
1075         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
1076
1077         aeq = &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)];
1078         aeq->aeq_vbase = vmem;
1079         aeq->aeq_pbase = pmem;
1080         aeq->aeq_size = nesadapter->max_qp;
1081         aeq->aeq_head = 0;
1082
1083         /* Setup QP Context */
1084         vmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
1085         pmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
1086
1087         cqp_qp_context = vmem;
1088         cqp_qp_context->context_words[0] =
1089                         cpu_to_le32((PCI_FUNC(nesdev->pcidev->devfn) << 12) + (2 << 10));
1090         cqp_qp_context->context_words[1] = 0;
1091         cqp_qp_context->context_words[2] = cpu_to_le32((u32)nesdev->cqp.sq_pbase);
1092         cqp_qp_context->context_words[3] = cpu_to_le32(((u64)nesdev->cqp.sq_pbase) >> 32);
1093
1094
1095         /* Write the address to Create CQP */
1096         if ((sizeof(dma_addr_t) > 4)) {
1097                 nes_write_indexed(nesdev,
1098                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1099                                 ((u64)pmem) >> 32);
1100         } else {
1101                 nes_write_indexed(nesdev,
1102                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8), 0);
1103         }
1104         nes_write_indexed(nesdev,
1105                         NES_IDX_CREATE_CQP_LOW + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1106                         (u32)pmem);
1107
1108         INIT_LIST_HEAD(&nesdev->cqp_avail_reqs);
1109         INIT_LIST_HEAD(&nesdev->cqp_pending_reqs);
1110
1111         for (count = 0; count < 2*NES_CQP_SQ_SIZE; count++) {
1112                 init_waitqueue_head(&nesdev->nes_cqp_requests[count].waitq);
1113                 list_add_tail(&nesdev->nes_cqp_requests[count].list, &nesdev->cqp_avail_reqs);
1114         }
1115
1116         /* Write Create CCQ WQE */
1117         cqp_head = nesdev->cqp.sq_head++;
1118         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1119         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1120         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1121                         (NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1122                         NES_CQP_CQ_CHK_OVERFLOW | ((u32)nesdev->ccq.cq_size << 16)));
1123         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1124                             (nesdev->ccq.cq_number |
1125                              ((u32)nesdev->ceq_index << 16)));
1126         u64temp = (u64)nesdev->ccq.cq_pbase;
1127         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1128         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 0;
1129         u64temp = (unsigned long)&nesdev->ccq;
1130         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =
1131                         cpu_to_le32((u32)(u64temp >> 1));
1132         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1133                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1134         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1135
1136         /* Write Create CEQ WQE */
1137         cqp_head = nesdev->cqp.sq_head++;
1138         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1139         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1140         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1141                             (NES_CQP_CREATE_CEQ + ((u32)nesdev->ceq_index << 8)));
1142         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, ceq->ceq_size);
1143         u64temp = (u64)ceq->ceq_pbase;
1144         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1145
1146         /* Write Create AEQ WQE */
1147         cqp_head = nesdev->cqp.sq_head++;
1148         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1149         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1150         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1151                         (NES_CQP_CREATE_AEQ + ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8)));
1152         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_AEQ_WQE_ELEMENT_COUNT_IDX, aeq->aeq_size);
1153         u64temp = (u64)aeq->aeq_pbase;
1154         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1155
1156         /* Write Create NIC CEQ WQE */
1157         cqp_head = nesdev->cqp.sq_head++;
1158         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1159         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1160         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1161                         (NES_CQP_CREATE_CEQ + ((u32)nesdev->nic_ceq_index << 8)));
1162         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, nic_ceq->ceq_size);
1163         u64temp = (u64)nic_ceq->ceq_pbase;
1164         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1165
1166         /* Poll until CCQP done */
1167         count = 0;
1168         do {
1169                 if (count++ > 1000) {
1170                         printk(KERN_ERR PFX "Error creating CQP\n");
1171                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1172                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1173                         return -1;
1174                 }
1175                 udelay(10);
1176         } while (!(nes_read_indexed(nesdev,
1177                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn) * 8)) & (1 << 8)));
1178
1179         nes_debug(NES_DBG_INIT, "CQP Status = 0x%08X\n", nes_read_indexed(nesdev,
1180                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1181
1182         u32temp = 0x04800000;
1183         nes_write32(nesdev->regs+NES_WQE_ALLOC, u32temp | nesdev->cqp.qp_id);
1184
1185         /* wait for the CCQ, CEQ, and AEQ to get created */
1186         count = 0;
1187         do {
1188                 if (count++ > 1000) {
1189                         printk(KERN_ERR PFX "Error creating CCQ, CEQ, and AEQ\n");
1190                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1191                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1192                         return -1;
1193                 }
1194                 udelay(10);
1195         } while (((nes_read_indexed(nesdev,
1196                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15<<8)) != (15<<8)));
1197
1198         /* dump the QP status value */
1199         nes_debug(NES_DBG_INIT, "QP Status = 0x%08X\n", nes_read_indexed(nesdev,
1200                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1201
1202         nesdev->cqp.sq_tail++;
1203
1204         return 0;
1205 }
1206
1207
1208 /**
1209  * nes_destroy_cqp
1210  */
1211 int nes_destroy_cqp(struct nes_device *nesdev)
1212 {
1213         struct nes_hw_cqp_wqe *cqp_wqe;
1214         u32 count = 0;
1215         u32 cqp_head;
1216         unsigned long flags;
1217
1218         do {
1219                 if (count++ > 1000)
1220                         break;
1221                 udelay(10);
1222         } while (!(nesdev->cqp.sq_head == nesdev->cqp.sq_tail));
1223
1224         /* Reset CCQ */
1225         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_RESET |
1226                         nesdev->ccq.cq_number);
1227
1228         /* Disable device interrupts */
1229         nes_write32(nesdev->regs+NES_INT_MASK, 0x7fffffff);
1230
1231         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1232
1233         /* Destroy the AEQ */
1234         cqp_head = nesdev->cqp.sq_head++;
1235         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1236         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1237         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_AEQ |
1238                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8));
1239         cqp_wqe->wqe_words[NES_CQP_WQE_COMP_CTX_HIGH_IDX] = 0;
1240
1241         /* Destroy the NIC CEQ */
1242         cqp_head = nesdev->cqp.sq_head++;
1243         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1244         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1245         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1246                         ((u32)nesdev->nic_ceq_index << 8));
1247
1248         /* Destroy the CEQ */
1249         cqp_head = nesdev->cqp.sq_head++;
1250         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1251         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1252         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1253                         (nesdev->ceq_index << 8));
1254
1255         /* Destroy the CCQ */
1256         cqp_head = nesdev->cqp.sq_head++;
1257         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1258         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1259         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CQ);
1260         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->ccq.cq_number |
1261                         ((u32)nesdev->ceq_index << 16));
1262
1263         /* Destroy CQP */
1264         cqp_head = nesdev->cqp.sq_head++;
1265         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1266         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1267         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_QP |
1268                         NES_CQP_QP_TYPE_CQP);
1269         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->cqp.qp_id);
1270
1271         barrier();
1272         /* Ring doorbell (5 WQEs) */
1273         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x05800000 | nesdev->cqp.qp_id);
1274
1275         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1276
1277         /* wait for the CCQ, CEQ, and AEQ to get destroyed */
1278         count = 0;
1279         do {
1280                 if (count++ > 1000) {
1281                         printk(KERN_ERR PFX "Function%d: Error destroying CCQ, CEQ, and AEQ\n",
1282                                         PCI_FUNC(nesdev->pcidev->devfn));
1283                         break;
1284                 }
1285                 udelay(10);
1286         } while (((nes_read_indexed(nesdev,
1287                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15 << 8)) != 0));
1288
1289         /* dump the QP status value */
1290         nes_debug(NES_DBG_SHUTDOWN, "Function%d: QP Status = 0x%08X\n",
1291                         PCI_FUNC(nesdev->pcidev->devfn),
1292                         nes_read_indexed(nesdev,
1293                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1294
1295         kfree(nesdev->nes_cqp_requests);
1296
1297         /* Free the control structures */
1298         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
1299                         nesdev->cqp.sq_pbase);
1300
1301         return 0;
1302 }
1303
1304
1305 /**
1306  * nes_init_1g_phy
1307  */
1308 static int nes_init_1g_phy(struct nes_device *nesdev, u8 phy_type, u8 phy_index)
1309 {
1310         u32 counter = 0;
1311         u16 phy_data;
1312         int ret = 0;
1313
1314         nes_read_1G_phy_reg(nesdev, 1, phy_index, &phy_data);
1315         nes_write_1G_phy_reg(nesdev, 23, phy_index, 0xb000);
1316
1317         /* Reset the PHY */
1318         nes_write_1G_phy_reg(nesdev, 0, phy_index, 0x8000);
1319         udelay(100);
1320         counter = 0;
1321         do {
1322                 nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1323                 if (counter++ > 100) {
1324                         ret = -1;
1325                         break;
1326                 }
1327         } while (phy_data & 0x8000);
1328
1329         /* Setting no phy loopback */
1330         phy_data &= 0xbfff;
1331         phy_data |= 0x1140;
1332         nes_write_1G_phy_reg(nesdev, 0, phy_index,  phy_data);
1333         nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1334         nes_read_1G_phy_reg(nesdev, 0x17, phy_index, &phy_data);
1335         nes_read_1G_phy_reg(nesdev, 0x1e, phy_index, &phy_data);
1336
1337         /* Setting the interrupt mask */
1338         nes_read_1G_phy_reg(nesdev, 0x19, phy_index, &phy_data);
1339         nes_write_1G_phy_reg(nesdev, 0x19, phy_index, 0xffee);
1340         nes_read_1G_phy_reg(nesdev, 0x19, phy_index, &phy_data);
1341
1342         /* turning on flow control */
1343         nes_read_1G_phy_reg(nesdev, 4, phy_index, &phy_data);
1344         nes_write_1G_phy_reg(nesdev, 4, phy_index, (phy_data & ~(0x03E0)) | 0xc00);
1345         nes_read_1G_phy_reg(nesdev, 4, phy_index, &phy_data);
1346
1347         /* Clear Half duplex */
1348         nes_read_1G_phy_reg(nesdev, 9, phy_index, &phy_data);
1349         nes_write_1G_phy_reg(nesdev, 9, phy_index, phy_data & ~(0x0100));
1350         nes_read_1G_phy_reg(nesdev, 9, phy_index, &phy_data);
1351
1352         nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1353         nes_write_1G_phy_reg(nesdev, 0, phy_index, phy_data | 0x0300);
1354
1355         return ret;
1356 }
1357
1358
1359 /**
1360  * nes_init_2025_phy
1361  */
1362 static int nes_init_2025_phy(struct nes_device *nesdev, u8 phy_type, u8 phy_index)
1363 {
1364         u32 temp_phy_data = 0;
1365         u32 temp_phy_data2 = 0;
1366         u32 counter = 0;
1367         u32 sds;
1368         u32 mac_index = nesdev->mac_index;
1369         int ret = 0;
1370         unsigned int first_attempt = 1;
1371
1372         /* Check firmware heartbeat */
1373         nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1374         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1375         udelay(1500);
1376         nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1377         temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1378
1379         if (temp_phy_data != temp_phy_data2) {
1380                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7fd);
1381                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1382                 if ((temp_phy_data & 0xff) > 0x20)
1383                         return 0;
1384                 printk(PFX "Reinitialize external PHY\n");
1385         }
1386
1387         /* no heartbeat, configure the PHY */
1388         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0x0000, 0x8000);
1389         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc300, 0x0000);
1390         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1391         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1392
1393         switch (phy_type) {
1394         case NES_PHY_TYPE_ARGUS:
1395                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1396                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1397                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x000C);
1398                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0008);
1399                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0001);
1400                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0098);
1401                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00);
1402
1403                 /* setup LEDs */
1404                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x0007);
1405                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x000A);
1406                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0009);
1407                 break;
1408
1409         case NES_PHY_TYPE_SFP_D:
1410                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1411                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1412                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x0004);
1413                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0038);
1414                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0013);
1415                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0098);
1416                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00);
1417
1418                 /* setup LEDs */
1419                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x0007);
1420                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x000A);
1421                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0009);
1422                 break;
1423
1424         case NES_PHY_TYPE_KR:
1425                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1426                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1427                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x000C);
1428                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0010);
1429                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0013);
1430                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0080);
1431                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00);
1432
1433                 /* setup LEDs */
1434                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x000B);
1435                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x0003);
1436                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0004);
1437
1438                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0022, 0x406D);
1439                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0023, 0x0020);
1440                 break;
1441         }
1442
1443         nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0028, 0xA528);
1444
1445         /* Bring PHY out of reset */
1446         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc300, 0x0002);
1447
1448         /* Check for heartbeat */
1449         counter = 0;
1450         mdelay(690);
1451         nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1452         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1453         do {
1454                 if (counter++ > 150) {
1455                         printk(PFX "No PHY heartbeat\n");
1456                         break;
1457                 }
1458                 mdelay(1);
1459                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1460                 temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1461         } while ((temp_phy_data2 == temp_phy_data));
1462
1463         /* wait for tracking */
1464         counter = 0;
1465         do {
1466                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7fd);
1467                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1468                 if (counter++ > 300) {
1469                         if (((temp_phy_data & 0xff) == 0x0) && first_attempt) {
1470                                 first_attempt = 0;
1471                                 counter = 0;
1472                                 /* reset AMCC PHY and try again */
1473                                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0xe854, 0x00c0);
1474                                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0xe854, 0x0040);
1475                                 continue;
1476                         } else {
1477                                 ret = 1;
1478                                 break;
1479                         }
1480                 }
1481                 mdelay(10);
1482         } while ((temp_phy_data & 0xff) < 0x30);
1483
1484         /* setup signal integrity */
1485         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd003, 0x0000);
1486         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00D, 0x00FE);
1487         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00E, 0x0032);
1488         if (phy_type == NES_PHY_TYPE_KR) {
1489                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00F, 0x000C);
1490         } else {
1491                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00F, 0x0002);
1492                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc314, 0x0063);
1493         }
1494
1495         /* reset serdes */
1496         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 + mac_index * 0x200);
1497         sds |= 0x1;
1498         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 + mac_index * 0x200, sds);
1499         sds &= 0xfffffffe;
1500         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 + mac_index * 0x200, sds);
1501
1502         counter = 0;
1503         while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET) & 0x00000040) != 0x00000040)
1504                         && (counter++ < 5000))
1505                 ;
1506
1507         return ret;
1508 }
1509
1510
1511 /**
1512  * nes_init_phy
1513  */
1514 int nes_init_phy(struct nes_device *nesdev)
1515 {
1516         struct nes_adapter *nesadapter = nesdev->nesadapter;
1517         u32 mac_index = nesdev->mac_index;
1518         u32 tx_config = 0;
1519         unsigned long flags;
1520         u8  phy_type = nesadapter->phy_type[mac_index];
1521         u8  phy_index = nesadapter->phy_index[mac_index];
1522         int ret = 0;
1523
1524         tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
1525         if (phy_type == NES_PHY_TYPE_1G) {
1526                 /* setup 1G MDIO operation */
1527                 tx_config &= 0xFFFFFFE3;
1528                 tx_config |= 0x04;
1529         } else {
1530                 /* setup 10G MDIO operation */
1531                 tx_config &= 0xFFFFFFE3;
1532                 tx_config |= 0x1D;
1533         }
1534         nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
1535
1536         spin_lock_irqsave(&nesdev->nesadapter->phy_lock, flags);
1537
1538         switch (phy_type) {
1539         case NES_PHY_TYPE_1G:
1540                 ret = nes_init_1g_phy(nesdev, phy_type, phy_index);
1541                 break;
1542         case NES_PHY_TYPE_ARGUS:
1543         case NES_PHY_TYPE_SFP_D:
1544         case NES_PHY_TYPE_KR:
1545                 ret = nes_init_2025_phy(nesdev, phy_type, phy_index);
1546                 break;
1547         }
1548
1549         spin_unlock_irqrestore(&nesdev->nesadapter->phy_lock, flags);
1550
1551         return ret;
1552 }
1553
1554
1555 /**
1556  * nes_replenish_nic_rq
1557  */
1558 static void nes_replenish_nic_rq(struct nes_vnic *nesvnic)
1559 {
1560         unsigned long flags;
1561         dma_addr_t bus_address;
1562         struct sk_buff *skb;
1563         struct nes_hw_nic_rq_wqe *nic_rqe;
1564         struct nes_hw_nic *nesnic;
1565         struct nes_device *nesdev;
1566         struct nes_rskb_cb *cb;
1567         u32 rx_wqes_posted = 0;
1568
1569         nesnic = &nesvnic->nic;
1570         nesdev = nesvnic->nesdev;
1571         spin_lock_irqsave(&nesnic->rq_lock, flags);
1572         if (nesnic->replenishing_rq !=0) {
1573                 if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1574                                 (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1575                         atomic_set(&nesvnic->rx_skb_timer_running, 1);
1576                         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1577                         nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1578                         add_timer(&nesvnic->rq_wqes_timer);
1579                 } else
1580                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1581                 return;
1582         }
1583         nesnic->replenishing_rq = 1;
1584         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1585         do {
1586                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1587                 if (skb) {
1588                         skb->dev = nesvnic->netdev;
1589
1590                         bus_address = pci_map_single(nesdev->pcidev,
1591                                         skb->data, nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1592                         cb = (struct nes_rskb_cb *)&skb->cb[0];
1593                         cb->busaddr = bus_address;
1594                         cb->maplen = nesvnic->max_frame_size;
1595
1596                         nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_head];
1597                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] =
1598                                         cpu_to_le32(nesvnic->max_frame_size);
1599                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1600                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] =
1601                                         cpu_to_le32((u32)bus_address);
1602                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] =
1603                                         cpu_to_le32((u32)((u64)bus_address >> 32));
1604                         nesnic->rx_skb[nesnic->rq_head] = skb;
1605                         nesnic->rq_head++;
1606                         nesnic->rq_head &= nesnic->rq_size - 1;
1607                         atomic_dec(&nesvnic->rx_skbs_needed);
1608                         barrier();
1609                         if (++rx_wqes_posted == 255) {
1610                                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1611                                 rx_wqes_posted = 0;
1612                         }
1613                 } else {
1614                         spin_lock_irqsave(&nesnic->rq_lock, flags);
1615                         if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1616                                         (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1617                                 atomic_set(&nesvnic->rx_skb_timer_running, 1);
1618                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1619                                 nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1620                                 add_timer(&nesvnic->rq_wqes_timer);
1621                         } else
1622                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1623                         break;
1624                 }
1625         } while (atomic_read(&nesvnic->rx_skbs_needed));
1626         barrier();
1627         if (rx_wqes_posted)
1628                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1629         nesnic->replenishing_rq = 0;
1630 }
1631
1632
1633 /**
1634  * nes_rq_wqes_timeout
1635  */
1636 static void nes_rq_wqes_timeout(unsigned long parm)
1637 {
1638         struct nes_vnic *nesvnic = (struct nes_vnic *)parm;
1639         printk("%s: Timer fired.\n", __func__);
1640         atomic_set(&nesvnic->rx_skb_timer_running, 0);
1641         if (atomic_read(&nesvnic->rx_skbs_needed))
1642                 nes_replenish_nic_rq(nesvnic);
1643 }
1644
1645
1646 static int nes_lro_get_skb_hdr(struct sk_buff *skb, void **iphdr,
1647                                void **tcph, u64 *hdr_flags, void *priv)
1648 {
1649         unsigned int ip_len;
1650         struct iphdr *iph;
1651         skb_reset_network_header(skb);
1652         iph = ip_hdr(skb);
1653         if (iph->protocol != IPPROTO_TCP)
1654                 return -1;
1655         ip_len = ip_hdrlen(skb);
1656         skb_set_transport_header(skb, ip_len);
1657         *tcph = tcp_hdr(skb);
1658
1659         *hdr_flags = LRO_IPV4 | LRO_TCP;
1660         *iphdr = iph;
1661         return 0;
1662 }
1663
1664
1665 /**
1666  * nes_init_nic_qp
1667  */
1668 int nes_init_nic_qp(struct nes_device *nesdev, struct net_device *netdev)
1669 {
1670         struct nes_hw_cqp_wqe *cqp_wqe;
1671         struct nes_hw_nic_sq_wqe *nic_sqe;
1672         struct nes_hw_nic_qp_context *nic_context;
1673         struct sk_buff *skb;
1674         struct nes_hw_nic_rq_wqe *nic_rqe;
1675         struct nes_vnic *nesvnic = netdev_priv(netdev);
1676         unsigned long flags;
1677         void *vmem;
1678         dma_addr_t pmem;
1679         u64 u64temp;
1680         int ret;
1681         u32 cqp_head;
1682         u32 counter;
1683         u32 wqe_count;
1684         struct nes_rskb_cb *cb;
1685         u8 jumbomode=0;
1686
1687         /* Allocate fragment, SQ, RQ, and CQ; Reuse CEQ based on the PCI function */
1688         nesvnic->nic_mem_size = 256 +
1689                         (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag)) +
1690                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe)) +
1691                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe)) +
1692                         (NES_NIC_WQ_SIZE * 2 * sizeof(struct nes_hw_nic_cqe)) +
1693                         sizeof(struct nes_hw_nic_qp_context);
1694
1695         nesvnic->nic_vbase = pci_alloc_consistent(nesdev->pcidev, nesvnic->nic_mem_size,
1696                         &nesvnic->nic_pbase);
1697         if (!nesvnic->nic_vbase) {
1698                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for NIC host descriptor rings\n");
1699                 return -ENOMEM;
1700         }
1701         memset(nesvnic->nic_vbase, 0, nesvnic->nic_mem_size);
1702         nes_debug(NES_DBG_INIT, "Allocated NIC QP structures at %p (phys = %016lX), size = %u.\n",
1703                         nesvnic->nic_vbase, (unsigned long)nesvnic->nic_pbase, nesvnic->nic_mem_size);
1704
1705         vmem = (void *)(((unsigned long)nesvnic->nic_vbase + (256 - 1)) &
1706                         ~(unsigned long)(256 - 1));
1707         pmem = (dma_addr_t)(((unsigned long long)nesvnic->nic_pbase + (256 - 1)) &
1708                         ~(unsigned long long)(256 - 1));
1709
1710         /* Setup the first Fragment buffers */
1711         nesvnic->nic.first_frag_vbase = vmem;
1712
1713         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1714                 nesvnic->nic.frag_paddr[counter] = pmem;
1715                 pmem += sizeof(struct nes_first_frag);
1716         }
1717
1718         /* setup the SQ */
1719         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag));
1720
1721         nesvnic->nic.sq_vbase = (void *)vmem;
1722         nesvnic->nic.sq_pbase = pmem;
1723         nesvnic->nic.sq_head = 0;
1724         nesvnic->nic.sq_tail = 0;
1725         nesvnic->nic.sq_size = NES_NIC_WQ_SIZE;
1726         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1727                 nic_sqe = &nesvnic->nic.sq_vbase[counter];
1728                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_MISC_IDX] =
1729                                 cpu_to_le32(NES_NIC_SQ_WQE_DISABLE_CHKSUM |
1730                                 NES_NIC_SQ_WQE_COMPLETION);
1731                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX] =
1732                                 cpu_to_le32((u32)NES_FIRST_FRAG_SIZE << 16);
1733                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX] =
1734                                 cpu_to_le32((u32)nesvnic->nic.frag_paddr[counter]);
1735                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX] =
1736                                 cpu_to_le32((u32)((u64)nesvnic->nic.frag_paddr[counter] >> 32));
1737         }
1738
1739         nesvnic->get_cqp_request = nes_get_cqp_request;
1740         nesvnic->post_cqp_request = nes_post_cqp_request;
1741         nesvnic->mcrq_mcast_filter = NULL;
1742
1743         spin_lock_init(&nesvnic->nic.rq_lock);
1744
1745         /* setup the RQ */
1746         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1747         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1748
1749
1750         nesvnic->nic.rq_vbase = vmem;
1751         nesvnic->nic.rq_pbase = pmem;
1752         nesvnic->nic.rq_head = 0;
1753         nesvnic->nic.rq_tail = 0;
1754         nesvnic->nic.rq_size = NES_NIC_WQ_SIZE;
1755
1756         /* setup the CQ */
1757         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1758         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1759
1760         if (nesdev->nesadapter->netdev_count > 2)
1761                 nesvnic->mcrq_qp_id = nesvnic->nic_index + 32;
1762         else
1763                 nesvnic->mcrq_qp_id = nesvnic->nic.qp_id + 4;
1764
1765         nesvnic->nic_cq.cq_vbase = vmem;
1766         nesvnic->nic_cq.cq_pbase = pmem;
1767         nesvnic->nic_cq.cq_head = 0;
1768         nesvnic->nic_cq.cq_size = NES_NIC_WQ_SIZE * 2;
1769
1770         nesvnic->nic_cq.ce_handler = nes_nic_napi_ce_handler;
1771
1772         /* Send CreateCQ request to CQP */
1773         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1774         cqp_head = nesdev->cqp.sq_head;
1775
1776         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1777         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1778
1779         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
1780                         NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1781                         ((u32)nesvnic->nic_cq.cq_size << 16));
1782         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(
1783                         nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16));
1784         u64temp = (u64)nesvnic->nic_cq.cq_pbase;
1785         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1786         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =  0;
1787         u64temp = (unsigned long)&nesvnic->nic_cq;
1788         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =  cpu_to_le32((u32)(u64temp >> 1));
1789         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1790                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1791         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1792         if (++cqp_head >= nesdev->cqp.sq_size)
1793                 cqp_head = 0;
1794         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1795         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1796
1797         /* Send CreateQP request to CQP */
1798         nic_context = (void *)(&nesvnic->nic_cq.cq_vbase[nesvnic->nic_cq.cq_size]);
1799         nic_context->context_words[NES_NIC_CTX_MISC_IDX] =
1800                         cpu_to_le32((u32)NES_NIC_CTX_SIZE |
1801                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 12));
1802         nes_debug(NES_DBG_INIT, "RX_WINDOW_BUFFER_PAGE_TABLE_SIZE = 0x%08X, RX_WINDOW_BUFFER_SIZE = 0x%08X\n",
1803                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_PAGE_TABLE_SIZE),
1804                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE));
1805         if (nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE) != 0) {
1806                 nic_context->context_words[NES_NIC_CTX_MISC_IDX] |= cpu_to_le32(NES_NIC_BACK_STORE);
1807         }
1808
1809         u64temp = (u64)nesvnic->nic.sq_pbase;
1810         nic_context->context_words[NES_NIC_CTX_SQ_LOW_IDX]  = cpu_to_le32((u32)u64temp);
1811         nic_context->context_words[NES_NIC_CTX_SQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1812         u64temp = (u64)nesvnic->nic.rq_pbase;
1813         nic_context->context_words[NES_NIC_CTX_RQ_LOW_IDX]  = cpu_to_le32((u32)u64temp);
1814         nic_context->context_words[NES_NIC_CTX_RQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1815
1816         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_CREATE_QP |
1817                         NES_CQP_QP_TYPE_NIC);
1818         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesvnic->nic.qp_id);
1819         u64temp = (u64)nesvnic->nic_cq.cq_pbase +
1820                         (nesvnic->nic_cq.cq_size * sizeof(struct nes_hw_nic_cqe));
1821         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp);
1822
1823         if (++cqp_head >= nesdev->cqp.sq_size)
1824                 cqp_head = 0;
1825         nesdev->cqp.sq_head = cqp_head;
1826
1827         barrier();
1828
1829         /* Ring doorbell (2 WQEs) */
1830         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1831
1832         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1833         nes_debug(NES_DBG_INIT, "Waiting for create NIC QP%u to complete.\n",
1834                         nesvnic->nic.qp_id);
1835
1836         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1837                         NES_EVENT_TIMEOUT);
1838         nes_debug(NES_DBG_INIT, "Create NIC QP%u completed, wait_event_timeout ret = %u.\n",
1839                         nesvnic->nic.qp_id, ret);
1840         if (!ret) {
1841                 nes_debug(NES_DBG_INIT, "NIC QP%u create timeout expired\n", nesvnic->nic.qp_id);
1842                 pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
1843                                 nesvnic->nic_pbase);
1844                 return -EIO;
1845         }
1846
1847         /* Populate the RQ */
1848         for (counter = 0; counter < (NES_NIC_WQ_SIZE - 1); counter++) {
1849                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1850                 if (!skb) {
1851                         nes_debug(NES_DBG_INIT, "%s: out of memory for receive skb\n", netdev->name);
1852
1853                         nes_destroy_nic_qp(nesvnic);
1854                         return -ENOMEM;
1855                 }
1856
1857                 skb->dev = netdev;
1858
1859                 pmem = pci_map_single(nesdev->pcidev, skb->data,
1860                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1861                 cb = (struct nes_rskb_cb *)&skb->cb[0];
1862                 cb->busaddr = pmem;
1863                 cb->maplen = nesvnic->max_frame_size;
1864
1865                 nic_rqe = &nesvnic->nic.rq_vbase[counter];
1866                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] = cpu_to_le32(nesvnic->max_frame_size);
1867                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1868                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]  = cpu_to_le32((u32)pmem);
1869                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] = cpu_to_le32((u32)((u64)pmem >> 32));
1870                 nesvnic->nic.rx_skb[counter] = skb;
1871         }
1872
1873         wqe_count = NES_NIC_WQ_SIZE - 1;
1874         nesvnic->nic.rq_head = wqe_count;
1875         barrier();
1876         do {
1877                 counter = min(wqe_count, ((u32)255));
1878                 wqe_count -= counter;
1879                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (counter << 24) | nesvnic->nic.qp_id);
1880         } while (wqe_count);
1881         init_timer(&nesvnic->rq_wqes_timer);
1882         nesvnic->rq_wqes_timer.function = nes_rq_wqes_timeout;
1883         nesvnic->rq_wqes_timer.data = (unsigned long)nesvnic;
1884         nes_debug(NES_DBG_INIT, "NAPI support Enabled\n");
1885         if (nesdev->nesadapter->et_use_adaptive_rx_coalesce)
1886         {
1887                 nes_nic_init_timer(nesdev);
1888                 if (netdev->mtu > 1500)
1889                         jumbomode = 1;
1890                 nes_nic_init_timer_defaults(nesdev, jumbomode);
1891         }
1892         if ((nesdev->nesadapter->allow_unaligned_fpdus) &&
1893                 (nes_init_mgt_qp(nesdev, netdev, nesvnic))) {
1894                         nes_debug(NES_DBG_INIT, "%s: Out of memory for pau nic\n", netdev->name);
1895                         nes_destroy_nic_qp(nesvnic);
1896                 return -ENOMEM;
1897         }
1898
1899         nesvnic->lro_mgr.max_aggr       = nes_lro_max_aggr;
1900         nesvnic->lro_mgr.max_desc       = NES_MAX_LRO_DESCRIPTORS;
1901         nesvnic->lro_mgr.lro_arr        = nesvnic->lro_desc;
1902         nesvnic->lro_mgr.get_skb_header = nes_lro_get_skb_hdr;
1903         nesvnic->lro_mgr.features       = LRO_F_NAPI | LRO_F_EXTRACT_VLAN_ID;
1904         nesvnic->lro_mgr.dev            = netdev;
1905         nesvnic->lro_mgr.ip_summed      = CHECKSUM_UNNECESSARY;
1906         nesvnic->lro_mgr.ip_summed_aggr = CHECKSUM_UNNECESSARY;
1907         return 0;
1908 }
1909
1910
1911 /**
1912  * nes_destroy_nic_qp
1913  */
1914 void nes_destroy_nic_qp(struct nes_vnic *nesvnic)
1915 {
1916         u64 u64temp;
1917         dma_addr_t bus_address;
1918         struct nes_device *nesdev = nesvnic->nesdev;
1919         struct nes_hw_cqp_wqe *cqp_wqe;
1920         struct nes_hw_nic_sq_wqe *nic_sqe;
1921         __le16 *wqe_fragment_length;
1922         u16  wqe_fragment_index;
1923         u32 cqp_head;
1924         u32 wqm_cfg0;
1925         unsigned long flags;
1926         struct sk_buff *rx_skb;
1927         struct nes_rskb_cb *cb;
1928         int ret;
1929
1930         if (nesdev->nesadapter->allow_unaligned_fpdus)
1931                 nes_destroy_mgt(nesvnic);
1932
1933         /* clear wqe stall before destroying NIC QP */
1934         wqm_cfg0 = nes_read_indexed(nesdev, NES_IDX_WQM_CONFIG0);
1935         nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG0, wqm_cfg0 & 0xFFFF7FFF);
1936
1937         /* Free remaining NIC receive buffers */
1938         while (nesvnic->nic.rq_head != nesvnic->nic.rq_tail) {
1939                 rx_skb = nesvnic->nic.rx_skb[nesvnic->nic.rq_tail];
1940                 cb = (struct nes_rskb_cb *)&rx_skb->cb[0];
1941                 pci_unmap_single(nesdev->pcidev, cb->busaddr, cb->maplen,
1942                         PCI_DMA_FROMDEVICE);
1943
1944                 dev_kfree_skb(nesvnic->nic.rx_skb[nesvnic->nic.rq_tail++]);
1945                 nesvnic->nic.rq_tail &= (nesvnic->nic.rq_size - 1);
1946         }
1947
1948         /* Free remaining NIC transmit buffers */
1949         while (nesvnic->nic.sq_head != nesvnic->nic.sq_tail) {
1950                 nic_sqe = &nesvnic->nic.sq_vbase[nesvnic->nic.sq_tail];
1951                 wqe_fragment_index = 1;
1952                 wqe_fragment_length = (__le16 *)
1953                         &nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
1954                 /* bump past the vlan tag */
1955                 wqe_fragment_length++;
1956                 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
1957                         u64temp = (u64)le32_to_cpu(
1958                                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX+
1959                                 wqe_fragment_index*2]);
1960                         u64temp += ((u64)le32_to_cpu(
1961                                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX
1962                                 + wqe_fragment_index*2]))<<32;
1963                         bus_address = (dma_addr_t)u64temp;
1964                         if (test_and_clear_bit(nesvnic->nic.sq_tail,
1965                                         nesvnic->nic.first_frag_overflow)) {
1966                                 pci_unmap_single(nesdev->pcidev,
1967                                                 bus_address,
1968                                                 le16_to_cpu(wqe_fragment_length[
1969                                                         wqe_fragment_index++]),
1970                                                 PCI_DMA_TODEVICE);
1971                         }
1972                         for (; wqe_fragment_index < 5; wqe_fragment_index++) {
1973                                 if (wqe_fragment_length[wqe_fragment_index]) {
1974                                         u64temp = le32_to_cpu(
1975                                                 nic_sqe->wqe_words[
1976                                                 NES_NIC_SQ_WQE_FRAG0_LOW_IDX+
1977                                                 wqe_fragment_index*2]);
1978                                         u64temp += ((u64)le32_to_cpu(
1979                                                 nic_sqe->wqe_words[
1980                                                 NES_NIC_SQ_WQE_FRAG0_HIGH_IDX+
1981                                                 wqe_fragment_index*2]))<<32;
1982                                         bus_address = (dma_addr_t)u64temp;
1983                                         pci_unmap_page(nesdev->pcidev,
1984                                                         bus_address,
1985                                                         le16_to_cpu(
1986                                                         wqe_fragment_length[
1987                                                         wqe_fragment_index]),
1988                                                         PCI_DMA_TODEVICE);
1989                                 } else
1990                                         break;
1991                         }
1992                 }
1993                 if (nesvnic->nic.tx_skb[nesvnic->nic.sq_tail])
1994                         dev_kfree_skb(
1995                                 nesvnic->nic.tx_skb[nesvnic->nic.sq_tail]);
1996
1997                 nesvnic->nic.sq_tail = (nesvnic->nic.sq_tail + 1)
1998                                         & (nesvnic->nic.sq_size - 1);
1999         }
2000
2001         spin_lock_irqsave(&nesdev->cqp.lock, flags);
2002
2003         /* Destroy NIC QP */
2004         cqp_head = nesdev->cqp.sq_head;
2005         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
2006         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
2007
2008         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
2009                 (NES_CQP_DESTROY_QP | NES_CQP_QP_TYPE_NIC));
2010         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
2011                 nesvnic->nic.qp_id);
2012
2013         if (++cqp_head >= nesdev->cqp.sq_size)
2014                 cqp_head = 0;
2015
2016         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
2017
2018         /* Destroy NIC CQ */
2019         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
2020         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
2021                 (NES_CQP_DESTROY_CQ | ((u32)nesvnic->nic_cq.cq_size << 16)));
2022         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
2023                 (nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16)));
2024
2025         if (++cqp_head >= nesdev->cqp.sq_size)
2026                 cqp_head = 0;
2027
2028         nesdev->cqp.sq_head = cqp_head;
2029         barrier();
2030
2031         /* Ring doorbell (2 WQEs) */
2032         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
2033
2034         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2035         nes_debug(NES_DBG_SHUTDOWN, "Waiting for CQP, cqp_head=%u, cqp.sq_head=%u,"
2036                         " cqp.sq_tail=%u, cqp.sq_size=%u\n",
2037                         cqp_head, nesdev->cqp.sq_head,
2038                         nesdev->cqp.sq_tail, nesdev->cqp.sq_size);
2039
2040         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
2041                         NES_EVENT_TIMEOUT);
2042
2043         nes_debug(NES_DBG_SHUTDOWN, "Destroy NIC QP returned, wait_event_timeout ret = %u, cqp_head=%u,"
2044                         " cqp.sq_head=%u, cqp.sq_tail=%u\n",
2045                         ret, cqp_head, nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
2046         if (!ret) {
2047                 nes_debug(NES_DBG_SHUTDOWN, "NIC QP%u destroy timeout expired\n",
2048                                 nesvnic->nic.qp_id);
2049         }
2050
2051         pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
2052                         nesvnic->nic_pbase);
2053
2054         /* restore old wqm_cfg0 value */
2055         nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG0, wqm_cfg0);
2056 }
2057
2058 /**
2059  * nes_napi_isr
2060  */
2061 int nes_napi_isr(struct nes_device *nesdev)
2062 {
2063         struct nes_adapter *nesadapter = nesdev->nesadapter;
2064         u32 int_stat;
2065
2066         if (nesdev->napi_isr_ran) {
2067                 /* interrupt status has already been read in ISR */
2068                 int_stat = nesdev->int_stat;
2069         } else {
2070                 int_stat = nes_read32(nesdev->regs + NES_INT_STAT);
2071                 nesdev->int_stat = int_stat;
2072                 nesdev->napi_isr_ran = 1;
2073         }
2074
2075         int_stat &= nesdev->int_req;
2076         /* iff NIC, process here, else wait for DPC */
2077         if ((int_stat) && ((int_stat & 0x0000ff00) == int_stat)) {
2078                 nesdev->napi_isr_ran = 0;
2079                 nes_write32(nesdev->regs + NES_INT_STAT,
2080                         (int_stat &
2081                         ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 | NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
2082
2083                 /* Process the CEQs */
2084                 nes_process_ceq(nesdev, &nesdev->nesadapter->ceq[nesdev->nic_ceq_index]);
2085
2086                 if (unlikely((((nesadapter->et_rx_coalesce_usecs_irq) &&
2087                                         (!nesadapter->et_use_adaptive_rx_coalesce)) ||
2088                                         ((nesadapter->et_use_adaptive_rx_coalesce) &&
2089                                          (nesdev->deepcq_count > nesadapter->et_pkt_rate_low))))) {
2090                         if ((nesdev->int_req & NES_INT_TIMER) == 0) {
2091                                 /* Enable Periodic timer interrupts */
2092                                 nesdev->int_req |= NES_INT_TIMER;
2093                                 /* ack any pending periodic timer interrupts so we don't get an immediate interrupt */
2094                                 /* TODO: need to also ack other unused periodic timer values, get from nesadapter */
2095                                 nes_write32(nesdev->regs+NES_TIMER_STAT,
2096                                                 nesdev->timer_int_req  | ~(nesdev->nesadapter->timer_int_req));
2097                                 nes_write32(nesdev->regs+NES_INTF_INT_MASK,
2098                                                 ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
2099                         }
2100
2101                         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2102                         {
2103                                 nes_nic_init_timer(nesdev);
2104                         }
2105                         /* Enable interrupts, except CEQs */
2106                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2107                 } else {
2108                         /* Enable interrupts, make sure timer is off */
2109                         nesdev->int_req &= ~NES_INT_TIMER;
2110                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2111                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2112                 }
2113                 nesdev->deepcq_count = 0;
2114                 return 1;
2115         } else {
2116                 return 0;
2117         }
2118 }
2119
2120 static void process_critical_error(struct nes_device *nesdev)
2121 {
2122         u32 debug_error;
2123         u32 nes_idx_debug_error_masks0 = 0;
2124         u16 error_module = 0;
2125
2126         debug_error = nes_read_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS);
2127         printk(KERN_ERR PFX "Critical Error reported by device!!! 0x%02X\n",
2128                         (u16)debug_error);
2129         nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS,
2130                         0x01010000 | (debug_error & 0x0000ffff));
2131         if (crit_err_count++ > 10)
2132                 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS1, 1 << 0x17);
2133         error_module = (u16) (debug_error & 0x1F00) >> 8;
2134         if (++nesdev->nesadapter->crit_error_count[error_module-1] >=
2135                         nes_max_critical_error_count) {
2136                 printk(KERN_ERR PFX "Masking off critical error for module "
2137                         "0x%02X\n", (u16)error_module);
2138                 nes_idx_debug_error_masks0 = nes_read_indexed(nesdev,
2139                         NES_IDX_DEBUG_ERROR_MASKS0);
2140                 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS0,
2141                         nes_idx_debug_error_masks0 | (1 << error_module));
2142         }
2143 }
2144 /**
2145  * nes_dpc
2146  */
2147 void nes_dpc(unsigned long param)
2148 {
2149         struct nes_device *nesdev = (struct nes_device *)param;
2150         struct nes_adapter *nesadapter = nesdev->nesadapter;
2151         u32 counter;
2152         u32 loop_counter = 0;
2153         u32 int_status_bit;
2154         u32 int_stat;
2155         u32 timer_stat;
2156         u32 temp_int_stat;
2157         u32 intf_int_stat;
2158         u32 processed_intf_int = 0;
2159         u16 processed_timer_int = 0;
2160         u16 completion_ints = 0;
2161         u16 timer_ints = 0;
2162
2163         /* nes_debug(NES_DBG_ISR, "\n"); */
2164
2165         do {
2166                 timer_stat = 0;
2167                 if (nesdev->napi_isr_ran) {
2168                         nesdev->napi_isr_ran = 0;
2169                         int_stat = nesdev->int_stat;
2170                 } else
2171                         int_stat = nes_read32(nesdev->regs+NES_INT_STAT);
2172                 if (processed_intf_int != 0)
2173                         int_stat &= nesdev->int_req & ~NES_INT_INTF;
2174                 else
2175                         int_stat &= nesdev->int_req;
2176                 if (processed_timer_int == 0) {
2177                         processed_timer_int = 1;
2178                         if (int_stat & NES_INT_TIMER) {
2179                                 timer_stat = nes_read32(nesdev->regs + NES_TIMER_STAT);
2180                                 if ((timer_stat & nesdev->timer_int_req) == 0) {
2181                                         int_stat &= ~NES_INT_TIMER;
2182                                 }
2183                         }
2184                 } else {
2185                         int_stat &= ~NES_INT_TIMER;
2186                 }
2187
2188                 if (int_stat) {
2189                         if (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
2190                                         NES_INT_MAC1|NES_INT_MAC2 | NES_INT_MAC3)) {
2191                                 /* Ack the interrupts */
2192                                 nes_write32(nesdev->regs+NES_INT_STAT,
2193                                         (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
2194                                         NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
2195                         }
2196
2197                         temp_int_stat = int_stat;
2198                         for (counter = 0, int_status_bit = 1; counter < 16; counter++) {
2199                                 if (int_stat & int_status_bit) {
2200                                         nes_process_ceq(nesdev, &nesadapter->ceq[counter]);
2201                                         temp_int_stat &= ~int_status_bit;
2202                                         completion_ints = 1;
2203                                 }
2204                                 if (!(temp_int_stat & 0x0000ffff))
2205                                         break;
2206                                 int_status_bit <<= 1;
2207                         }
2208
2209                         /* Process the AEQ for this pci function */
2210                         int_status_bit = 1 << (16 + PCI_FUNC(nesdev->pcidev->devfn));
2211                         if (int_stat & int_status_bit) {
2212                                 nes_process_aeq(nesdev, &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)]);
2213                         }
2214
2215                         /* Process the MAC interrupt for this pci function */
2216                         int_status_bit = 1 << (24 + nesdev->mac_index);
2217                         if (int_stat & int_status_bit) {
2218                                 nes_process_mac_intr(nesdev, nesdev->mac_index);
2219                         }
2220
2221                         if (int_stat & NES_INT_TIMER) {
2222                                 if (timer_stat & nesdev->timer_int_req) {
2223                                         nes_write32(nesdev->regs + NES_TIMER_STAT,
2224                                                         (timer_stat & nesdev->timer_int_req) |
2225                                                         ~(nesdev->nesadapter->timer_int_req));
2226                                         timer_ints = 1;
2227                                 }
2228                         }
2229
2230                         if (int_stat & NES_INT_INTF) {
2231                                 processed_intf_int = 1;
2232                                 intf_int_stat = nes_read32(nesdev->regs+NES_INTF_INT_STAT);
2233                                 intf_int_stat &= nesdev->intf_int_req;
2234                                 if (NES_INTF_INT_CRITERR & intf_int_stat) {
2235                                         process_critical_error(nesdev);
2236                                 }
2237                                 if (NES_INTF_INT_PCIERR & intf_int_stat) {
2238                                         printk(KERN_ERR PFX "PCI Error reported by device!!!\n");
2239                                         BUG();
2240                                 }
2241                                 if (NES_INTF_INT_AEQ_OFLOW & intf_int_stat) {
2242                                         printk(KERN_ERR PFX "AEQ Overflow reported by device!!!\n");
2243                                         BUG();
2244                                 }
2245                                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, intf_int_stat);
2246                         }
2247
2248                         if (int_stat & NES_INT_TSW) {
2249                         }
2250                 }
2251                 /* Don't use the interface interrupt bit stay in loop */
2252                 int_stat &= ~NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 |
2253                                 NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3;
2254         } while ((int_stat != 0) && (loop_counter++ < MAX_DPC_ITERATIONS));
2255
2256         if (timer_ints == 1) {
2257                 if ((nesadapter->et_rx_coalesce_usecs_irq) || (nesadapter->et_use_adaptive_rx_coalesce)) {
2258                         if (completion_ints == 0) {
2259                                 nesdev->timer_only_int_count++;
2260                                 if (nesdev->timer_only_int_count>=nesadapter->timer_int_limit) {
2261                                         nesdev->timer_only_int_count = 0;
2262                                         nesdev->int_req &= ~NES_INT_TIMER;
2263                                         nes_write32(nesdev->regs + NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2264                                         nes_write32(nesdev->regs + NES_INT_MASK, ~nesdev->int_req);
2265                                 } else {
2266                                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2267                                 }
2268                         } else {
2269                                 if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2270                                 {
2271                                         nes_nic_init_timer(nesdev);
2272                                 }
2273                                 nesdev->timer_only_int_count = 0;
2274                                 nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2275                         }
2276                 } else {
2277                         nesdev->timer_only_int_count = 0;
2278                         nesdev->int_req &= ~NES_INT_TIMER;
2279                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2280                         nes_write32(nesdev->regs+NES_TIMER_STAT,
2281                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2282                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2283                 }
2284         } else {
2285                 if ( (completion_ints == 1) &&
2286                          (((nesadapter->et_rx_coalesce_usecs_irq) &&
2287                            (!nesadapter->et_use_adaptive_rx_coalesce)) ||
2288                           ((nesdev->deepcq_count > nesadapter->et_pkt_rate_low) &&
2289                            (nesadapter->et_use_adaptive_rx_coalesce) )) ) {
2290                         /* nes_debug(NES_DBG_ISR, "Enabling periodic timer interrupt.\n" ); */
2291                         nesdev->timer_only_int_count = 0;
2292                         nesdev->int_req |= NES_INT_TIMER;
2293                         nes_write32(nesdev->regs+NES_TIMER_STAT,
2294                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2295                         nes_write32(nesdev->regs+NES_INTF_INT_MASK,
2296                                         ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
2297                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2298                 } else {
2299                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2300                 }
2301         }
2302         nesdev->deepcq_count = 0;
2303 }
2304
2305
2306 /**
2307  * nes_process_ceq
2308  */
2309 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq)
2310 {
2311         u64 u64temp;
2312         struct nes_hw_cq *cq;
2313         u32 head;
2314         u32 ceq_size;
2315
2316         /* nes_debug(NES_DBG_CQ, "\n"); */
2317         head = ceq->ceq_head;
2318         ceq_size = ceq->ceq_size;
2319
2320         do {
2321                 if (le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]) &
2322                                 NES_CEQE_VALID) {
2323                         u64temp = (((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]))) << 32) |
2324                                                 ((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_LOW_IDX])));
2325                         u64temp <<= 1;
2326                         cq = *((struct nes_hw_cq **)&u64temp);
2327                         /* nes_debug(NES_DBG_CQ, "pCQ = %p\n", cq); */
2328                         barrier();
2329                         ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX] = 0;
2330
2331                         /* call the event handler */
2332                         cq->ce_handler(nesdev, cq);
2333
2334                         if (++head >= ceq_size)
2335                                 head = 0;
2336                 } else {
2337                         break;
2338                 }
2339
2340         } while (1);
2341
2342         ceq->ceq_head = head;
2343 }
2344
2345
2346 /**
2347  * nes_process_aeq
2348  */
2349 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq)
2350 {
2351         /* u64 u64temp; */
2352         u32 head;
2353         u32 aeq_size;
2354         u32 aeqe_misc;
2355         u32 aeqe_cq_id;
2356         struct nes_hw_aeqe volatile *aeqe;
2357
2358         head = aeq->aeq_head;
2359         aeq_size = aeq->aeq_size;
2360
2361         do {
2362                 aeqe = &aeq->aeq_vbase[head];
2363                 if ((le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]) & NES_AEQE_VALID) == 0)
2364                         break;
2365                 aeqe_misc  = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
2366                 aeqe_cq_id = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]);
2367                 if (aeqe_misc & (NES_AEQE_QP|NES_AEQE_CQ)) {
2368                         if (aeqe_cq_id >= NES_FIRST_QPN) {
2369                                 /* dealing with an accelerated QP related AE */
2370                                 /*
2371                                  * u64temp = (((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX]))) << 32) |
2372                                  *           ((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX])));
2373                                  */
2374                                 nes_process_iwarp_aeqe(nesdev, (struct nes_hw_aeqe *)aeqe);
2375                         } else {
2376                                 /* TODO: dealing with a CQP related AE */
2377                                 nes_debug(NES_DBG_AEQ, "Processing CQP related AE, misc = 0x%04X\n",
2378                                                 (u16)(aeqe_misc >> 16));
2379                         }
2380                 }
2381
2382                 aeqe->aeqe_words[NES_AEQE_MISC_IDX] = 0;
2383
2384                 if (++head >= aeq_size)
2385                         head = 0;
2386
2387                 nes_write32(nesdev->regs + NES_AEQ_ALLOC, 1 << 16);
2388         }
2389         while (1);
2390         aeq->aeq_head = head;
2391 }
2392
2393 static void nes_reset_link(struct nes_device *nesdev, u32 mac_index)
2394 {
2395         struct nes_adapter *nesadapter = nesdev->nesadapter;
2396         u32 reset_value;
2397         u32 i=0;
2398         u32 u32temp;
2399
2400         if (nesadapter->hw_rev == NE020_REV) {
2401                 return;
2402         }
2403         mh_detected++;
2404
2405         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
2406
2407         if ((mac_index == 0) || ((mac_index == 1) && (nesadapter->OneG_Mode)))
2408                 reset_value |= 0x0000001d;
2409         else
2410                 reset_value |= 0x0000002d;
2411
2412         if (4 <= (nesadapter->link_interrupt_count[mac_index] / ((u16)NES_MAX_LINK_INTERRUPTS))) {
2413                 if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
2414                         nesadapter->link_interrupt_count[0] = 0;
2415                         nesadapter->link_interrupt_count[1] = 0;
2416                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2417                         if (0x00000040 & u32temp)
2418                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2419                         else
2420                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2421
2422                         reset_value |= 0x0000003d;
2423                 }
2424                 nesadapter->link_interrupt_count[mac_index] = 0;
2425         }
2426
2427         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2428
2429         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
2430                         & 0x00000040) != 0x00000040) && (i++ < 5000));
2431
2432         if (0x0000003d == (reset_value & 0x0000003d)) {
2433                 u32 pcs_control_status0, pcs_control_status1;
2434
2435                 for (i = 0; i < 10; i++) {
2436                         pcs_control_status0 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0);
2437                         pcs_control_status1 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2438                         if (((0x0F000000 == (pcs_control_status0 & 0x0F000000))
2439                              && (pcs_control_status0 & 0x00100000))
2440                             || ((0x0F000000 == (pcs_control_status1 & 0x0F000000))
2441                                 && (pcs_control_status1 & 0x00100000)))
2442                                 continue;
2443                         else
2444                                 break;
2445                 }
2446                 if (10 == i) {
2447                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2448                         if (0x00000040 & u32temp)
2449                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2450                         else
2451                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2452
2453                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2454
2455                         while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET)
2456                                  & 0x00000040) != 0x00000040) && (i++ < 5000));
2457                 }
2458         }
2459 }
2460
2461 /**
2462  * nes_process_mac_intr
2463  */
2464 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number)
2465 {
2466         unsigned long flags;
2467         u32 pcs_control_status;
2468         struct nes_adapter *nesadapter = nesdev->nesadapter;
2469         struct nes_vnic *nesvnic;
2470         u32 mac_status;
2471         u32 mac_index = nesdev->mac_index;
2472         u32 u32temp;
2473         u16 phy_data;
2474         u16 temp_phy_data;
2475         u32 pcs_val  = 0x0f0f0000;
2476         u32 pcs_mask = 0x0f1f0000;
2477         u32 cdr_ctrl;
2478
2479         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2480         if (nesadapter->mac_sw_state[mac_number] != NES_MAC_SW_IDLE) {
2481                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2482                 return;
2483         }
2484         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_INTERRUPT;
2485
2486         /* ack the MAC interrupt */
2487         mac_status = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200));
2488         /* Clear the interrupt */
2489         nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200), mac_status);
2490
2491         nes_debug(NES_DBG_PHY, "MAC%u interrupt status = 0x%X.\n", mac_number, mac_status);
2492
2493         if (mac_status & (NES_MAC_INT_LINK_STAT_CHG | NES_MAC_INT_XGMII_EXT)) {
2494                 nesdev->link_status_interrupts++;
2495                 if (0 == (++nesadapter->link_interrupt_count[mac_index] % ((u16)NES_MAX_LINK_INTERRUPTS)))
2496                         nes_reset_link(nesdev, mac_index);
2497
2498                 /* read the PHY interrupt status register */
2499                 if ((nesadapter->OneG_Mode) &&
2500                 (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
2501                         do {
2502                                 nes_read_1G_phy_reg(nesdev, 0x1a,
2503                                                 nesadapter->phy_index[mac_index], &phy_data);
2504                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1a = 0x%X.\n",
2505                                                 nesadapter->phy_index[mac_index], phy_data);
2506                         } while (phy_data&0x8000);
2507
2508                         temp_phy_data = 0;
2509                         do {
2510                                 nes_read_1G_phy_reg(nesdev, 0x11,
2511                                                 nesadapter->phy_index[mac_index], &phy_data);
2512                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x11 = 0x%X.\n",
2513                                                 nesadapter->phy_index[mac_index], phy_data);
2514                                 if (temp_phy_data == phy_data)
2515                                         break;
2516                                 temp_phy_data = phy_data;
2517                         } while (1);
2518
2519                         nes_read_1G_phy_reg(nesdev, 0x1e,
2520                                         nesadapter->phy_index[mac_index], &phy_data);
2521                         nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1e = 0x%X.\n",
2522                                         nesadapter->phy_index[mac_index], phy_data);
2523
2524                         nes_read_1G_phy_reg(nesdev, 1,
2525                                         nesadapter->phy_index[mac_index], &phy_data);
2526                         nes_debug(NES_DBG_PHY, "1G phy%u data from register 1 = 0x%X\n",
2527                                         nesadapter->phy_index[mac_index], phy_data);
2528
2529                         if (temp_phy_data & 0x1000) {
2530                                 nes_debug(NES_DBG_PHY, "The Link is up according to the PHY\n");
2531                                 phy_data = 4;
2532                         } else {
2533                                 nes_debug(NES_DBG_PHY, "The Link is down according to the PHY\n");
2534                         }
2535                 }
2536                 nes_debug(NES_DBG_PHY, "Eth SERDES Common Status: 0=0x%08X, 1=0x%08X\n",
2537                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0),
2538                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0+0x200));
2539
2540                 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_PUMA_1G) {
2541                         switch (mac_index) {
2542                         case 1:
2543                         case 3:
2544                                 pcs_control_status = nes_read_indexed(nesdev,
2545                                                 NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2546                                 break;
2547                         default:
2548                                 pcs_control_status = nes_read_indexed(nesdev,
2549                                                 NES_IDX_PHY_PCS_CONTROL_STATUS0);
2550                                 break;
2551                         }
2552                 } else {
2553                         pcs_control_status = nes_read_indexed(nesdev,
2554                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2555                         pcs_control_status = nes_read_indexed(nesdev,
2556                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2557                 }
2558
2559                 nes_debug(NES_DBG_PHY, "PCS PHY Control/Status%u: 0x%08X\n",
2560                                 mac_index, pcs_control_status);
2561                 if ((nesadapter->OneG_Mode) &&
2562                                 (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
2563                         u32temp = 0x01010000;
2564                         if (nesadapter->port_count > 2) {
2565                                 u32temp |= 0x02020000;
2566                         }
2567                         if ((pcs_control_status & u32temp)!= u32temp) {
2568                                 phy_data = 0;
2569                                 nes_debug(NES_DBG_PHY, "PCS says the link is down\n");
2570                         }
2571                 } else {
2572                         switch (nesadapter->phy_type[mac_index]) {
2573                         case NES_PHY_TYPE_ARGUS:
2574                         case NES_PHY_TYPE_SFP_D:
2575                         case NES_PHY_TYPE_KR:
2576                                 /* clear the alarms */
2577                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0x0008);
2578                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc001);
2579                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc002);
2580                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc005);
2581                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc006);
2582                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2583                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9004);
2584                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9005);
2585                                 /* check link status */
2586                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2587                                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2588
2589                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2590                                 nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2591                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2592                                 phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2593
2594                                 phy_data = (!temp_phy_data && (phy_data == 0x8000)) ? 0x4 : 0x0;
2595
2596                                 nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2597                                         __func__, phy_data, nesadapter->mac_link_down[mac_index] ? "DOWN" : "UP");
2598                                 break;
2599
2600                         case NES_PHY_TYPE_PUMA_1G:
2601                                 if (mac_index < 2)
2602                                         pcs_val = pcs_mask = 0x01010000;
2603                                 else
2604                                         pcs_val = pcs_mask = 0x02020000;
2605                                 /* fall through */
2606                         default:
2607                                 phy_data = (pcs_val == (pcs_control_status & pcs_mask)) ? 0x4 : 0x0;
2608                                 break;
2609                         }
2610                 }
2611
2612                 if (phy_data & 0x0004) {
2613                         if (wide_ppm_offset &&
2614                             (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_CX4) &&
2615                             (nesadapter->hw_rev != NE020_REV)) {
2616                                 cdr_ctrl = nes_read_indexed(nesdev,
2617                                                             NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2618                                                             mac_index * 0x200);
2619                                 nes_write_indexed(nesdev,
2620                                                   NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2621                                                   mac_index * 0x200,
2622                                                   cdr_ctrl | 0x000F0000);
2623                         }
2624                         nesadapter->mac_link_down[mac_index] = 0;
2625                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2626                                 nes_debug(NES_DBG_PHY, "The Link is UP!!.  linkup was %d\n",
2627                                                 nesvnic->linkup);
2628                                 if (nesvnic->linkup == 0) {
2629                                         printk(PFX "The Link is now up for port %s, netdev %p.\n",
2630                                                         nesvnic->netdev->name, nesvnic->netdev);
2631                                         if (netif_queue_stopped(nesvnic->netdev))
2632                                                 netif_start_queue(nesvnic->netdev);
2633                                         nesvnic->linkup = 1;
2634                                         netif_carrier_on(nesvnic->netdev);
2635
2636                                         spin_lock(&nesvnic->port_ibevent_lock);
2637                                         if (nesvnic->of_device_registered) {
2638                                                 if (nesdev->iw_status == 0) {
2639                                                         nesdev->iw_status = 1;
2640                                                         nes_port_ibevent(nesvnic);
2641                                                 }
2642                                         }
2643                                         spin_unlock(&nesvnic->port_ibevent_lock);
2644                                 }
2645                         }
2646                 } else {
2647                         if (wide_ppm_offset &&
2648                             (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_CX4) &&
2649                             (nesadapter->hw_rev != NE020_REV)) {
2650                                 cdr_ctrl = nes_read_indexed(nesdev,
2651                                                             NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2652                                                             mac_index * 0x200);
2653                                 nes_write_indexed(nesdev,
2654                                                   NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2655                                                   mac_index * 0x200,
2656                                                   cdr_ctrl & 0xFFF0FFFF);
2657                         }
2658                         nesadapter->mac_link_down[mac_index] = 1;
2659                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2660                                 nes_debug(NES_DBG_PHY, "The Link is Down!!. linkup was %d\n",
2661                                                 nesvnic->linkup);
2662                                 if (nesvnic->linkup == 1) {
2663                                         printk(PFX "The Link is now down for port %s, netdev %p.\n",
2664                                                         nesvnic->netdev->name, nesvnic->netdev);
2665                                         if (!(netif_queue_stopped(nesvnic->netdev)))
2666                                                 netif_stop_queue(nesvnic->netdev);
2667                                         nesvnic->linkup = 0;
2668                                         netif_carrier_off(nesvnic->netdev);
2669
2670                                         spin_lock(&nesvnic->port_ibevent_lock);
2671                                         if (nesvnic->of_device_registered) {
2672                                                 if (nesdev->iw_status == 1) {
2673                                                         nesdev->iw_status = 0;
2674                                                         nes_port_ibevent(nesvnic);
2675                                                 }
2676                                         }
2677                                         spin_unlock(&nesvnic->port_ibevent_lock);
2678                                 }
2679                         }
2680                 }
2681                 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_SFP_D) {
2682                         if (nesdev->link_recheck)
2683                                 cancel_delayed_work(&nesdev->work);
2684                         nesdev->link_recheck = 1;
2685                         schedule_delayed_work(&nesdev->work,
2686                                               NES_LINK_RECHECK_DELAY);
2687                 }
2688         }
2689
2690         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2691
2692         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_IDLE;
2693 }
2694
2695 void nes_recheck_link_status(struct work_struct *work)
2696 {
2697         unsigned long flags;
2698         struct nes_device *nesdev = container_of(work, struct nes_device, work.work);
2699         struct nes_adapter *nesadapter = nesdev->nesadapter;
2700         struct nes_vnic *nesvnic;
2701         u32 mac_index = nesdev->mac_index;
2702         u16 phy_data;
2703         u16 temp_phy_data;
2704
2705         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2706
2707         /* check link status */
2708         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2709         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2710
2711         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2712         nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2713         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2714         phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2715
2716         phy_data = (!temp_phy_data && (phy_data == 0x8000)) ? 0x4 : 0x0;
2717
2718         nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2719                 __func__, phy_data,
2720                 nesadapter->mac_link_down[mac_index] ? "DOWN" : "UP");
2721
2722         if (phy_data & 0x0004) {
2723                 nesadapter->mac_link_down[mac_index] = 0;
2724                 list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2725                         if (nesvnic->linkup == 0) {
2726                                 printk(PFX "The Link is now up for port %s, netdev %p.\n",
2727                                                 nesvnic->netdev->name, nesvnic->netdev);
2728                                 if (netif_queue_stopped(nesvnic->netdev))
2729                                         netif_start_queue(nesvnic->netdev);
2730                                 nesvnic->linkup = 1;
2731                                 netif_carrier_on(nesvnic->netdev);
2732
2733                                 spin_lock(&nesvnic->port_ibevent_lock);
2734                                 if (nesvnic->of_device_registered) {
2735                                         if (nesdev->iw_status == 0) {
2736                                                 nesdev->iw_status = 1;
2737                                                 nes_port_ibevent(nesvnic);
2738                                         }
2739                                 }
2740                                 spin_unlock(&nesvnic->port_ibevent_lock);
2741                         }
2742                 }
2743
2744         } else {
2745                 nesadapter->mac_link_down[mac_index] = 1;
2746                 list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2747                         if (nesvnic->linkup == 1) {
2748                                 printk(PFX "The Link is now down for port %s, netdev %p.\n",
2749                                                 nesvnic->netdev->name, nesvnic->netdev);
2750                                 if (!(netif_queue_stopped(nesvnic->netdev)))
2751                                         netif_stop_queue(nesvnic->netdev);
2752                                 nesvnic->linkup = 0;
2753                                 netif_carrier_off(nesvnic->netdev);
2754
2755                                 spin_lock(&nesvnic->port_ibevent_lock);
2756                                 if (nesvnic->of_device_registered) {
2757                                         if (nesdev->iw_status == 1) {
2758                                                 nesdev->iw_status = 0;
2759                                                 nes_port_ibevent(nesvnic);
2760                                         }
2761                                 }
2762                                 spin_unlock(&nesvnic->port_ibevent_lock);
2763                         }
2764                 }
2765         }
2766         if (nesdev->link_recheck++ < NES_LINK_RECHECK_MAX)
2767                 schedule_delayed_work(&nesdev->work, NES_LINK_RECHECK_DELAY);
2768         else
2769                 nesdev->link_recheck = 0;
2770
2771         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2772 }
2773
2774
2775 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2776 {
2777         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2778
2779         napi_schedule(&nesvnic->napi);
2780 }
2781
2782
2783 /* The MAX_RQES_TO_PROCESS defines how many max read requests to complete before
2784 * getting out of nic_ce_handler
2785 */
2786 #define MAX_RQES_TO_PROCESS     384
2787
2788 /**
2789  * nes_nic_ce_handler
2790  */
2791 void nes_nic_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2792 {
2793         u64 u64temp;
2794         dma_addr_t bus_address;
2795         struct nes_hw_nic *nesnic;
2796         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2797         struct nes_adapter *nesadapter = nesdev->nesadapter;
2798         struct nes_hw_nic_rq_wqe *nic_rqe;
2799         struct nes_hw_nic_sq_wqe *nic_sqe;
2800         struct sk_buff *skb;
2801         struct sk_buff *rx_skb;
2802         struct nes_rskb_cb *cb;
2803         __le16 *wqe_fragment_length;
2804         u32 head;
2805         u32 cq_size;
2806         u32 rx_pkt_size;
2807         u32 cqe_count=0;
2808         u32 cqe_errv;
2809         u32 cqe_misc;
2810         u16 wqe_fragment_index = 1;     /* first fragment (0) is used by copy buffer */
2811         u16 vlan_tag;
2812         u16 pkt_type;
2813         u16 rqes_processed = 0;
2814         u8 sq_cqes = 0;
2815         u8 nes_use_lro = 0;
2816
2817         head = cq->cq_head;
2818         cq_size = cq->cq_size;
2819         cq->cqes_pending = 1;
2820         if (nesvnic->netdev->features & NETIF_F_LRO)
2821                 nes_use_lro = 1;
2822         do {
2823                 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]) &
2824                                 NES_NIC_CQE_VALID) {
2825                         nesnic = &nesvnic->nic;
2826                         cqe_misc = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]);
2827                         if (cqe_misc & NES_NIC_CQE_SQ) {
2828                                 sq_cqes++;
2829                                 wqe_fragment_index = 1;
2830                                 nic_sqe = &nesnic->sq_vbase[nesnic->sq_tail];
2831                                 skb = nesnic->tx_skb[nesnic->sq_tail];
2832                                 wqe_fragment_length = (__le16 *)&nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
2833                                 /* bump past the vlan tag */
2834                                 wqe_fragment_length++;
2835                                 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
2836                                         u64temp = (u64) le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
2837                                                         wqe_fragment_index * 2]);
2838                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX +
2839                                                         wqe_fragment_index * 2])) << 32;
2840                                         bus_address = (dma_addr_t)u64temp;
2841                                         if (test_and_clear_bit(nesnic->sq_tail, nesnic->first_frag_overflow)) {
2842                                                 pci_unmap_single(nesdev->pcidev,
2843                                                                 bus_address,
2844                                                                 le16_to_cpu(wqe_fragment_length[wqe_fragment_index++]),
2845                                                                 PCI_DMA_TODEVICE);
2846                                         }
2847                                         for (; wqe_fragment_index < 5; wqe_fragment_index++) {
2848                                                 if (wqe_fragment_length[wqe_fragment_index]) {
2849                                                         u64temp = le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
2850                                                                                 wqe_fragment_index * 2]);
2851                                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX
2852                                                                                 + wqe_fragment_index * 2])) <<32;
2853                                                         bus_address = (dma_addr_t)u64temp;
2854                                                         pci_unmap_page(nesdev->pcidev,
2855                                                                         bus_address,
2856                                                                         le16_to_cpu(wqe_fragment_length[wqe_fragment_index]),
2857                                                                         PCI_DMA_TODEVICE);
2858                                                 } else
2859                                                         break;
2860                                         }
2861                                 }
2862                                 if (skb)
2863                                         dev_kfree_skb_any(skb);
2864                                 nesnic->sq_tail++;
2865                                 nesnic->sq_tail &= nesnic->sq_size-1;
2866                                 if (sq_cqes > 128) {
2867                                         barrier();
2868                                         /* restart the queue if it had been stopped */
2869                                         if (netif_queue_stopped(nesvnic->netdev))
2870                                                 netif_wake_queue(nesvnic->netdev);
2871                                         sq_cqes = 0;
2872                                 }
2873                         } else {
2874                                 rqes_processed ++;
2875
2876                                 cq->rx_cqes_completed++;
2877                                 cq->rx_pkts_indicated++;
2878                                 rx_pkt_size = cqe_misc & 0x0000ffff;
2879                                 nic_rqe = &nesnic->rq_vbase[nesnic->rq_tail];
2880                                 /* Get the skb */
2881                                 rx_skb = nesnic->rx_skb[nesnic->rq_tail];
2882                                 nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_tail];
2883                                 bus_address = (dma_addr_t)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
2884                                 bus_address += ((u64)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX])) << 32;
2885                                 pci_unmap_single(nesdev->pcidev, bus_address,
2886                                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
2887                                 cb = (struct nes_rskb_cb *)&rx_skb->cb[0];
2888                                 cb->busaddr = 0;
2889                                 /* rx_skb->tail = rx_skb->data + rx_pkt_size; */
2890                                 /* rx_skb->len = rx_pkt_size; */
2891                                 rx_skb->len = 0;  /* TODO: see if this is necessary */
2892                                 skb_put(rx_skb, rx_pkt_size);
2893                                 rx_skb->protocol = eth_type_trans(rx_skb, nesvnic->netdev);
2894                                 nesnic->rq_tail++;
2895                                 nesnic->rq_tail &= nesnic->rq_size - 1;
2896
2897                                 atomic_inc(&nesvnic->rx_skbs_needed);
2898                                 if (atomic_read(&nesvnic->rx_skbs_needed) > (nesvnic->nic.rq_size>>1)) {
2899                                         nes_write32(nesdev->regs+NES_CQE_ALLOC,
2900                                                         cq->cq_number | (cqe_count << 16));
2901                                         /* nesadapter->tune_timer.cq_count += cqe_count; */
2902                                         nesdev->currcq_count += cqe_count;
2903                                         cqe_count = 0;
2904                                         nes_replenish_nic_rq(nesvnic);
2905                                 }
2906                                 pkt_type = (u16)(le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX]));
2907                                 cqe_errv = (cqe_misc & NES_NIC_CQE_ERRV_MASK) >> NES_NIC_CQE_ERRV_SHIFT;
2908                                 rx_skb->ip_summed = CHECKSUM_NONE;
2909
2910                                 if ((NES_PKT_TYPE_TCPV4_BITS == (pkt_type & NES_PKT_TYPE_TCPV4_MASK)) ||
2911                                                 (NES_PKT_TYPE_UDPV4_BITS == (pkt_type & NES_PKT_TYPE_UDPV4_MASK))) {
2912                                         if ((cqe_errv &
2913                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_TCPUDP_CSUM_ERR |
2914                                                         NES_NIC_ERRV_BITS_IPH_ERR | NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2915                                                 if (nesvnic->netdev->features & NETIF_F_RXCSUM)
2916                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2917                                         } else
2918                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2919                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2920                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2921
2922                                 } else if ((pkt_type & NES_PKT_TYPE_IPV4_MASK) == NES_PKT_TYPE_IPV4_BITS) {
2923                                         if ((cqe_errv &
2924                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_IPH_ERR |
2925                                                         NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2926                                                 if (nesvnic->netdev->features & NETIF_F_RXCSUM) {
2927                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2928                                                         /* nes_debug(NES_DBG_CQ, "%s: Reporting successfully checksummed IPv4 packet.\n",
2929                                                                   nesvnic->netdev->name); */
2930                                                 }
2931                                         } else
2932                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2933                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2934                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2935                                         }
2936                                 /* nes_debug(NES_DBG_CQ, "pkt_type=%x, APBVT_MASK=%x\n",
2937                                                         pkt_type, (pkt_type & NES_PKT_TYPE_APBVT_MASK)); */
2938
2939                                 if ((pkt_type & NES_PKT_TYPE_APBVT_MASK) == NES_PKT_TYPE_APBVT_BITS) {
2940                                         if (nes_cm_recv(rx_skb, nesvnic->netdev))
2941                                                 rx_skb = NULL;
2942                                 }
2943                                 if (rx_skb == NULL)
2944                                         goto skip_rx_indicate0;
2945
2946
2947                                 if (cqe_misc & NES_NIC_CQE_TAG_VALID) {
2948                                         vlan_tag = (u16)(le32_to_cpu(
2949                                                         cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX])
2950                                                         >> 16);
2951                                         nes_debug(NES_DBG_CQ, "%s: Reporting stripped VLAN packet. Tag = 0x%04X\n",
2952                                                         nesvnic->netdev->name, vlan_tag);
2953
2954                                         __vlan_hwaccel_put_tag(rx_skb, vlan_tag);
2955                                 }
2956                                 if (nes_use_lro)
2957                                         lro_receive_skb(&nesvnic->lro_mgr, rx_skb, NULL);
2958                                 else
2959                                         netif_receive_skb(rx_skb);
2960
2961 skip_rx_indicate0:
2962                                 ;
2963                                 /* nesvnic->netstats.rx_packets++; */
2964                                 /* nesvnic->netstats.rx_bytes += rx_pkt_size; */
2965                         }
2966
2967                         cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX] = 0;
2968                         /* Accounting... */
2969                         cqe_count++;
2970                         if (++head >= cq_size)
2971                                 head = 0;
2972                         if (cqe_count == 255) {
2973                                 /* Replenish Nic CQ */
2974                                 nes_write32(nesdev->regs+NES_CQE_ALLOC,
2975                                                 cq->cq_number | (cqe_count << 16));
2976                                 /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
2977                                 nesdev->currcq_count += cqe_count;
2978                                 cqe_count = 0;
2979                         }
2980
2981                         if (cq->rx_cqes_completed >= nesvnic->budget)
2982                                 break;
2983                 } else {
2984                         cq->cqes_pending = 0;
2985                         break;
2986                 }
2987
2988         } while (1);
2989
2990         if (nes_use_lro)
2991                 lro_flush_all(&nesvnic->lro_mgr);
2992         if (sq_cqes) {
2993                 barrier();
2994                 /* restart the queue if it had been stopped */
2995                 if (netif_queue_stopped(nesvnic->netdev))
2996                         netif_wake_queue(nesvnic->netdev);
2997         }
2998         cq->cq_head = head;
2999         /* nes_debug(NES_DBG_CQ, "CQ%u Processed = %u cqes, new head = %u.\n",
3000                         cq->cq_number, cqe_count, cq->cq_head); */
3001         cq->cqe_allocs_pending = cqe_count;
3002         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
3003         {
3004                 /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
3005                 nesdev->currcq_count += cqe_count;
3006                 nes_nic_tune_timer(nesdev);
3007         }
3008         if (atomic_read(&nesvnic->rx_skbs_needed))
3009                 nes_replenish_nic_rq(nesvnic);
3010 }
3011
3012
3013
3014 /**
3015  * nes_cqp_ce_handler
3016  */
3017 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq)
3018 {
3019         u64 u64temp;
3020         unsigned long flags;
3021         struct nes_hw_cqp *cqp = NULL;
3022         struct nes_cqp_request *cqp_request;
3023         struct nes_hw_cqp_wqe *cqp_wqe;
3024         u32 head;
3025         u32 cq_size;
3026         u32 cqe_count=0;
3027         u32 error_code;
3028         u32 opcode;
3029         u32 ctx_index;
3030         /* u32 counter; */
3031
3032         head = cq->cq_head;
3033         cq_size = cq->cq_size;
3034
3035         do {
3036                 /* process the CQE */
3037                 /* nes_debug(NES_DBG_CQP, "head=%u cqe_words=%08X\n", head,
3038                           le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])); */
3039
3040                 opcode = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX]);
3041                 if (opcode & NES_CQE_VALID) {
3042                         cqp = &nesdev->cqp;
3043
3044                         error_code = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_ERROR_CODE_IDX]);
3045                         if (error_code) {
3046                                 nes_debug(NES_DBG_CQP, "Bad Completion code for opcode 0x%02X from CQP,"
3047                                                 " Major/Minor codes = 0x%04X:%04X.\n",
3048                                                 le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])&0x3f,
3049                                                 (u16)(error_code >> 16),
3050                                                 (u16)error_code);
3051                         }
3052
3053                         u64temp = (((u64)(le32_to_cpu(cq->cq_vbase[head].
3054                                         cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX]))) << 32) |
3055                                         ((u64)(le32_to_cpu(cq->cq_vbase[head].
3056                                         cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX])));
3057
3058                         cqp_request = (struct nes_cqp_request *)(unsigned long)u64temp;
3059                         if (cqp_request) {
3060                                 if (cqp_request->waiting) {
3061                                         /* nes_debug(NES_DBG_CQP, "%s: Waking up requestor\n"); */
3062                                         cqp_request->major_code = (u16)(error_code >> 16);
3063                                         cqp_request->minor_code = (u16)error_code;
3064                                         barrier();
3065                                         cqp_request->request_done = 1;
3066                                         wake_up(&cqp_request->waitq);
3067                                         nes_put_cqp_request(nesdev, cqp_request);
3068                                 } else {
3069                                         if (cqp_request->callback)
3070                                                 cqp_request->cqp_callback(nesdev, cqp_request);
3071                                         nes_free_cqp_request(nesdev, cqp_request);
3072                                 }
3073                         } else {
3074                                 wake_up(&nesdev->cqp.waitq);
3075                         }
3076
3077                         cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX] = 0;
3078                         nes_write32(nesdev->regs + NES_CQE_ALLOC, cq->cq_number | (1 << 16));
3079                         if (++cqp->sq_tail >= cqp->sq_size)
3080                                 cqp->sq_tail = 0;
3081
3082                         /* Accounting... */
3083                         cqe_count++;
3084                         if (++head >= cq_size)
3085                                 head = 0;
3086                 } else {
3087                         break;
3088                 }
3089         } while (1);
3090         cq->cq_head = head;
3091
3092         spin_lock_irqsave(&nesdev->cqp.lock, flags);
3093         while ((!list_empty(&nesdev->cqp_pending_reqs)) &&
3094                         ((((nesdev->cqp.sq_tail+nesdev->cqp.sq_size)-nesdev->cqp.sq_head) &
3095                         (nesdev->cqp.sq_size - 1)) != 1)) {
3096                 cqp_request = list_entry(nesdev->cqp_pending_reqs.next,
3097                                 struct nes_cqp_request, list);
3098                 list_del_init(&cqp_request->list);
3099                 head = nesdev->cqp.sq_head++;
3100                 nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
3101                 cqp_wqe = &nesdev->cqp.sq_vbase[head];
3102                 memcpy(cqp_wqe, &cqp_request->cqp_wqe, sizeof(*cqp_wqe));
3103                 barrier();
3104
3105                 opcode = cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX];
3106                 if ((opcode & NES_CQP_OPCODE_MASK) == NES_CQP_DOWNLOAD_SEGMENT)
3107                         ctx_index = NES_CQP_WQE_DL_COMP_CTX_LOW_IDX;
3108                 else
3109                         ctx_index = NES_CQP_WQE_COMP_CTX_LOW_IDX;
3110                 cqp_wqe->wqe_words[ctx_index] =
3111                         cpu_to_le32((u32)((unsigned long)cqp_request));
3112                 cqp_wqe->wqe_words[ctx_index + 1] =
3113                         cpu_to_le32((u32)(upper_32_bits((unsigned long)cqp_request)));
3114                 nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) put on CQPs SQ wqe%u.\n",
3115                                 cqp_request, le32_to_cpu(cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX])&0x3f, head);
3116                 /* Ring doorbell (1 WQEs) */
3117                 barrier();
3118                 nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x01800000 | nesdev->cqp.qp_id);
3119         }
3120         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
3121
3122         /* Arm the CCQ */
3123         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_NOTIFY_NEXT |
3124                         cq->cq_number);
3125         nes_read32(nesdev->regs+NES_CQE_ALLOC);
3126 }
3127
3128 static u8 *locate_mpa(u8 *pkt, u32 aeq_info)
3129 {
3130         if (aeq_info & NES_AEQE_Q2_DATA_ETHERNET) {
3131                 /* skip over ethernet header */
3132                 pkt += ETH_HLEN;
3133
3134                 /* Skip over IP and TCP headers */
3135                 pkt += 4 * (pkt[0] & 0x0f);
3136                 pkt += 4 * ((pkt[12] >> 4) & 0x0f);
3137         }
3138         return pkt;
3139 }
3140
3141 /* Determine if incoming error pkt is rdma layer */
3142 static u32 iwarp_opcode(struct nes_qp *nesqp, u32 aeq_info)
3143 {
3144         u8 *pkt;
3145         u16 *mpa;
3146         u32 opcode = 0xffffffff;
3147
3148         if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) {
3149                 pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET;
3150                 mpa = (u16 *)locate_mpa(pkt, aeq_info);
3151                 opcode = be16_to_cpu(mpa[1]) & 0xf;
3152         }
3153
3154         return opcode;
3155 }
3156
3157 /* Build iWARP terminate header */
3158 static int nes_bld_terminate_hdr(struct nes_qp *nesqp, u16 async_event_id, u32 aeq_info)
3159 {
3160         u8 *pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET;
3161         u16 ddp_seg_len;
3162         int copy_len = 0;
3163         u8 is_tagged = 0;
3164         u8 flush_code = 0;
3165         struct nes_terminate_hdr *termhdr;
3166
3167         termhdr = (struct nes_terminate_hdr *)nesqp->hwqp.q2_vbase;
3168         memset(termhdr, 0, 64);
3169
3170         if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) {
3171
3172                 /* Use data from offending packet to fill in ddp & rdma hdrs */
3173                 pkt = locate_mpa(pkt, aeq_info);
3174                 ddp_seg_len = be16_to_cpu(*(u16 *)pkt);
3175                 if (ddp_seg_len) {
3176                         copy_len = 2;
3177                         termhdr->hdrct = DDP_LEN_FLAG;
3178                         if (pkt[2] & 0x80) {
3179                                 is_tagged = 1;
3180                                 if (ddp_seg_len >= TERM_DDP_LEN_TAGGED) {
3181                                         copy_len += TERM_DDP_LEN_TAGGED;
3182                                         termhdr->hdrct |= DDP_HDR_FLAG;
3183                                 }
3184                         } else {
3185                                 if (ddp_seg_len >= TERM_DDP_LEN_UNTAGGED) {
3186                                         copy_len += TERM_DDP_LEN_UNTAGGED;
3187                                         termhdr->hdrct |= DDP_HDR_FLAG;
3188                                 }
3189
3190                                 if (ddp_seg_len >= (TERM_DDP_LEN_UNTAGGED + TERM_RDMA_LEN)) {
3191                                         if ((pkt[3] & RDMA_OPCODE_MASK) == RDMA_READ_REQ_OPCODE) {
3192                                                 copy_len += TERM_RDMA_LEN;
3193                                                 termhdr->hdrct |= RDMA_HDR_FLAG;
3194                                         }
3195                                 }
3196                         }
3197                 }
3198         }
3199
3200         switch (async_event_id) {
3201         case NES_AEQE_AEID_AMP_UNALLOCATED_STAG:
3202                 switch (iwarp_opcode(nesqp, aeq_info)) {
3203                 case IWARP_OPCODE_WRITE:
3204                         flush_code = IB_WC_LOC_PROT_ERR;
3205                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3206                         termhdr->error_code = DDP_TAGGED_INV_STAG;
3207                         break;
3208                 default:
3209                         flush_code = IB_WC_REM_ACCESS_ERR;
3210                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3211                         termhdr->error_code = RDMAP_INV_STAG;
3212                 }
3213                 break;
3214         case NES_AEQE_AEID_AMP_INVALID_STAG:
3215                 flush_code = IB_WC_REM_ACCESS_ERR;
3216                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3217                 termhdr->error_code = RDMAP_INV_STAG;
3218                 break;
3219         case NES_AEQE_AEID_AMP_BAD_QP:
3220                 flush_code = IB_WC_LOC_QP_OP_ERR;
3221                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3222                 termhdr->error_code = DDP_UNTAGGED_INV_QN;
3223                 break;
3224         case NES_AEQE_AEID_AMP_BAD_STAG_KEY:
3225         case NES_AEQE_AEID_AMP_BAD_STAG_INDEX:
3226                 switch (iwarp_opcode(nesqp, aeq_info)) {
3227                 case IWARP_OPCODE_SEND_INV:
3228                 case IWARP_OPCODE_SEND_SE_INV:
3229                         flush_code = IB_WC_REM_OP_ERR;
3230                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3231                         termhdr->error_code = RDMAP_CANT_INV_STAG;
3232                         break;
3233                 default:
3234                         flush_code = IB_WC_REM_ACCESS_ERR;
3235                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3236                         termhdr->error_code = RDMAP_INV_STAG;
3237                 }
3238                 break;
3239         case NES_AEQE_AEID_AMP_BOUNDS_VIOLATION:
3240                 if (aeq_info & (NES_AEQE_Q2_DATA_ETHERNET | NES_AEQE_Q2_DATA_MPA)) {
3241                         flush_code = IB_WC_LOC_PROT_ERR;
3242                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3243                         termhdr->error_code = DDP_TAGGED_BOUNDS;
3244                 } else {
3245                         flush_code = IB_WC_REM_ACCESS_ERR;
3246                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3247                         termhdr->error_code = RDMAP_INV_BOUNDS;
3248                 }
3249                 break;
3250         case NES_AEQE_AEID_AMP_RIGHTS_VIOLATION:
3251         case NES_AEQE_AEID_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:
3252         case NES_AEQE_AEID_PRIV_OPERATION_DENIED:
3253                 flush_code = IB_WC_REM_ACCESS_ERR;
3254                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3255                 termhdr->error_code = RDMAP_ACCESS;
3256                 break;
3257         case NES_AEQE_AEID_AMP_TO_WRAP:
3258                 flush_code = IB_WC_REM_ACCESS_ERR;
3259                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3260                 termhdr->error_code = RDMAP_TO_WRAP;
3261                 break;
3262         case NES_AEQE_AEID_AMP_BAD_PD:
3263                 switch (iwarp_opcode(nesqp, aeq_info)) {
3264                 case IWARP_OPCODE_WRITE:
3265                         flush_code = IB_WC_LOC_PROT_ERR;
3266                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3267                         termhdr->error_code = DDP_TAGGED_UNASSOC_STAG;
3268                         break;
3269                 case IWARP_OPCODE_SEND_INV:
3270                 case IWARP_OPCODE_SEND_SE_INV:
3271                         flush_code = IB_WC_REM_ACCESS_ERR;
3272                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3273                         termhdr->error_code = RDMAP_CANT_INV_STAG;
3274                         break;
3275                 default:
3276                         flush_code = IB_WC_REM_ACCESS_ERR;
3277                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3278                         termhdr->error_code = RDMAP_UNASSOC_STAG;
3279                 }
3280                 break;
3281         case NES_AEQE_AEID_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH:
3282                 flush_code = IB_WC_LOC_LEN_ERR;
3283                 termhdr->layer_etype = (LAYER_MPA << 4) | DDP_LLP;
3284                 termhdr->error_code = MPA_MARKER;
3285                 break;
3286         case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR:
3287                 flush_code = IB_WC_GENERAL_ERR;
3288                 termhdr->layer_etype = (LAYER_MPA << 4) | DDP_LLP;
3289                 termhdr->error_code = MPA_CRC;
3290                 break;
3291         case NES_AEQE_AEID_LLP_SEGMENT_TOO_LARGE:
3292         case NES_AEQE_AEID_LLP_SEGMENT_TOO_SMALL:
3293                 flush_code = IB_WC_LOC_LEN_ERR;
3294                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_CATASTROPHIC;
3295                 termhdr->error_code = DDP_CATASTROPHIC_LOCAL;
3296                 break;
3297         case NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC:
3298         case NES_AEQE_AEID_DDP_NO_L_BIT:
3299                 flush_code = IB_WC_FATAL_ERR;
3300                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_CATASTROPHIC;
3301                 termhdr->error_code = DDP_CATASTROPHIC_LOCAL;
3302                 break;
3303         case NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN:
3304         case NES_AEQE_AEID_DDP_INVALID_MSN_RANGE_IS_NOT_VALID:
3305                 flush_code = IB_WC_GENERAL_ERR;
3306                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3307                 termhdr->error_code = DDP_UNTAGGED_INV_MSN_RANGE;
3308                 break;
3309         case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
3310                 flush_code = IB_WC_LOC_LEN_ERR;
3311                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3312                 termhdr->error_code = DDP_UNTAGGED_INV_TOO_LONG;
3313                 break;
3314         case NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION:
3315                 flush_code = IB_WC_GENERAL_ERR;
3316                 if (is_tagged) {
3317                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3318                         termhdr->error_code = DDP_TAGGED_INV_DDP_VER;
3319                 } else {
3320                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3321                         termhdr->error_code = DDP_UNTAGGED_INV_DDP_VER;
3322                 }
3323                 break;
3324         case NES_AEQE_AEID_DDP_UBE_INVALID_MO:
3325                 flush_code = IB_WC_GENERAL_ERR;
3326                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3327                 termhdr->error_code = DDP_UNTAGGED_INV_MO;
3328                 break;
3329         case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
3330                 flush_code = IB_WC_REM_OP_ERR;
3331                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3332                 termhdr->error_code = DDP_UNTAGGED_INV_MSN_NO_BUF;
3333                 break;
3334         case NES_AEQE_AEID_DDP_UBE_INVALID_QN:
3335                 flush_code = IB_WC_GENERAL_ERR;
3336                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3337                 termhdr->error_code = DDP_UNTAGGED_INV_QN;
3338                 break;
3339         case NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION:
3340                 flush_code = IB_WC_GENERAL_ERR;
3341                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3342                 termhdr->error_code = RDMAP_INV_RDMAP_VER;
3343                 break;
3344         case NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE:
3345                 flush_code = IB_WC_LOC_QP_OP_ERR;
3346                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3347                 termhdr->error_code = RDMAP_UNEXPECTED_OP;
3348                 break;
3349         default:
3350                 flush_code = IB_WC_FATAL_ERR;
3351                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3352                 termhdr->error_code = RDMAP_UNSPECIFIED;
3353                 break;
3354         }
3355
3356         if (copy_len)
3357                 memcpy(termhdr + 1, pkt, copy_len);
3358
3359         if ((flush_code) && ((NES_AEQE_INBOUND_RDMA & aeq_info) == 0)) {
3360                 if (aeq_info & NES_AEQE_SQ)
3361                         nesqp->term_sq_flush_code = flush_code;
3362                 else
3363                         nesqp->term_rq_flush_code = flush_code;
3364         }
3365
3366         return sizeof(struct nes_terminate_hdr) + copy_len;
3367 }
3368
3369 static void nes_terminate_connection(struct nes_device *nesdev, struct nes_qp *nesqp,
3370                  struct nes_hw_aeqe *aeqe, enum ib_event_type eventtype)
3371 {
3372         u64 context;
3373         unsigned long flags;
3374         u32 aeq_info;
3375         u16 async_event_id;
3376         u8 tcp_state;
3377         u8 iwarp_state;
3378         u32 termlen = 0;
3379         u32 mod_qp_flags = NES_CQP_QP_IWARP_STATE_TERMINATE |
3380                            NES_CQP_QP_TERM_DONT_SEND_FIN;
3381         struct nes_adapter *nesadapter = nesdev->nesadapter;
3382
3383         if (nesqp->term_flags & NES_TERM_SENT)
3384                 return; /* Sanity check */
3385
3386         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3387         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
3388         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
3389         async_event_id = (u16)aeq_info;
3390
3391         context = (unsigned long)nesadapter->qp_table[le32_to_cpu(
3392                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]) - NES_FIRST_QPN];
3393         if (!context) {
3394                 WARN_ON(!context);
3395                 return;
3396         }
3397
3398         nesqp = (struct nes_qp *)(unsigned long)context;
3399         spin_lock_irqsave(&nesqp->lock, flags);
3400         nesqp->hw_iwarp_state = iwarp_state;
3401         nesqp->hw_tcp_state = tcp_state;
3402         nesqp->last_aeq = async_event_id;
3403         nesqp->terminate_eventtype = eventtype;
3404         spin_unlock_irqrestore(&nesqp->lock, flags);
3405
3406         if (nesadapter->send_term_ok)
3407                 termlen = nes_bld_terminate_hdr(nesqp, async_event_id, aeq_info);
3408         else
3409                 mod_qp_flags |= NES_CQP_QP_TERM_DONT_SEND_TERM_MSG;
3410
3411         if (!nesdev->iw_status)  {
3412                 nesqp->term_flags = NES_TERM_DONE;
3413                 nes_hw_modify_qp(nesdev, nesqp, NES_CQP_QP_IWARP_STATE_ERROR, 0, 0);
3414                 nes_cm_disconn(nesqp);
3415         } else {
3416                 nes_terminate_start_timer(nesqp);
3417                 nesqp->term_flags |= NES_TERM_SENT;
3418                 nes_hw_modify_qp(nesdev, nesqp, mod_qp_flags, termlen, 0);
3419         }
3420 }
3421
3422 static void nes_terminate_send_fin(struct nes_device *nesdev,
3423                           struct nes_qp *nesqp, struct nes_hw_aeqe *aeqe)
3424 {
3425         u32 aeq_info;
3426         u16 async_event_id;
3427         u8 tcp_state;
3428         u8 iwarp_state;
3429         unsigned long flags;
3430
3431         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3432         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
3433         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
3434         async_event_id = (u16)aeq_info;
3435
3436         spin_lock_irqsave(&nesqp->lock, flags);
3437         nesqp->hw_iwarp_state = iwarp_state;
3438         nesqp->hw_tcp_state = tcp_state;
3439         nesqp->last_aeq = async_event_id;
3440         spin_unlock_irqrestore(&nesqp->lock, flags);
3441
3442         /* Send the fin only */
3443         nes_hw_modify_qp(nesdev, nesqp, NES_CQP_QP_IWARP_STATE_TERMINATE |
3444                 NES_CQP_QP_TERM_DONT_SEND_TERM_MSG, 0, 0);
3445 }
3446
3447 /* Cleanup after a terminate sent or received */
3448 static void nes_terminate_done(struct nes_qp *nesqp, int timeout_occurred)
3449 {
3450         u32 next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR;
3451         unsigned long flags;
3452         struct nes_vnic *nesvnic = to_nesvnic(nesqp->ibqp.device);
3453         struct nes_device *nesdev = nesvnic->nesdev;
3454         u8 first_time = 0;
3455
3456         spin_lock_irqsave(&nesqp->lock, flags);
3457         if (nesqp->hte_added) {
3458                 nesqp->hte_added = 0;
3459                 next_iwarp_state |= NES_CQP_QP_DEL_HTE;
3460         }
3461
3462         first_time = (nesqp->term_flags & NES_TERM_DONE) == 0;
3463         nesqp->term_flags |= NES_TERM_DONE;
3464         spin_unlock_irqrestore(&nesqp->lock, flags);
3465
3466         /* Make sure we go through this only once */
3467         if (first_time) {
3468                 if (timeout_occurred == 0)
3469                         del_timer(&nesqp->terminate_timer);
3470                 else
3471                         next_iwarp_state |= NES_CQP_QP_RESET;
3472
3473                 nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0);
3474                 nes_cm_disconn(nesqp);
3475         }
3476 }
3477
3478 static void nes_terminate_received(struct nes_device *nesdev,
3479                                 struct nes_qp *nesqp, struct nes_hw_aeqe *aeqe)
3480 {
3481         u32 aeq_info;
3482         u8 *pkt;
3483         u32 *mpa;
3484         u8 ddp_ctl;
3485         u8 rdma_ctl;
3486         u16 aeq_id = 0;
3487
3488         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3489         if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) {
3490                 /* Terminate is not a performance path so the silicon */
3491                 /* did not validate the frame - do it now */
3492                 pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET;
3493                 mpa = (u32 *)locate_mpa(pkt, aeq_info);
3494                 ddp_ctl = (be32_to_cpu(mpa[0]) >> 8) & 0xff;
3495                 rdma_ctl = be32_to_cpu(mpa[0]) & 0xff;
3496                 if ((ddp_ctl & 0xc0) != 0x40)
3497                         aeq_id = NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC;
3498                 else if ((ddp_ctl & 0x03) != 1)
3499                         aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION;
3500                 else if (be32_to_cpu(mpa[2]) != 2)
3501                         aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_QN;
3502                 else if (be32_to_cpu(mpa[3]) != 1)
3503                         aeq_id = NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN;
3504                 else if (be32_to_cpu(mpa[4]) != 0)
3505                         aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_MO;
3506                 else if ((rdma_ctl & 0xc0) != 0x40)
3507                         aeq_id = NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION;
3508
3509                 if (aeq_id) {
3510                         /* Bad terminate recvd - send back a terminate */
3511                         aeq_info = (aeq_info & 0xffff0000) | aeq_id;
3512                         aeqe->aeqe_words[NES_AEQE_MISC_IDX] = cpu_to_le32(aeq_info);
3513                         nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_FATAL);
3514                         return;
3515                 }
3516         }
3517
3518         nesqp->term_flags |= NES_TERM_RCVD;
3519         nesqp->terminate_eventtype = IB_EVENT_QP_FATAL;
3520         nes_terminate_start_timer(nesqp);
3521         nes_terminate_send_fin(nesdev, nesqp, aeqe);
3522 }
3523
3524 /* Timeout routine in case terminate fails to complete */
3525 static void nes_terminate_timeout(unsigned long context)
3526 {
3527         struct nes_qp *nesqp = (struct nes_qp *)(unsigned long)context;
3528
3529         nes_terminate_done(nesqp, 1);
3530 }
3531
3532 /* Set a timer in case hw cannot complete the terminate sequence */
3533 static void nes_terminate_start_timer(struct nes_qp *nesqp)
3534 {
3535         init_timer(&nesqp->terminate_timer);
3536         nesqp->terminate_timer.function = nes_terminate_timeout;
3537         nesqp->terminate_timer.expires = jiffies + HZ;
3538         nesqp->terminate_timer.data = (unsigned long)nesqp;
3539         add_timer(&nesqp->terminate_timer);
3540 }
3541
3542 /**
3543  * nes_process_iwarp_aeqe
3544  */
3545 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
3546                                    struct nes_hw_aeqe *aeqe)
3547 {
3548         u64 context;
3549         unsigned long flags;
3550         struct nes_qp *nesqp;
3551         struct nes_hw_cq *hw_cq;
3552         struct nes_cq *nescq;
3553         int resource_allocated;
3554         struct nes_adapter *nesadapter = nesdev->nesadapter;
3555         u32 aeq_info;
3556         u32 next_iwarp_state = 0;
3557         u32 aeqe_cq_id;
3558         u16 async_event_id;
3559         u8 tcp_state;
3560         u8 iwarp_state;
3561         struct ib_event ibevent;
3562
3563         nes_debug(NES_DBG_AEQ, "\n");
3564         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3565         if ((NES_AEQE_INBOUND_RDMA & aeq_info) || (!(NES_AEQE_QP & aeq_info))) {
3566                 context  = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]);
3567                 context += ((u64)le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])) << 32;
3568         } else {
3569                 context = (unsigned long)nesadapter->qp_table[le32_to_cpu(
3570                                                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]) - NES_FIRST_QPN];
3571                 BUG_ON(!context);
3572         }
3573
3574         /* context is nesqp unless async_event_id == CQ ERROR */
3575         nesqp = (struct nes_qp *)(unsigned long)context;
3576         async_event_id = (u16)aeq_info;
3577         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
3578         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
3579         nes_debug(NES_DBG_AEQ, "aeid = 0x%04X, qp-cq id = %d, aeqe = %p,"
3580                         " Tcp state = %d, iWARP state = %d\n",
3581                         async_event_id,
3582                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), aeqe,
3583                         tcp_state, iwarp_state);
3584
3585         aeqe_cq_id = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]);
3586         if (aeq_info & NES_AEQE_QP) {
3587                 if (!nes_is_resource_allocated(nesadapter,
3588                                 nesadapter->allocated_qps,
3589                                 aeqe_cq_id))
3590                         return;
3591         }
3592
3593         switch (async_event_id) {
3594                 case NES_AEQE_AEID_LLP_FIN_RECEIVED:
3595                         if (nesqp->term_flags)
3596                                 return; /* Ignore it, wait for close complete */
3597
3598                         if (atomic_inc_return(&nesqp->close_timer_started) == 1) {
3599                                 if ((tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) &&
3600                                         (nesqp->ibqp_state == IB_QPS_RTS)) {
3601                                         spin_lock_irqsave(&nesqp->lock, flags);
3602                                         nesqp->hw_iwarp_state = iwarp_state;
3603                                         nesqp->hw_tcp_state = tcp_state;
3604                                         nesqp->last_aeq = async_event_id;
3605                                         next_iwarp_state = NES_CQP_QP_IWARP_STATE_CLOSING;
3606                                         nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_CLOSING;
3607                                         spin_unlock_irqrestore(&nesqp->lock, flags);
3608                                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0);
3609                                         nes_cm_disconn(nesqp);
3610                                 }
3611                                 nesqp->cm_id->add_ref(nesqp->cm_id);
3612                                 schedule_nes_timer(nesqp->cm_node, (struct sk_buff *)nesqp,
3613                                                 NES_TIMER_TYPE_CLOSE, 1, 0);
3614                                 nes_debug(NES_DBG_AEQ, "QP%u Not decrementing QP refcount (%d),"
3615                                                 " need ae to finish up, original_last_aeq = 0x%04X."
3616                                                 " last_aeq = 0x%04X, scheduling timer. TCP state = %d\n",
3617                                                 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),
3618                                                 async_event_id, nesqp->last_aeq, tcp_state);
3619                         }
3620                         break;
3621                 case NES_AEQE_AEID_LLP_CLOSE_COMPLETE:
3622                         spin_lock_irqsave(&nesqp->lock, flags);
3623                         nesqp->hw_iwarp_state = iwarp_state;
3624                         nesqp->hw_tcp_state = tcp_state;
3625                         nesqp->last_aeq = async_event_id;
3626                         spin_unlock_irqrestore(&nesqp->lock, flags);
3627                         nes_cm_disconn(nesqp);
3628                         break;
3629
3630                 case NES_AEQE_AEID_RESET_SENT:
3631                         tcp_state = NES_AEQE_TCP_STATE_CLOSED;
3632                         spin_lock_irqsave(&nesqp->lock, flags);
3633                         nesqp->hw_iwarp_state = iwarp_state;
3634                         nesqp->hw_tcp_state = tcp_state;
3635                         nesqp->last_aeq = async_event_id;
3636                         nesqp->hte_added = 0;
3637                         spin_unlock_irqrestore(&nesqp->lock, flags);
3638                         next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_DEL_HTE;
3639                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0);
3640                         nes_cm_disconn(nesqp);
3641                         break;
3642
3643                 case NES_AEQE_AEID_LLP_CONNECTION_RESET:
3644                         if (atomic_read(&nesqp->close_timer_started))
3645                                 return;
3646                         spin_lock_irqsave(&nesqp->lock, flags);
3647                         nesqp->hw_iwarp_state = iwarp_state;
3648                         nesqp->hw_tcp_state = tcp_state;
3649                         nesqp->last_aeq = async_event_id;
3650                         spin_unlock_irqrestore(&nesqp->lock, flags);
3651                         nes_cm_disconn(nesqp);
3652                         break;
3653
3654                 case NES_AEQE_AEID_TERMINATE_SENT:
3655                         nes_terminate_send_fin(nesdev, nesqp, aeqe);
3656                         break;
3657
3658                 case NES_AEQE_AEID_LLP_TERMINATE_RECEIVED:
3659                         nes_terminate_received(nesdev, nesqp, aeqe);
3660                         break;
3661
3662                 case NES_AEQE_AEID_AMP_BAD_STAG_KEY:
3663                 case NES_AEQE_AEID_AMP_BAD_STAG_INDEX:
3664                 case NES_AEQE_AEID_AMP_UNALLOCATED_STAG:
3665                 case NES_AEQE_AEID_AMP_INVALID_STAG:
3666                 case NES_AEQE_AEID_AMP_RIGHTS_VIOLATION:
3667                 case NES_AEQE_AEID_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:
3668                 case NES_AEQE_AEID_PRIV_OPERATION_DENIED:
3669                 case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
3670                 case NES_AEQE_AEID_AMP_BOUNDS_VIOLATION:
3671                 case NES_AEQE_AEID_AMP_TO_WRAP:
3672                         printk(KERN_ERR PFX "QP[%u] async_event_id=0x%04X IB_EVENT_QP_ACCESS_ERR\n",
3673                                         nesqp->hwqp.qp_id, async_event_id);
3674                         nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_ACCESS_ERR);
3675                         break;
3676
3677                 case NES_AEQE_AEID_LLP_SEGMENT_TOO_LARGE:
3678                 case NES_AEQE_AEID_LLP_SEGMENT_TOO_SMALL:
3679                 case NES_AEQE_AEID_DDP_UBE_INVALID_MO:
3680                 case NES_AEQE_AEID_DDP_UBE_INVALID_QN:
3681                         if (iwarp_opcode(nesqp, aeq_info) > IWARP_OPCODE_TERM) {
3682                                 aeq_info &= 0xffff0000;
3683                                 aeq_info |= NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE;
3684                                 aeqe->aeqe_words[NES_AEQE_MISC_IDX] = cpu_to_le32(aeq_info);
3685                         }
3686
3687                 case NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE:
3688                 case NES_AEQE_AEID_LLP_TOO_MANY_RETRIES:
3689                 case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
3690                 case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR:
3691                 case NES_AEQE_AEID_AMP_BAD_QP:
3692                 case NES_AEQE_AEID_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH:
3693                 case NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC:
3694                 case NES_AEQE_AEID_DDP_NO_L_BIT:
3695                 case NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN:
3696                 case NES_AEQE_AEID_DDP_INVALID_MSN_RANGE_IS_NOT_VALID:
3697                 case NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION:
3698                 case NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION:
3699                 case NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE:
3700                 case NES_AEQE_AEID_AMP_BAD_PD:
3701                 case NES_AEQE_AEID_AMP_FASTREG_SHARED:
3702                 case NES_AEQE_AEID_AMP_FASTREG_VALID_STAG:
3703                 case NES_AEQE_AEID_AMP_FASTREG_MW_STAG:
3704                 case NES_AEQE_AEID_AMP_FASTREG_INVALID_RIGHTS:
3705                 case NES_AEQE_AEID_AMP_FASTREG_PBL_TABLE_OVERFLOW:
3706                 case NES_AEQE_AEID_AMP_FASTREG_INVALID_LENGTH:
3707                 case NES_AEQE_AEID_AMP_INVALIDATE_SHARED:
3708                 case NES_AEQE_AEID_AMP_INVALIDATE_MR_WITH_BOUND_WINDOWS:
3709                 case NES_AEQE_AEID_AMP_MWBIND_VALID_STAG:
3710                 case NES_AEQE_AEID_AMP_MWBIND_OF_MR_STAG:
3711                 case NES_AEQE_AEID_AMP_MWBIND_TO_ZERO_BASED_STAG:
3712                 case NES_AEQE_AEID_AMP_MWBIND_TO_MW_STAG:
3713                 case NES_AEQE_AEID_AMP_MWBIND_INVALID_RIGHTS:
3714                 case NES_AEQE_AEID_AMP_MWBIND_INVALID_BOUNDS:
3715                 case NES_AEQE_AEID_AMP_MWBIND_TO_INVALID_PARENT:
3716                 case NES_AEQE_AEID_AMP_MWBIND_BIND_DISABLED:
3717                 case NES_AEQE_AEID_BAD_CLOSE:
3718                 case NES_AEQE_AEID_RDMA_READ_WHILE_ORD_ZERO:
3719                 case NES_AEQE_AEID_STAG_ZERO_INVALID:
3720                 case NES_AEQE_AEID_ROE_INVALID_RDMA_READ_REQUEST:
3721                 case NES_AEQE_AEID_ROE_INVALID_RDMA_WRITE_OR_READ_RESP:
3722                         printk(KERN_ERR PFX "QP[%u] async_event_id=0x%04X IB_EVENT_QP_FATAL\n",
3723                                         nesqp->hwqp.qp_id, async_event_id);
3724                         print_ip(nesqp->cm_node);
3725                         if (!atomic_read(&nesqp->close_timer_started))
3726                                 nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_FATAL);
3727                         break;
3728
3729                 case NES_AEQE_AEID_CQ_OPERATION_ERROR:
3730                         context <<= 1;
3731                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u, %p\n",
3732                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), (void *)(unsigned long)context);
3733                         resource_allocated = nes_is_resource_allocated(nesadapter, nesadapter->allocated_cqs,
3734                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
3735                         if (resource_allocated) {
3736                                 printk(KERN_ERR PFX "%s: Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u\n",
3737                                                 __func__, le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
3738                                 hw_cq = (struct nes_hw_cq *)(unsigned long)context;
3739                                 if (hw_cq) {
3740                                         nescq = container_of(hw_cq, struct nes_cq, hw_cq);
3741                                         if (nescq->ibcq.event_handler) {
3742                                                 ibevent.device = nescq->ibcq.device;
3743                                                 ibevent.event = IB_EVENT_CQ_ERR;
3744                                                 ibevent.element.cq = &nescq->ibcq;
3745                                                 nescq->ibcq.event_handler(&ibevent, nescq->ibcq.cq_context);
3746                                         }
3747                                 }
3748                         }
3749                         break;
3750
3751                 default:
3752                         nes_debug(NES_DBG_AEQ, "Processing an iWARP related AE for QP, misc = 0x%04X\n",
3753                                         async_event_id);
3754                         break;
3755         }
3756
3757 }
3758
3759 /**
3760  * nes_iwarp_ce_handler
3761  */
3762 void nes_iwarp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *hw_cq)
3763 {
3764         struct nes_cq *nescq = container_of(hw_cq, struct nes_cq, hw_cq);
3765
3766         /* nes_debug(NES_DBG_CQ, "Processing completion event for iWARP CQ%u.\n",
3767                         nescq->hw_cq.cq_number); */
3768         nes_write32(nesdev->regs+NES_CQ_ACK, nescq->hw_cq.cq_number);
3769
3770         if (nescq->ibcq.comp_handler)
3771                 nescq->ibcq.comp_handler(&nescq->ibcq, nescq->ibcq.cq_context);
3772
3773         return;
3774 }
3775
3776
3777 /**
3778  * nes_manage_apbvt()
3779  */
3780 int nes_manage_apbvt(struct nes_vnic *nesvnic, u32 accel_local_port,
3781                 u32 nic_index, u32 add_port)
3782 {
3783         struct nes_device *nesdev = nesvnic->nesdev;
3784         struct nes_hw_cqp_wqe *cqp_wqe;
3785         struct nes_cqp_request *cqp_request;
3786         int ret = 0;
3787         u16 major_code;
3788
3789         /* Send manage APBVT request to CQP */
3790         cqp_request = nes_get_cqp_request(nesdev);
3791         if (cqp_request == NULL) {
3792                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3793                 return -ENOMEM;
3794         }
3795         cqp_request->waiting = 1;
3796         cqp_wqe = &cqp_request->cqp_wqe;
3797
3798         nes_debug(NES_DBG_QP, "%s APBV for local port=%u(0x%04x), nic_index=%u\n",
3799                         (add_port == NES_MANAGE_APBVT_ADD) ? "ADD" : "DEL",
3800                         accel_local_port, accel_local_port, nic_index);
3801
3802         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3803         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, (NES_CQP_MANAGE_APBVT |
3804                         ((add_port == NES_MANAGE_APBVT_ADD) ? NES_CQP_APBVT_ADD : 0)));
3805         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
3806                         ((nic_index << NES_CQP_APBVT_NIC_SHIFT) | accel_local_port));
3807
3808         nes_debug(NES_DBG_QP, "Waiting for CQP completion for APBVT.\n");
3809
3810         atomic_set(&cqp_request->refcount, 2);
3811         nes_post_cqp_request(nesdev, cqp_request);
3812
3813         if (add_port == NES_MANAGE_APBVT_ADD)
3814                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3815                                 NES_EVENT_TIMEOUT);
3816         nes_debug(NES_DBG_QP, "Completed, ret=%u,  CQP Major:Minor codes = 0x%04X:0x%04X\n",
3817                         ret, cqp_request->major_code, cqp_request->minor_code);
3818         major_code = cqp_request->major_code;
3819
3820         nes_put_cqp_request(nesdev, cqp_request);
3821
3822         if (!ret)
3823                 return -ETIME;
3824         else if (major_code)
3825                 return -EIO;
3826         else
3827                 return 0;
3828 }
3829
3830
3831 /**
3832  * nes_manage_arp_cache
3833  */
3834 void nes_manage_arp_cache(struct net_device *netdev, unsigned char *mac_addr,
3835                 u32 ip_addr, u32 action)
3836 {
3837         struct nes_hw_cqp_wqe *cqp_wqe;
3838         struct nes_vnic *nesvnic = netdev_priv(netdev);
3839         struct nes_device *nesdev;
3840         struct nes_cqp_request *cqp_request;
3841         int arp_index;
3842
3843         nesdev = nesvnic->nesdev;
3844         arp_index = nes_arp_table(nesdev, ip_addr, mac_addr, action);
3845         if (arp_index == -1) {
3846                 return;
3847         }
3848
3849         /* update the ARP entry */
3850         cqp_request = nes_get_cqp_request(nesdev);
3851         if (cqp_request == NULL) {
3852                 nes_debug(NES_DBG_NETDEV, "Failed to get a cqp_request.\n");
3853                 return;
3854         }
3855         cqp_request->waiting = 0;
3856         cqp_wqe = &cqp_request->cqp_wqe;
3857         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3858
3859         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
3860                         NES_CQP_MANAGE_ARP_CACHE | NES_CQP_ARP_PERM);
3861         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(
3862                         (u32)PCI_FUNC(nesdev->pcidev->devfn) << NES_CQP_ARP_AEQ_INDEX_SHIFT);
3863         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(arp_index);
3864
3865         if (action == NES_ARP_ADD) {
3866                 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(NES_CQP_ARP_VALID);
3867                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = cpu_to_le32(
3868                                 (((u32)mac_addr[2]) << 24) | (((u32)mac_addr[3]) << 16) |
3869                                 (((u32)mac_addr[4]) << 8)  | (u32)mac_addr[5]);
3870                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = cpu_to_le32(
3871                                 (((u32)mac_addr[0]) << 16) | (u32)mac_addr[1]);
3872         } else {
3873                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = 0;
3874                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = 0;
3875         }
3876
3877         nes_debug(NES_DBG_NETDEV, "Not waiting for CQP, cqp.sq_head=%u, cqp.sq_tail=%u\n",
3878                         nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
3879
3880         atomic_set(&cqp_request->refcount, 1);
3881         nes_post_cqp_request(nesdev, cqp_request);
3882 }
3883
3884
3885 /**
3886  * flush_wqes
3887  */
3888 void flush_wqes(struct nes_device *nesdev, struct nes_qp *nesqp,
3889                 u32 which_wq, u32 wait_completion)
3890 {
3891         struct nes_cqp_request *cqp_request;
3892         struct nes_hw_cqp_wqe *cqp_wqe;
3893         u32 sq_code = (NES_IWARP_CQE_MAJOR_FLUSH << 16) | NES_IWARP_CQE_MINOR_FLUSH;
3894         u32 rq_code = (NES_IWARP_CQE_MAJOR_FLUSH << 16) | NES_IWARP_CQE_MINOR_FLUSH;
3895         int ret;
3896
3897         cqp_request = nes_get_cqp_request(nesdev);
3898         if (cqp_request == NULL) {
3899                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3900                 return;
3901         }
3902         if (wait_completion) {
3903                 cqp_request->waiting = 1;
3904                 atomic_set(&cqp_request->refcount, 2);
3905         } else {
3906                 cqp_request->waiting = 0;
3907         }
3908         cqp_wqe = &cqp_request->cqp_wqe;
3909         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3910
3911         /* If wqe in error was identified, set code to be put into cqe */
3912         if ((nesqp->term_sq_flush_code) && (which_wq & NES_CQP_FLUSH_SQ)) {
3913                 which_wq |= NES_CQP_FLUSH_MAJ_MIN;
3914                 sq_code = (CQE_MAJOR_DRV << 16) | nesqp->term_sq_flush_code;
3915                 nesqp->term_sq_flush_code = 0;
3916         }
3917
3918         if ((nesqp->term_rq_flush_code) && (which_wq & NES_CQP_FLUSH_RQ)) {
3919                 which_wq |= NES_CQP_FLUSH_MAJ_MIN;
3920                 rq_code = (CQE_MAJOR_DRV << 16) | nesqp->term_rq_flush_code;
3921                 nesqp->term_rq_flush_code = 0;
3922         }
3923
3924         if (which_wq & NES_CQP_FLUSH_MAJ_MIN) {
3925                 cqp_wqe->wqe_words[NES_CQP_QP_WQE_FLUSH_SQ_CODE] = cpu_to_le32(sq_code);
3926                 cqp_wqe->wqe_words[NES_CQP_QP_WQE_FLUSH_RQ_CODE] = cpu_to_le32(rq_code);
3927         }
3928
3929         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] =
3930                         cpu_to_le32(NES_CQP_FLUSH_WQES | which_wq);
3931         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesqp->hwqp.qp_id);
3932
3933         nes_post_cqp_request(nesdev, cqp_request);
3934
3935         if (wait_completion) {
3936                 /* Wait for CQP */
3937                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3938                                 NES_EVENT_TIMEOUT);
3939                 nes_debug(NES_DBG_QP, "Flush SQ QP WQEs completed, ret=%u,"
3940                                 " CQP Major:Minor codes = 0x%04X:0x%04X\n",
3941                                 ret, cqp_request->major_code, cqp_request->minor_code);
3942                 nes_put_cqp_request(nesdev, cqp_request);
3943         }
3944 }