drivers/infiniband/hw/ipath/ipath_iba7220.c

   1 /*
   2  * Copyright (c) 2006, 2007, 2008 QLogic Corporation. All rights reserved.
   3  * Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved.
   4  *
   5  * This software is available to you under a choice of one of two
   6  * licenses.  You may choose to be licensed under the terms of the GNU
   7  * General Public License (GPL) Version 2, available from the file
   8  * COPYING in the main directory of this source tree, or the
   9  * OpenIB.org BSD license below:
  10  *
  11  *     Redistribution and use in source and binary forms, with or
  12  *     without modification, are permitted provided that the following
  13  *     conditions are met:
  14  *
  15  *      - Redistributions of source code must retain the above
  16  *        copyright notice, this list of conditions and the following
  17  *        disclaimer.
  18  *
  19  *      - Redistributions in binary form must reproduce the above
  20  *        copyright notice, this list of conditions and the following
  21  *        disclaimer in the documentation and/or other materials
  22  *        provided with the distribution.
  23  *
  24  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  25  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  26  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  27  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  28  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  29  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  30  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  31  * SOFTWARE.
  32  */
  33 /*
  34  * This file contains all of the code that is specific to the
  35  * InfiniPath 7220 chip (except that specific to the SerDes)
  36  */
  37
  38 #include <linux/interrupt.h>
  39 #include <linux/pci.h>
  40 #include <linux/sched.h>
  41 #include <linux/delay.h>
  42 #include <linux/io.h>
  43 #include <rdma/ib_verbs.h>
  44
  45 #include "ipath_kernel.h"
  46 #include "ipath_registers.h"
  47 #include "ipath_7220.h"
  48
  49 static void ipath_setup_7220_setextled(struct ipath_devdata *, u64, u64);
  50
  51 static unsigned ipath_compat_ddr_negotiate = 1;
  52
  53 module_param_named(compat_ddr_negotiate, ipath_compat_ddr_negotiate, uint,
  54                         S_IWUSR | S_IRUGO);
  55 MODULE_PARM_DESC(compat_ddr_negotiate,
  56                 "Attempt pre-IBTA 1.2 DDR speed negotiation");
  57
  58 static unsigned ipath_sdma_fetch_arb = 1;
  59 module_param_named(fetch_arb, ipath_sdma_fetch_arb, uint, S_IRUGO);
  60 MODULE_PARM_DESC(fetch_arb, "IBA7220: change SDMA descriptor arbitration");
  61
  62 /*
  63  * This file contains almost all the chip-specific register information and
  64  * access functions for the QLogic InfiniPath 7220 PCI-Express chip, with the
  65  * exception of SerDes support, which in in ipath_sd7220.c.
  66  *
  67  * This lists the InfiniPath registers, in the actual chip layout.
  68  * This structure should never be directly accessed.
  69  */
  70 struct _infinipath_do_not_use_kernel_regs {
  71         unsigned long long Revision;
  72         unsigned long long Control;
  73         unsigned long long PageAlign;
  74         unsigned long long PortCnt;
  75         unsigned long long DebugPortSelect;
  76         unsigned long long DebugSigsIntSel; /* was Reserved0;*/
  77         unsigned long long SendRegBase;
  78         unsigned long long UserRegBase;
  79         unsigned long long CounterRegBase;
  80         unsigned long long Scratch;
  81         unsigned long long EEPROMAddrCmd; /* was Reserved1; */
  82         unsigned long long EEPROMData; /* was Reserved2; */
  83         unsigned long long IntBlocked;
  84         unsigned long long IntMask;
  85         unsigned long long IntStatus;
  86         unsigned long long IntClear;
  87         unsigned long long ErrorMask;
  88         unsigned long long ErrorStatus;
  89         unsigned long long ErrorClear;
  90         unsigned long long HwErrMask;
  91         unsigned long long HwErrStatus;
  92         unsigned long long HwErrClear;
  93         unsigned long long HwDiagCtrl;
  94         unsigned long long MDIO;
  95         unsigned long long IBCStatus;
  96         unsigned long long IBCCtrl;
  97         unsigned long long ExtStatus;
  98         unsigned long long ExtCtrl;
  99         unsigned long long GPIOOut;
 100         unsigned long long GPIOMask;
 101         unsigned long long GPIOStatus;
 102         unsigned long long GPIOClear;
 103         unsigned long long RcvCtrl;
 104         unsigned long long RcvBTHQP;
 105         unsigned long long RcvHdrSize;
 106         unsigned long long RcvHdrCnt;
 107         unsigned long long RcvHdrEntSize;
 108         unsigned long long RcvTIDBase;
 109         unsigned long long RcvTIDCnt;
 110         unsigned long long RcvEgrBase;
 111         unsigned long long RcvEgrCnt;
 112         unsigned long long RcvBufBase;
 113         unsigned long long RcvBufSize;
 114         unsigned long long RxIntMemBase;
 115         unsigned long long RxIntMemSize;
 116         unsigned long long RcvPartitionKey;
 117         unsigned long long RcvQPMulticastPort;
 118         unsigned long long RcvPktLEDCnt;
 119         unsigned long long IBCDDRCtrl;
 120         unsigned long long HRTBT_GUID;
 121         unsigned long long IB_SDTEST_IF_TX;
 122         unsigned long long IB_SDTEST_IF_RX;
 123         unsigned long long IBCDDRCtrl2;
 124         unsigned long long IBCDDRStatus;
 125         unsigned long long JIntReload;
 126         unsigned long long IBNCModeCtrl;
 127         unsigned long long SendCtrl;
 128         unsigned long long SendBufBase;
 129         unsigned long long SendBufSize;
 130         unsigned long long SendBufCnt;
 131         unsigned long long SendAvailAddr;
 132         unsigned long long TxIntMemBase;
 133         unsigned long long TxIntMemSize;
 134         unsigned long long SendDmaBase;
 135         unsigned long long SendDmaLenGen;
 136         unsigned long long SendDmaTail;
 137         unsigned long long SendDmaHead;
 138         unsigned long long SendDmaHeadAddr;
 139         unsigned long long SendDmaBufMask0;
 140         unsigned long long SendDmaBufMask1;
 141         unsigned long long SendDmaBufMask2;
 142         unsigned long long SendDmaStatus;
 143         unsigned long long SendBufferError;
 144         unsigned long long SendBufferErrorCONT1;
 145         unsigned long long SendBufErr2; /* was Reserved6SBE[0/6] */
 146         unsigned long long Reserved6L[2];
 147         unsigned long long AvailUpdCount;
 148         unsigned long long RcvHdrAddr0;
 149         unsigned long long RcvHdrAddrs[16]; /* Why enumerate? */
 150         unsigned long long Reserved7hdtl; /* Align next to 300 */
 151         unsigned long long RcvHdrTailAddr0; /* 300, like others */
 152         unsigned long long RcvHdrTailAddrs[16];
 153         unsigned long long Reserved9SW[7]; /* was [8]; we have 17 ports */
 154         unsigned long long IbsdEpbAccCtl; /* IB Serdes EPB access control */
 155         unsigned long long IbsdEpbTransReg; /* IB Serdes EPB Transaction */
 156         unsigned long long Reserved10sds; /* was SerdesStatus on */
 157         unsigned long long XGXSConfig;
 158         unsigned long long IBSerDesCtrl; /* Was IBPLLCfg on Monty */
 159         unsigned long long EEPCtlStat; /* for "boot" EEPROM/FLASH */
 160         unsigned long long EEPAddrCmd;
 161         unsigned long long EEPData;
 162         unsigned long long PcieEpbAccCtl;
 163         unsigned long long PcieEpbTransCtl;
 164         unsigned long long EfuseCtl; /* E-Fuse control */
 165         unsigned long long EfuseData[4];
 166         unsigned long long ProcMon;
 167         /* this chip moves following two from previous 200, 208 */
 168         unsigned long long PCIeRBufTestReg0;
 169         unsigned long long PCIeRBufTestReg1;
 170         /* added for this chip */
 171         unsigned long long PCIeRBufTestReg2;
 172         unsigned long long PCIeRBufTestReg3;
 173         /* added for this chip, debug only */
 174         unsigned long long SPC_JTAG_ACCESS_REG;
 175         unsigned long long LAControlReg;
 176         unsigned long long GPIODebugSelReg;
 177         unsigned long long DebugPortValueReg;
 178         /* added for this chip, DMA */
 179         unsigned long long SendDmaBufUsed[3];
 180         unsigned long long SendDmaReqTagUsed;
 181         /*
 182          * added for this chip, EFUSE: note that these program 64-bit
 183          * words 2 and 3 */
 184         unsigned long long efuse_pgm_data[2];
 185         unsigned long long Reserved11LAalign[10]; /* Skip 4B0..4F8 */
 186         /* we have 30 regs for DDS and RXEQ in IB SERDES */
 187         unsigned long long SerDesDDSRXEQ[30];
 188         unsigned long long Reserved12LAalign[2]; /* Skip 5F0, 5F8 */
 189         /* added for LA debug support */
 190         unsigned long long LAMemory[32];
 191 };
 192
 193 struct _infinipath_do_not_use_counters {
 194         __u64 LBIntCnt;
 195         __u64 LBFlowStallCnt;
 196         __u64 TxSDmaDescCnt;    /* was Reserved1 */
 197         __u64 TxUnsupVLErrCnt;
 198         __u64 TxDataPktCnt;
 199         __u64 TxFlowPktCnt;
 200         __u64 TxDwordCnt;
 201         __u64 TxLenErrCnt;
 202         __u64 TxMaxMinLenErrCnt;
 203         __u64 TxUnderrunCnt;
 204         __u64 TxFlowStallCnt;
 205         __u64 TxDroppedPktCnt;
 206         __u64 RxDroppedPktCnt;
 207         __u64 RxDataPktCnt;
 208         __u64 RxFlowPktCnt;
 209         __u64 RxDwordCnt;
 210         __u64 RxLenErrCnt;
 211         __u64 RxMaxMinLenErrCnt;
 212         __u64 RxICRCErrCnt;
 213         __u64 RxVCRCErrCnt;
 214         __u64 RxFlowCtrlErrCnt;
 215         __u64 RxBadFormatCnt;
 216         __u64 RxLinkProblemCnt;
 217         __u64 RxEBPCnt;
 218         __u64 RxLPCRCErrCnt;
 219         __u64 RxBufOvflCnt;
 220         __u64 RxTIDFullErrCnt;
 221         __u64 RxTIDValidErrCnt;
 222         __u64 RxPKeyMismatchCnt;
 223         __u64 RxP0HdrEgrOvflCnt;
 224         __u64 RxP1HdrEgrOvflCnt;
 225         __u64 RxP2HdrEgrOvflCnt;
 226         __u64 RxP3HdrEgrOvflCnt;
 227         __u64 RxP4HdrEgrOvflCnt;
 228         __u64 RxP5HdrEgrOvflCnt;
 229         __u64 RxP6HdrEgrOvflCnt;
 230         __u64 RxP7HdrEgrOvflCnt;
 231         __u64 RxP8HdrEgrOvflCnt;
 232         __u64 RxP9HdrEgrOvflCnt;        /* was Reserved6 */
 233         __u64 RxP10HdrEgrOvflCnt;       /* was Reserved7 */
 234         __u64 RxP11HdrEgrOvflCnt;       /* new for IBA7220 */
 235         __u64 RxP12HdrEgrOvflCnt;       /* new for IBA7220 */
 236         __u64 RxP13HdrEgrOvflCnt;       /* new for IBA7220 */
 237         __u64 RxP14HdrEgrOvflCnt;       /* new for IBA7220 */
 238         __u64 RxP15HdrEgrOvflCnt;       /* new for IBA7220 */
 239         __u64 RxP16HdrEgrOvflCnt;       /* new for IBA7220 */
 240         __u64 IBStatusChangeCnt;
 241         __u64 IBLinkErrRecoveryCnt;
 242         __u64 IBLinkDownedCnt;
 243         __u64 IBSymbolErrCnt;
 244         /* The following are new for IBA7220 */
 245         __u64 RxVL15DroppedPktCnt;
 246         __u64 RxOtherLocalPhyErrCnt;
 247         __u64 PcieRetryBufDiagQwordCnt;
 248         __u64 ExcessBufferOvflCnt;
 249         __u64 LocalLinkIntegrityErrCnt;
 250         __u64 RxVlErrCnt;
 251         __u64 RxDlidFltrCnt;
 252         __u64 Reserved8[7];
 253         __u64 PSStat;
 254         __u64 PSStart;
 255         __u64 PSInterval;
 256         __u64 PSRcvDataCount;
 257         __u64 PSRcvPktsCount;
 258         __u64 PSXmitDataCount;
 259         __u64 PSXmitPktsCount;
 260         __u64 PSXmitWaitCount;
 261 };
 262
 263 #define IPATH_KREG_OFFSET(field) (offsetof( \
 264         struct _infinipath_do_not_use_kernel_regs, field) / sizeof(u64))
 265 #define IPATH_CREG_OFFSET(field) (offsetof( \
 266         struct _infinipath_do_not_use_counters, field) / sizeof(u64))
 267
 268 static const struct ipath_kregs ipath_7220_kregs = {
 269         .kr_control = IPATH_KREG_OFFSET(Control),
 270         .kr_counterregbase = IPATH_KREG_OFFSET(CounterRegBase),
 271         .kr_debugportselect = IPATH_KREG_OFFSET(DebugPortSelect),
 272         .kr_errorclear = IPATH_KREG_OFFSET(ErrorClear),
 273         .kr_errormask = IPATH_KREG_OFFSET(ErrorMask),
 274         .kr_errorstatus = IPATH_KREG_OFFSET(ErrorStatus),
 275         .kr_extctrl = IPATH_KREG_OFFSET(ExtCtrl),
 276         .kr_extstatus = IPATH_KREG_OFFSET(ExtStatus),
 277         .kr_gpio_clear = IPATH_KREG_OFFSET(GPIOClear),
 278         .kr_gpio_mask = IPATH_KREG_OFFSET(GPIOMask),
 279         .kr_gpio_out = IPATH_KREG_OFFSET(GPIOOut),
 280         .kr_gpio_status = IPATH_KREG_OFFSET(GPIOStatus),
 281         .kr_hwdiagctrl = IPATH_KREG_OFFSET(HwDiagCtrl),
 282         .kr_hwerrclear = IPATH_KREG_OFFSET(HwErrClear),
 283         .kr_hwerrmask = IPATH_KREG_OFFSET(HwErrMask),
 284         .kr_hwerrstatus = IPATH_KREG_OFFSET(HwErrStatus),
 285         .kr_ibcctrl = IPATH_KREG_OFFSET(IBCCtrl),
 286         .kr_ibcstatus = IPATH_KREG_OFFSET(IBCStatus),
 287         .kr_intblocked = IPATH_KREG_OFFSET(IntBlocked),
 288         .kr_intclear = IPATH_KREG_OFFSET(IntClear),
 289         .kr_intmask = IPATH_KREG_OFFSET(IntMask),
 290         .kr_intstatus = IPATH_KREG_OFFSET(IntStatus),
 291         .kr_mdio = IPATH_KREG_OFFSET(MDIO),
 292         .kr_pagealign = IPATH_KREG_OFFSET(PageAlign),
 293         .kr_partitionkey = IPATH_KREG_OFFSET(RcvPartitionKey),
 294         .kr_portcnt = IPATH_KREG_OFFSET(PortCnt),
 295         .kr_rcvbthqp = IPATH_KREG_OFFSET(RcvBTHQP),
 296         .kr_rcvbufbase = IPATH_KREG_OFFSET(RcvBufBase),
 297         .kr_rcvbufsize = IPATH_KREG_OFFSET(RcvBufSize),
 298         .kr_rcvctrl = IPATH_KREG_OFFSET(RcvCtrl),
 299         .kr_rcvegrbase = IPATH_KREG_OFFSET(RcvEgrBase),
 300         .kr_rcvegrcnt = IPATH_KREG_OFFSET(RcvEgrCnt),
 301         .kr_rcvhdrcnt = IPATH_KREG_OFFSET(RcvHdrCnt),
 302         .kr_rcvhdrentsize = IPATH_KREG_OFFSET(RcvHdrEntSize),
 303         .kr_rcvhdrsize = IPATH_KREG_OFFSET(RcvHdrSize),
 304         .kr_rcvintmembase = IPATH_KREG_OFFSET(RxIntMemBase),
 305         .kr_rcvintmemsize = IPATH_KREG_OFFSET(RxIntMemSize),
 306         .kr_rcvtidbase = IPATH_KREG_OFFSET(RcvTIDBase),
 307         .kr_rcvtidcnt = IPATH_KREG_OFFSET(RcvTIDCnt),
 308         .kr_revision = IPATH_KREG_OFFSET(Revision),
 309         .kr_scratch = IPATH_KREG_OFFSET(Scratch),
 310         .kr_sendbuffererror = IPATH_KREG_OFFSET(SendBufferError),
 311         .kr_sendctrl = IPATH_KREG_OFFSET(SendCtrl),
 312         .kr_sendpioavailaddr = IPATH_KREG_OFFSET(SendAvailAddr),
 313         .kr_sendpiobufbase = IPATH_KREG_OFFSET(SendBufBase),
 314         .kr_sendpiobufcnt = IPATH_KREG_OFFSET(SendBufCnt),
 315         .kr_sendpiosize = IPATH_KREG_OFFSET(SendBufSize),
 316         .kr_sendregbase = IPATH_KREG_OFFSET(SendRegBase),
 317         .kr_txintmembase = IPATH_KREG_OFFSET(TxIntMemBase),
 318         .kr_txintmemsize = IPATH_KREG_OFFSET(TxIntMemSize),
 319         .kr_userregbase = IPATH_KREG_OFFSET(UserRegBase),
 320
 321         .kr_xgxsconfig = IPATH_KREG_OFFSET(XGXSConfig),
 322
 323         /* send dma related regs */
 324         .kr_senddmabase = IPATH_KREG_OFFSET(SendDmaBase),
 325         .kr_senddmalengen = IPATH_KREG_OFFSET(SendDmaLenGen),
 326         .kr_senddmatail = IPATH_KREG_OFFSET(SendDmaTail),
 327         .kr_senddmahead = IPATH_KREG_OFFSET(SendDmaHead),
 328         .kr_senddmaheadaddr = IPATH_KREG_OFFSET(SendDmaHeadAddr),
 329         .kr_senddmabufmask0 = IPATH_KREG_OFFSET(SendDmaBufMask0),
 330         .kr_senddmabufmask1 = IPATH_KREG_OFFSET(SendDmaBufMask1),
 331         .kr_senddmabufmask2 = IPATH_KREG_OFFSET(SendDmaBufMask2),
 332         .kr_senddmastatus = IPATH_KREG_OFFSET(SendDmaStatus),
 333
 334         /* SerDes related regs */
 335         .kr_ibserdesctrl = IPATH_KREG_OFFSET(IBSerDesCtrl),
 336         .kr_ib_epbacc = IPATH_KREG_OFFSET(IbsdEpbAccCtl),
 337         .kr_ib_epbtrans = IPATH_KREG_OFFSET(IbsdEpbTransReg),
 338         .kr_pcie_epbacc = IPATH_KREG_OFFSET(PcieEpbAccCtl),
 339         .kr_pcie_epbtrans = IPATH_KREG_OFFSET(PcieEpbTransCtl),
 340         .kr_ib_ddsrxeq = IPATH_KREG_OFFSET(SerDesDDSRXEQ),
 341
 342         /*
 343          * These should not be used directly via ipath_read_kreg64(),
 344          * use them with ipath_read_kreg64_port()
 345          */
 346         .kr_rcvhdraddr = IPATH_KREG_OFFSET(RcvHdrAddr0),
 347         .kr_rcvhdrtailaddr = IPATH_KREG_OFFSET(RcvHdrTailAddr0),
 348
 349         /*
 350          * The rcvpktled register controls one of the debug port signals, so
 351          * a packet activity LED can be connected to it.
 352          */
 353         .kr_rcvpktledcnt = IPATH_KREG_OFFSET(RcvPktLEDCnt),
 354         .kr_pcierbuftestreg0 = IPATH_KREG_OFFSET(PCIeRBufTestReg0),
 355         .kr_pcierbuftestreg1 = IPATH_KREG_OFFSET(PCIeRBufTestReg1),
 356
 357         .kr_hrtbt_guid = IPATH_KREG_OFFSET(HRTBT_GUID),
 358         .kr_ibcddrctrl = IPATH_KREG_OFFSET(IBCDDRCtrl),
 359         .kr_ibcddrstatus = IPATH_KREG_OFFSET(IBCDDRStatus),
 360         .kr_jintreload = IPATH_KREG_OFFSET(JIntReload)
 361 };
 362
 363 static const struct ipath_cregs ipath_7220_cregs = {
 364         .cr_badformatcnt = IPATH_CREG_OFFSET(RxBadFormatCnt),
 365         .cr_erricrccnt = IPATH_CREG_OFFSET(RxICRCErrCnt),
 366         .cr_errlinkcnt = IPATH_CREG_OFFSET(RxLinkProblemCnt),
 367         .cr_errlpcrccnt = IPATH_CREG_OFFSET(RxLPCRCErrCnt),
 368         .cr_errpkey = IPATH_CREG_OFFSET(RxPKeyMismatchCnt),
 369         .cr_errrcvflowctrlcnt = IPATH_CREG_OFFSET(RxFlowCtrlErrCnt),
 370         .cr_err_rlencnt = IPATH_CREG_OFFSET(RxLenErrCnt),
 371         .cr_errslencnt = IPATH_CREG_OFFSET(TxLenErrCnt),
 372         .cr_errtidfull = IPATH_CREG_OFFSET(RxTIDFullErrCnt),
 373         .cr_errtidvalid = IPATH_CREG_OFFSET(RxTIDValidErrCnt),
 374         .cr_errvcrccnt = IPATH_CREG_OFFSET(RxVCRCErrCnt),
 375         .cr_ibstatuschange = IPATH_CREG_OFFSET(IBStatusChangeCnt),
 376         .cr_intcnt = IPATH_CREG_OFFSET(LBIntCnt),
 377         .cr_invalidrlencnt = IPATH_CREG_OFFSET(RxMaxMinLenErrCnt),
 378         .cr_invalidslencnt = IPATH_CREG_OFFSET(TxMaxMinLenErrCnt),
 379         .cr_lbflowstallcnt = IPATH_CREG_OFFSET(LBFlowStallCnt),
 380         .cr_pktrcvcnt = IPATH_CREG_OFFSET(RxDataPktCnt),
 381         .cr_pktrcvflowctrlcnt = IPATH_CREG_OFFSET(RxFlowPktCnt),
 382         .cr_pktsendcnt = IPATH_CREG_OFFSET(TxDataPktCnt),
 383         .cr_pktsendflowcnt = IPATH_CREG_OFFSET(TxFlowPktCnt),
 384         .cr_portovflcnt = IPATH_CREG_OFFSET(RxP0HdrEgrOvflCnt),
 385         .cr_rcvebpcnt = IPATH_CREG_OFFSET(RxEBPCnt),
 386         .cr_rcvovflcnt = IPATH_CREG_OFFSET(RxBufOvflCnt),
 387         .cr_senddropped = IPATH_CREG_OFFSET(TxDroppedPktCnt),
 388         .cr_sendstallcnt = IPATH_CREG_OFFSET(TxFlowStallCnt),
 389         .cr_sendunderruncnt = IPATH_CREG_OFFSET(TxUnderrunCnt),
 390         .cr_wordrcvcnt = IPATH_CREG_OFFSET(RxDwordCnt),
 391         .cr_wordsendcnt = IPATH_CREG_OFFSET(TxDwordCnt),
 392         .cr_unsupvlcnt = IPATH_CREG_OFFSET(TxUnsupVLErrCnt),
 393         .cr_rxdroppktcnt = IPATH_CREG_OFFSET(RxDroppedPktCnt),
 394         .cr_iblinkerrrecovcnt = IPATH_CREG_OFFSET(IBLinkErrRecoveryCnt),
 395         .cr_iblinkdowncnt = IPATH_CREG_OFFSET(IBLinkDownedCnt),
 396         .cr_ibsymbolerrcnt = IPATH_CREG_OFFSET(IBSymbolErrCnt),
 397         .cr_vl15droppedpktcnt = IPATH_CREG_OFFSET(RxVL15DroppedPktCnt),
 398         .cr_rxotherlocalphyerrcnt =
 399                 IPATH_CREG_OFFSET(RxOtherLocalPhyErrCnt),
 400         .cr_excessbufferovflcnt = IPATH_CREG_OFFSET(ExcessBufferOvflCnt),
 401         .cr_locallinkintegrityerrcnt =
 402                 IPATH_CREG_OFFSET(LocalLinkIntegrityErrCnt),
 403         .cr_rxvlerrcnt = IPATH_CREG_OFFSET(RxVlErrCnt),
 404         .cr_rxdlidfltrcnt = IPATH_CREG_OFFSET(RxDlidFltrCnt),
 405         .cr_psstat = IPATH_CREG_OFFSET(PSStat),
 406         .cr_psstart = IPATH_CREG_OFFSET(PSStart),
 407         .cr_psinterval = IPATH_CREG_OFFSET(PSInterval),
 408         .cr_psrcvdatacount = IPATH_CREG_OFFSET(PSRcvDataCount),
 409         .cr_psrcvpktscount = IPATH_CREG_OFFSET(PSRcvPktsCount),
 410         .cr_psxmitdatacount = IPATH_CREG_OFFSET(PSXmitDataCount),
 411         .cr_psxmitpktscount = IPATH_CREG_OFFSET(PSXmitPktsCount),
 412         .cr_psxmitwaitcount = IPATH_CREG_OFFSET(PSXmitWaitCount),
 413 };
 414
 415 /* kr_control bits */
 416 #define INFINIPATH_C_RESET (1U<<7)
 417
 418 /* kr_intstatus, kr_intclear, kr_intmask bits */
 419 #define INFINIPATH_I_RCVURG_MASK ((1ULL<<17)-1)
 420 #define INFINIPATH_I_RCVURG_SHIFT 32
 421 #define INFINIPATH_I_RCVAVAIL_MASK ((1ULL<<17)-1)
 422 #define INFINIPATH_I_RCVAVAIL_SHIFT 0
 423 #define INFINIPATH_I_SERDESTRIMDONE (1ULL<<27)
 424
 425 /* kr_hwerrclear, kr_hwerrmask, kr_hwerrstatus, bits */
 426 #define INFINIPATH_HWE_PCIEMEMPARITYERR_MASK  0x00000000000000ffULL
 427 #define INFINIPATH_HWE_PCIEMEMPARITYERR_SHIFT 0
 428 #define INFINIPATH_HWE_PCIEPOISONEDTLP      0x0000000010000000ULL
 429 #define INFINIPATH_HWE_PCIECPLTIMEOUT       0x0000000020000000ULL
 430 #define INFINIPATH_HWE_PCIEBUSPARITYXTLH    0x0000000040000000ULL
 431 #define INFINIPATH_HWE_PCIEBUSPARITYXADM    0x0000000080000000ULL
 432 #define INFINIPATH_HWE_PCIEBUSPARITYRADM    0x0000000100000000ULL
 433 #define INFINIPATH_HWE_COREPLL_FBSLIP       0x0080000000000000ULL
 434 #define INFINIPATH_HWE_COREPLL_RFSLIP       0x0100000000000000ULL
 435 #define INFINIPATH_HWE_PCIE1PLLFAILED       0x0400000000000000ULL
 436 #define INFINIPATH_HWE_PCIE0PLLFAILED       0x0800000000000000ULL
 437 #define INFINIPATH_HWE_SERDESPLLFAILED      0x1000000000000000ULL
 438 /* specific to this chip */
 439 #define INFINIPATH_HWE_PCIECPLDATAQUEUEERR         0x0000000000000040ULL
 440 #define INFINIPATH_HWE_PCIECPLHDRQUEUEERR          0x0000000000000080ULL
 441 #define INFINIPATH_HWE_SDMAMEMREADERR              0x0000000010000000ULL
 442 #define INFINIPATH_HWE_CLK_UC_PLLNOTLOCKED         0x2000000000000000ULL
 443 #define INFINIPATH_HWE_PCIESERDESQ0PCLKNOTDETECT   0x0100000000000000ULL
 444 #define INFINIPATH_HWE_PCIESERDESQ1PCLKNOTDETECT   0x0200000000000000ULL
 445 #define INFINIPATH_HWE_PCIESERDESQ2PCLKNOTDETECT   0x0400000000000000ULL
 446 #define INFINIPATH_HWE_PCIESERDESQ3PCLKNOTDETECT   0x0800000000000000ULL
 447 #define INFINIPATH_HWE_DDSRXEQMEMORYPARITYERR      0x0000008000000000ULL
 448 #define INFINIPATH_HWE_IB_UC_MEMORYPARITYERR       0x0000004000000000ULL
 449 #define INFINIPATH_HWE_PCIE_UC_OCT0MEMORYPARITYERR 0x0000001000000000ULL
 450 #define INFINIPATH_HWE_PCIE_UC_OCT1MEMORYPARITYERR 0x0000002000000000ULL
 451
 452 #define IBA7220_IBCS_LINKTRAININGSTATE_MASK 0x1F
 453 #define IBA7220_IBCS_LINKSTATE_SHIFT 5
 454 #define IBA7220_IBCS_LINKSPEED_SHIFT 8
 455 #define IBA7220_IBCS_LINKWIDTH_SHIFT 9
 456
 457 #define IBA7220_IBCC_LINKINITCMD_MASK 0x7ULL
 458 #define IBA7220_IBCC_LINKCMD_SHIFT 19
 459 #define IBA7220_IBCC_MAXPKTLEN_SHIFT 21
 460
 461 /* kr_ibcddrctrl bits */
 462 #define IBA7220_IBC_DLIDLMC_MASK        0xFFFFFFFFUL
 463 #define IBA7220_IBC_DLIDLMC_SHIFT       32
 464 #define IBA7220_IBC_HRTBT_MASK  3
 465 #define IBA7220_IBC_HRTBT_SHIFT 16
 466 #define IBA7220_IBC_HRTBT_ENB   0x10000UL
 467 #define IBA7220_IBC_LANE_REV_SUPPORTED (1<<8)
 468 #define IBA7220_IBC_LREV_MASK   1
 469 #define IBA7220_IBC_LREV_SHIFT  8
 470 #define IBA7220_IBC_RXPOL_MASK  1
 471 #define IBA7220_IBC_RXPOL_SHIFT 7
 472 #define IBA7220_IBC_WIDTH_SHIFT 5
 473 #define IBA7220_IBC_WIDTH_MASK  0x3
 474 #define IBA7220_IBC_WIDTH_1X_ONLY       (0<<IBA7220_IBC_WIDTH_SHIFT)
 475 #define IBA7220_IBC_WIDTH_4X_ONLY       (1<<IBA7220_IBC_WIDTH_SHIFT)
 476 #define IBA7220_IBC_WIDTH_AUTONEG       (2<<IBA7220_IBC_WIDTH_SHIFT)
 477 #define IBA7220_IBC_SPEED_AUTONEG       (1<<1)
 478 #define IBA7220_IBC_SPEED_SDR           (1<<2)
 479 #define IBA7220_IBC_SPEED_DDR           (1<<3)
 480 #define IBA7220_IBC_SPEED_AUTONEG_MASK  (0x7<<1)
 481 #define IBA7220_IBC_IBTA_1_2_MASK       (1)
 482
 483 /* kr_ibcddrstatus */
 484 /* link latency shift is 0, don't bother defining */
 485 #define IBA7220_DDRSTAT_LINKLAT_MASK    0x3ffffff
 486
 487 /* kr_extstatus bits */
 488 #define INFINIPATH_EXTS_FREQSEL 0x2
 489 #define INFINIPATH_EXTS_SERDESSEL 0x4
 490 #define INFINIPATH_EXTS_MEMBIST_ENDTEST     0x0000000000004000
 491 #define INFINIPATH_EXTS_MEMBIST_DISABLED    0x0000000000008000
 492
 493 /* kr_xgxsconfig bits */
 494 #define INFINIPATH_XGXS_RESET          0x5ULL
 495 #define INFINIPATH_XGXS_FC_SAFE        (1ULL<<63)
 496
 497 /* kr_rcvpktledcnt */
 498 #define IBA7220_LEDBLINK_ON_SHIFT 32 /* 4ns period on after packet */
 499 #define IBA7220_LEDBLINK_OFF_SHIFT 0 /* 4ns period off before next on */
 500
 501 #define _IPATH_GPIO_SDA_NUM 1
 502 #define _IPATH_GPIO_SCL_NUM 0
 503
 504 #define IPATH_GPIO_SDA (1ULL << \
 505         (_IPATH_GPIO_SDA_NUM+INFINIPATH_EXTC_GPIOOE_SHIFT))
 506 #define IPATH_GPIO_SCL (1ULL << \
 507         (_IPATH_GPIO_SCL_NUM+INFINIPATH_EXTC_GPIOOE_SHIFT))
 508
 509 #define IBA7220_R_INTRAVAIL_SHIFT 17
 510 #define IBA7220_R_TAILUPD_SHIFT 35
 511 #define IBA7220_R_PORTCFG_SHIFT 36
 512
 513 #define INFINIPATH_JINT_PACKETSHIFT 16
 514 #define INFINIPATH_JINT_DEFAULT_IDLE_TICKS  0
 515 #define INFINIPATH_JINT_DEFAULT_MAX_PACKETS 0
 516
 517 #define IBA7220_HDRHEAD_PKTINT_SHIFT 32 /* interrupt cnt in upper 32 bits */
 518
 519 /*
 520  * the size bits give us 2^N, in KB units.  0 marks as invalid,
 521  * and 7 is reserved.  We currently use only 2KB and 4KB
 522  */
 523 #define IBA7220_TID_SZ_SHIFT 37 /* shift to 3bit size selector */
 524 #define IBA7220_TID_SZ_2K (1UL<<IBA7220_TID_SZ_SHIFT) /* 2KB */
 525 #define IBA7220_TID_SZ_4K (2UL<<IBA7220_TID_SZ_SHIFT) /* 4KB */
 526 #define IBA7220_TID_PA_SHIFT 11U /* TID addr in chip stored w/o low bits */
 527
 528 #define IPATH_AUTONEG_TRIES 5 /* sequential retries to negotiate DDR */
 529
 530 static char int_type[16] = "auto";
 531 module_param_string(interrupt_type, int_type, sizeof(int_type), 0444);
 532 MODULE_PARM_DESC(int_type, " interrupt_type=auto|force_msi|force_intx");
 533
 534 /* packet rate matching delay; chip has support */
 535 static u8 rate_to_delay[2][2] = {
 536         /* 1x, 4x */
 537         {   8, 2 }, /* SDR */
 538         {   4, 1 }  /* DDR */
 539 };
 540
 541 /* 7220 specific hardware errors... */
 542 static const struct ipath_hwerror_msgs ipath_7220_hwerror_msgs[] = {
 543         INFINIPATH_HWE_MSG(PCIEPOISONEDTLP, "PCIe Poisoned TLP"),
 544         INFINIPATH_HWE_MSG(PCIECPLTIMEOUT, "PCIe completion timeout"),
 545         /*
 546          * In practice, it's unlikely wthat we'll see PCIe PLL, or bus
 547          * parity or memory parity error failures, because most likely we
 548          * won't be able to talk to the core of the chip.  Nonetheless, we
 549          * might see them, if they are in parts of the PCIe core that aren't
 550          * essential.
 551          */
 552         INFINIPATH_HWE_MSG(PCIE1PLLFAILED, "PCIePLL1"),
 553         INFINIPATH_HWE_MSG(PCIE0PLLFAILED, "PCIePLL0"),
 554         INFINIPATH_HWE_MSG(PCIEBUSPARITYXTLH, "PCIe XTLH core parity"),
 555         INFINIPATH_HWE_MSG(PCIEBUSPARITYXADM, "PCIe ADM TX core parity"),
 556         INFINIPATH_HWE_MSG(PCIEBUSPARITYRADM, "PCIe ADM RX core parity"),
 557         INFINIPATH_HWE_MSG(RXDSYNCMEMPARITYERR, "Rx Dsync"),
 558         INFINIPATH_HWE_MSG(SERDESPLLFAILED, "SerDes PLL"),
 559         INFINIPATH_HWE_MSG(PCIECPLDATAQUEUEERR, "PCIe cpl header queue"),
 560         INFINIPATH_HWE_MSG(PCIECPLHDRQUEUEERR, "PCIe cpl data queue"),
 561         INFINIPATH_HWE_MSG(SDMAMEMREADERR, "Send DMA memory read"),
 562         INFINIPATH_HWE_MSG(CLK_UC_PLLNOTLOCKED, "uC PLL clock not locked"),
 563         INFINIPATH_HWE_MSG(PCIESERDESQ0PCLKNOTDETECT,
 564                 "PCIe serdes Q0 no clock"),
 565         INFINIPATH_HWE_MSG(PCIESERDESQ1PCLKNOTDETECT,
 566                 "PCIe serdes Q1 no clock"),
 567         INFINIPATH_HWE_MSG(PCIESERDESQ2PCLKNOTDETECT,
 568                 "PCIe serdes Q2 no clock"),
 569         INFINIPATH_HWE_MSG(PCIESERDESQ3PCLKNOTDETECT,
 570                 "PCIe serdes Q3 no clock"),
 571         INFINIPATH_HWE_MSG(DDSRXEQMEMORYPARITYERR,
 572                 "DDS RXEQ memory parity"),
 573         INFINIPATH_HWE_MSG(IB_UC_MEMORYPARITYERR, "IB uC memory parity"),
 574         INFINIPATH_HWE_MSG(PCIE_UC_OCT0MEMORYPARITYERR,
 575                 "PCIe uC oct0 memory parity"),
 576         INFINIPATH_HWE_MSG(PCIE_UC_OCT1MEMORYPARITYERR,
 577                 "PCIe uC oct1 memory parity"),
 578 };
 579
 580 static void autoneg_work(struct work_struct *);
 581
 582 /*
 583  * the offset is different for different configured port numbers, since
 584  * port0 is fixed in size, but others can vary.   Make it a function to
 585  * make the issue more obvious.
 586 */
 587 static inline u32 port_egrtid_idx(struct ipath_devdata *dd, unsigned port)
 588 {
 589          return port ? dd->ipath_p0_rcvegrcnt +
 590                  (port-1) * dd->ipath_rcvegrcnt : 0;
 591 }
 592
 593 static void ipath_7220_txe_recover(struct ipath_devdata *dd)
 594 {
 595         ++ipath_stats.sps_txeparity;
 596
 597         dev_info(&dd->pcidev->dev,
 598                 "Recovering from TXE PIO parity error\n");
 599         ipath_disarm_senderrbufs(dd);
 600 }
 601
 602
 603 /**
 604  * ipath_7220_handle_hwerrors - display hardware errors.
 605  * @dd: the infinipath device
 606  * @msg: the output buffer
 607  * @msgl: the size of the output buffer
 608  *
 609  * Use same msg buffer as regular errors to avoid excessive stack
 610  * use.  Most hardware errors are catastrophic, but for right now,
 611  * we'll print them and continue.  We reuse the same message buffer as
 612  * ipath_handle_errors() to avoid excessive stack usage.
 613  */
 614 static void ipath_7220_handle_hwerrors(struct ipath_devdata *dd, char *msg,
 615                                        size_t msgl)
 616 {
 617         ipath_err_t hwerrs;
 618         u32 bits, ctrl;
 619         int isfatal = 0;
 620         char bitsmsg[64];
 621         int log_idx;
 622
 623         hwerrs = ipath_read_kreg64(dd, dd->ipath_kregs->kr_hwerrstatus);
 624         if (!hwerrs) {
 625                 /*
 626                  * better than printing cofusing messages
 627                  * This seems to be related to clearing the crc error, or
 628                  * the pll error during init.
 629                  */
 630                 ipath_cdbg(VERBOSE, "Called but no hardware errors set\n");
 631                 goto bail;
 632         } else if (hwerrs == ~0ULL) {
 633                 ipath_dev_err(dd, "Read of hardware error status failed "
 634                               "(all bits set); ignoring\n");
 635                 goto bail;
 636         }
 637         ipath_stats.sps_hwerrs++;
 638
 639         /*
 640          * Always clear the error status register, except MEMBISTFAIL,
 641          * regardless of whether we continue or stop using the chip.
 642          * We want that set so we know it failed, even across driver reload.
 643          * We'll still ignore it in the hwerrmask.  We do this partly for
 644          * diagnostics, but also for support.
 645          */
 646         ipath_write_kreg(dd, dd->ipath_kregs->kr_hwerrclear,
 647                          hwerrs&~INFINIPATH_HWE_MEMBISTFAILED);
 648
 649         hwerrs &= dd->ipath_hwerrmask;
 650
 651         /* We log some errors to EEPROM, check if we have any of those. */
 652         for (log_idx = 0; log_idx < IPATH_EEP_LOG_CNT; ++log_idx)
 653                 if (hwerrs & dd->ipath_eep_st_masks[log_idx].hwerrs_to_log)
 654                         ipath_inc_eeprom_err(dd, log_idx, 1);
 655         /*
 656          * Make sure we get this much out, unless told to be quiet,
 657          * or it's occurred within the last 5 seconds.
 658          */
 659         if ((hwerrs & ~(dd->ipath_lasthwerror |
 660                         ((INFINIPATH_HWE_TXEMEMPARITYERR_PIOBUF |
 661                           INFINIPATH_HWE_TXEMEMPARITYERR_PIOPBC)
 662                          << INFINIPATH_HWE_TXEMEMPARITYERR_SHIFT))) ||
 663             (ipath_debug & __IPATH_VERBDBG))
 664                 dev_info(&dd->pcidev->dev, "Hardware error: hwerr=0x%llx "
 665                          "(cleared)\n", (unsigned long long) hwerrs);
 666         dd->ipath_lasthwerror |= hwerrs;
 667
 668         if (hwerrs & ~dd->ipath_hwe_bitsextant)
 669                 ipath_dev_err(dd, "hwerror interrupt with unknown errors "
 670                               "%llx set\n", (unsigned long long)
 671                               (hwerrs & ~dd->ipath_hwe_bitsextant));
 672
 673         if (hwerrs & INFINIPATH_HWE_IB_UC_MEMORYPARITYERR)
 674                 ipath_sd7220_clr_ibpar(dd);
 675
 676         ctrl = ipath_read_kreg32(dd, dd->ipath_kregs->kr_control);
 677         if ((ctrl & INFINIPATH_C_FREEZEMODE) && !ipath_diag_inuse) {
 678                 /*
 679                  * Parity errors in send memory are recoverable by h/w
 680                  * just do housekeeping, exit freeze mode and continue.
 681                  */
 682                 if (hwerrs & ((INFINIPATH_HWE_TXEMEMPARITYERR_PIOBUF |
 683                                INFINIPATH_HWE_TXEMEMPARITYERR_PIOPBC)
 684                               << INFINIPATH_HWE_TXEMEMPARITYERR_SHIFT)) {
 685                         ipath_7220_txe_recover(dd);
 686                         hwerrs &= ~((INFINIPATH_HWE_TXEMEMPARITYERR_PIOBUF |
 687                                      INFINIPATH_HWE_TXEMEMPARITYERR_PIOPBC)
 688                                     << INFINIPATH_HWE_TXEMEMPARITYERR_SHIFT);
 689                 }
 690                 if (hwerrs) {
 691                         /*
 692                          * If any set that we aren't ignoring only make the
 693                          * complaint once, in case it's stuck or recurring,
 694                          * and we get here multiple times
 695                          * Force link down, so switch knows, and
 696                          * LEDs are turned off.
 697                          */
 698                         if (dd->ipath_flags & IPATH_INITTED) {
 699                                 ipath_set_linkstate(dd, IPATH_IB_LINKDOWN);
 700                                 ipath_setup_7220_setextled(dd,
 701                                         INFINIPATH_IBCS_L_STATE_DOWN,
 702                                         INFINIPATH_IBCS_LT_STATE_DISABLED);
 703                                 ipath_dev_err(dd, "Fatal Hardware Error "
 704                                               "(freeze mode), no longer"
 705                                               " usable, SN %.16s\n",
 706                                                   dd->ipath_serial);
 707                                 isfatal = 1;
 708                         }
 709                         /*
 710                          * Mark as having had an error for driver, and also
 711                          * for /sys and status word mapped to user programs.
 712                          * This marks unit as not usable, until reset.
 713                          */
 714                         *dd->ipath_statusp &= ~IPATH_STATUS_IB_READY;
 715                         *dd->ipath_statusp |= IPATH_STATUS_HWERROR;
 716                         dd->ipath_flags &= ~IPATH_INITTED;
 717                 } else {
 718                         ipath_dbg("Clearing freezemode on ignored or "
 719                                 "recovered hardware error\n");
 720                         ipath_clear_freeze(dd);
 721                 }
 722         }
 723
 724         *msg = '\0';
 725
 726         if (hwerrs & INFINIPATH_HWE_MEMBISTFAILED) {
 727                 strlcat(msg, "[Memory BIST test failed, "
 728                         "InfiniPath hardware unusable]", msgl);
 729                 /* ignore from now on, so disable until driver reloaded */
 730                 *dd->ipath_statusp |= IPATH_STATUS_HWERROR;
 731                 dd->ipath_hwerrmask &= ~INFINIPATH_HWE_MEMBISTFAILED;
 732                 ipath_write_kreg(dd, dd->ipath_kregs->kr_hwerrmask,
 733                                  dd->ipath_hwerrmask);
 734         }
 735
 736         ipath_format_hwerrors(hwerrs,
 737                               ipath_7220_hwerror_msgs,
 738                               ARRAY_SIZE(ipath_7220_hwerror_msgs),
 739                               msg, msgl);
 740
 741         if (hwerrs & (INFINIPATH_HWE_PCIEMEMPARITYERR_MASK
 742                       << INFINIPATH_HWE_PCIEMEMPARITYERR_SHIFT)) {
 743                 bits = (u32) ((hwerrs >>
 744                                INFINIPATH_HWE_PCIEMEMPARITYERR_SHIFT) &
 745                               INFINIPATH_HWE_PCIEMEMPARITYERR_MASK);
 746                 snprintf(bitsmsg, sizeof bitsmsg,
 747                          "[PCIe Mem Parity Errs %x] ", bits);
 748                 strlcat(msg, bitsmsg, msgl);
 749         }
 750
 751 #define _IPATH_PLL_FAIL (INFINIPATH_HWE_COREPLL_FBSLIP |        \
 752                          INFINIPATH_HWE_COREPLL_RFSLIP)
 753
 754         if (hwerrs & _IPATH_PLL_FAIL) {
 755                 snprintf(bitsmsg, sizeof bitsmsg,
 756                          "[PLL failed (%llx), InfiniPath hardware unusable]",
 757                          (unsigned long long) hwerrs & _IPATH_PLL_FAIL);
 758                 strlcat(msg, bitsmsg, msgl);
 759                 /* ignore from now on, so disable until driver reloaded */
 760                 dd->ipath_hwerrmask &= ~(hwerrs & _IPATH_PLL_FAIL);
 761                 ipath_write_kreg(dd, dd->ipath_kregs->kr_hwerrmask,
 762                                  dd->ipath_hwerrmask);
 763         }
 764
 765         if (hwerrs & INFINIPATH_HWE_SERDESPLLFAILED) {
 766                 /*
 767                  * If it occurs, it is left masked since the eternal
 768                  * interface is unused.
 769                  */
 770                 dd->ipath_hwerrmask &= ~INFINIPATH_HWE_SERDESPLLFAILED;
 771                 ipath_write_kreg(dd, dd->ipath_kregs->kr_hwerrmask,
 772                                  dd->ipath_hwerrmask);
 773         }
 774
 775         ipath_dev_err(dd, "%s hardware error\n", msg);
 776         /*
 777          * For /sys status file. if no trailing } is copied, we'll
 778          * know it was truncated.
 779          */
 780         if (isfatal && !ipath_diag_inuse && dd->ipath_freezemsg)
 781                 snprintf(dd->ipath_freezemsg, dd->ipath_freezelen,
 782                          "{%s}", msg);
 783 bail:;
 784 }
 785
 786 /**
 787  * ipath_7220_boardname - fill in the board name
 788  * @dd: the infinipath device
 789  * @name: the output buffer
 790  * @namelen: the size of the output buffer
 791  *
 792  * info is based on the board revision register
 793  */
 794 static int ipath_7220_boardname(struct ipath_devdata *dd, char *name,
 795         size_t namelen)
 796 {
 797         char *n = NULL;
 798         u8 boardrev = dd->ipath_boardrev;
 799         int ret;
 800
 801         if (boardrev == 15) {
 802                 /*
 803                  * Emulator sometimes comes up all-ones, rather than zero.
 804                  */
 805                 boardrev = 0;
 806                 dd->ipath_boardrev = boardrev;
 807         }
 808         switch (boardrev) {
 809         case 0:
 810                 n = "InfiniPath_7220_Emulation";
 811                 break;
 812         case 1:
 813                 n = "InfiniPath_QLE7240";
 814                 break;
 815         case 2:
 816                 n = "InfiniPath_QLE7280";
 817                 break;
 818         case 3:
 819                 n = "InfiniPath_QLE7242";
 820                 break;
 821         case 4:
 822                 n = "InfiniPath_QEM7240";
 823                 break;
 824         case 5:
 825                 n = "InfiniPath_QMI7240";
 826                 break;
 827         case 6:
 828                 n = "InfiniPath_QMI7264";
 829                 break;
 830         case 7:
 831                 n = "InfiniPath_QMH7240";
 832                 break;
 833         case 8:
 834                 n = "InfiniPath_QME7240";
 835                 break;
 836         case 9:
 837                 n = "InfiniPath_QLE7250";
 838                 break;
 839         case 10:
 840                 n = "InfiniPath_QLE7290";
 841                 break;
 842         case 11:
 843                 n = "InfiniPath_QEM7250";
 844                 break;
 845         case 12:
 846                 n = "InfiniPath_QLE-Bringup";
 847                 break;
 848         default:
 849                 ipath_dev_err(dd,
 850                               "Don't yet know about board with ID %u\n",
 851                               boardrev);
 852                 snprintf(name, namelen, "Unknown_InfiniPath_PCIe_%u",
 853                          boardrev);
 854                 break;
 855         }
 856         if (n)
 857                 snprintf(name, namelen, "%s", n);
 858
 859         if (dd->ipath_majrev != 5 || !dd->ipath_minrev ||
 860                 dd->ipath_minrev > 2) {
 861                 ipath_dev_err(dd, "Unsupported InfiniPath hardware "
 862                               "revision %u.%u!\n",
 863                               dd->ipath_majrev, dd->ipath_minrev);
 864                 ret = 1;
 865         } else if (dd->ipath_minrev == 1 &&
 866                 !(dd->ipath_flags & IPATH_INITTED)) {
 867                 /* Rev1 chips are prototype. Complain at init, but allow use */
 868                 ipath_dev_err(dd, "Unsupported hardware "
 869                               "revision %u.%u, Contact support@qlogic.com\n",
 870                               dd->ipath_majrev, dd->ipath_minrev);
 871                 ret = 0;
 872         } else
 873                 ret = 0;
 874
 875         /*
 876          * Set here not in ipath_init_*_funcs because we have to do
 877          * it after we can read chip registers.
 878          */
 879         dd->ipath_ureg_align = 0x10000;  /* 64KB alignment */
 880
 881         return ret;
 882 }
 883
 884 /**
 885  * ipath_7220_init_hwerrors - enable hardware errors
 886  * @dd: the infinipath device
 887  *
 888  * now that we have finished initializing everything that might reasonably
 889  * cause a hardware error, and cleared those errors bits as they occur,
 890  * we can enable hardware errors in the mask (potentially enabling
 891  * freeze mode), and enable hardware errors as errors (along with
 892  * everything else) in errormask
 893  */
 894 static void ipath_7220_init_hwerrors(struct ipath_devdata *dd)
 895 {
 896         ipath_err_t val;
 897         u64 extsval;
 898
 899         extsval = ipath_read_kreg64(dd, dd->ipath_kregs->kr_extstatus);
 900
 901         if (!(extsval & (INFINIPATH_EXTS_MEMBIST_ENDTEST |
 902                         INFINIPATH_EXTS_MEMBIST_DISABLED)))
 903                 ipath_dev_err(dd, "MemBIST did not complete!\n");
 904         if (extsval & INFINIPATH_EXTS_MEMBIST_DISABLED)
 905                 dev_info(&dd->pcidev->dev, "MemBIST is disabled.\n");
 906
 907         val = ~0ULL;    /* barring bugs, all hwerrors become interrupts, */
 908
 909         if (!dd->ipath_boardrev)        /* no PLL for Emulator */
 910                 val &= ~INFINIPATH_HWE_SERDESPLLFAILED;
 911
 912         if (dd->ipath_minrev == 1)
 913                 val &= ~(1ULL << 42); /* TXE LaunchFIFO Parity rev1 issue */
 914
 915         val &= ~INFINIPATH_HWE_IB_UC_MEMORYPARITYERR;
 916         dd->ipath_hwerrmask = val;
 917
 918         /*
 919          * special trigger "error" is for debugging purposes. It
 920          * works around a processor/chipset problem.  The error
 921          * interrupt allows us to count occurrences, but we don't
 922          * want to pay the overhead for normal use.  Emulation only
 923          */
 924         if (!dd->ipath_boardrev)
 925                 dd->ipath_maskederrs = INFINIPATH_E_SENDSPECIALTRIGGER;
 926 }
 927
 928 /*
 929  * All detailed interaction with the SerDes has been moved to ipath_sd7220.c
 930  *
 931  * The portion of IBA7220-specific bringup_serdes() that actually deals with
 932  * registers and memory within the SerDes itself is ipath_sd7220_init().
 933  */
 934
 935 /**
 936  * ipath_7220_bringup_serdes - bring up the serdes
 937  * @dd: the infinipath device
 938  */
 939 static int ipath_7220_bringup_serdes(struct ipath_devdata *dd)
 940 {
 941         int ret = 0;
 942         u64 val, prev_val, guid;
 943         int was_reset;          /* Note whether uC was reset */
 944
 945         ipath_dbg("Trying to bringup serdes\n");
 946
 947         if (ipath_read_kreg64(dd, dd->ipath_kregs->kr_hwerrstatus) &
 948             INFINIPATH_HWE_SERDESPLLFAILED) {
 949                 ipath_dbg("At start, serdes PLL failed bit set "
 950                           "in hwerrstatus, clearing and continuing\n");
 951                 ipath_write_kreg(dd, dd->ipath_kregs->kr_hwerrclear,
 952                                  INFINIPATH_HWE_SERDESPLLFAILED);
 953         }
 954
 955         dd->ibdeltainprog = 1;
 956         dd->ibsymsnap =
 957              ipath_read_creg32(dd, dd->ipath_cregs->cr_ibsymbolerrcnt);
 958         dd->iblnkerrsnap =
 959              ipath_read_creg32(dd, dd->ipath_cregs->cr_iblinkerrrecovcnt);
 960
 961         if (!dd->ipath_ibcddrctrl) {
 962                 /* not on re-init after reset */
 963                 dd->ipath_ibcddrctrl =
 964                         ipath_read_kreg64(dd, dd->ipath_kregs->kr_ibcddrctrl);
 965
 966                 if (dd->ipath_link_speed_enabled ==
 967                         (IPATH_IB_SDR | IPATH_IB_DDR))
 968                         dd->ipath_ibcddrctrl |=
 969                                 IBA7220_IBC_SPEED_AUTONEG_MASK |
 970                                 IBA7220_IBC_IBTA_1_2_MASK;
 971                 else
 972                         dd->ipath_ibcddrctrl |=
 973                                 dd->ipath_link_speed_enabled == IPATH_IB_DDR
 974                                 ?  IBA7220_IBC_SPEED_DDR :
 975                                 IBA7220_IBC_SPEED_SDR;
 976                 if ((dd->ipath_link_width_enabled & (IB_WIDTH_1X |
 977                         IB_WIDTH_4X)) == (IB_WIDTH_1X | IB_WIDTH_4X))
 978                         dd->ipath_ibcddrctrl |= IBA7220_IBC_WIDTH_AUTONEG;
 979                 else
 980                         dd->ipath_ibcddrctrl |=
 981                                 dd->ipath_link_width_enabled == IB_WIDTH_4X
 982                                 ? IBA7220_IBC_WIDTH_4X_ONLY :
 983                                 IBA7220_IBC_WIDTH_1X_ONLY;
 984
 985                 /* always enable these on driver reload, not sticky */
 986                 dd->ipath_ibcddrctrl |=
 987                         IBA7220_IBC_RXPOL_MASK << IBA7220_IBC_RXPOL_SHIFT;
 988                 dd->ipath_ibcddrctrl |=
 989                         IBA7220_IBC_HRTBT_MASK << IBA7220_IBC_HRTBT_SHIFT;
 990                 /*
 991                  * automatic lane reversal detection for receive
 992                  * doesn't work correctly in rev 1, so disable it
 993                  * on that rev, otherwise enable (disabling not
 994                  * sticky across reload for >rev1)
 995                  */
 996                 if (dd->ipath_minrev == 1)
 997                         dd->ipath_ibcddrctrl &=
 998                         ~IBA7220_IBC_LANE_REV_SUPPORTED;
 999                 else
1000                         dd->ipath_ibcddrctrl |=
1001                                 IBA7220_IBC_LANE_REV_SUPPORTED;
1002         }
1003
1004         ipath_write_kreg(dd, dd->ipath_kregs->kr_ibcddrctrl,
1005                         dd->ipath_ibcddrctrl);
1006
1007         ipath_write_kreg(dd, IPATH_KREG_OFFSET(IBNCModeCtrl), 0Ull);
1008
1009         /* IBA7220 has SERDES MPU reset in D0 of what _was_ IBPLLCfg */
1010         val = ipath_read_kreg64(dd, dd->ipath_kregs->kr_ibserdesctrl);
1011         /* remember if uC was in Reset or not, for dactrim */
1012         was_reset = (val & 1);
1013         ipath_cdbg(VERBOSE, "IBReset %s xgxsconfig %llx\n",
1014                    was_reset ? "Asserted" : "Negated", (unsigned long long)
1015                    ipath_read_kreg64(dd, dd->ipath_kregs->kr_xgxsconfig));
1016
1017         if (dd->ipath_boardrev) {
1018                 /*
1019                  * Hardware is not emulator, and may have been reset. Init it.
1020                  * Below will release reset, but needs to know if chip was
1021                  * originally in reset, to only trim DACs on first time
1022                  * after chip reset or powercycle (not driver reload)
1023                  */
1024                 ret = ipath_sd7220_init(dd, was_reset);
1025         }
1026
1027         val = ipath_read_kreg64(dd, dd->ipath_kregs->kr_xgxsconfig);
1028         prev_val = val;
1029         val |= INFINIPATH_XGXS_FC_SAFE;
1030         if (val != prev_val) {
1031                 ipath_write_kreg(dd, dd->ipath_kregs->kr_xgxsconfig, val);
1032                 ipath_read_kreg32(dd, dd->ipath_kregs->kr_scratch);
1033         }
1034         if (val & INFINIPATH_XGXS_RESET)
1035                 val &= ~INFINIPATH_XGXS_RESET;
1036         if (val != prev_val)
1037                 ipath_write_kreg(dd, dd->ipath_kregs->kr_xgxsconfig, val);
1038
1039         ipath_cdbg(VERBOSE, "done: xgxs=%llx from %llx\n",
1040                    (unsigned long long)
1041                    ipath_read_kreg64(dd, dd->ipath_kregs->kr_xgxsconfig),
1042                    (unsigned long long) prev_val);
1043
1044         guid = be64_to_cpu(dd->ipath_guid);
1045
1046         if (!guid) {
1047                 /* have to have something, so use likely unique tsc */
1048                 guid = get_cycles();
1049                 ipath_dbg("No GUID for heartbeat, faking %llx\n",
1050                         (unsigned long long)guid);
1051         } else
1052                 ipath_cdbg(VERBOSE, "Wrote %llX to HRTBT_GUID\n",
1053                         (unsigned long long) guid);
1054         ipath_write_kreg(dd, dd->ipath_kregs->kr_hrtbt_guid, guid);
1055         return ret;
1056 }
1057
1058 static void ipath_7220_config_jint(struct ipath_devdata *dd,
1059                                    u16 idle_ticks, u16 max_packets)
1060 {
1061
1062         /*
1063          * We can request a receive interrupt for 1 or more packets
1064          * from current offset.
1065          */
1066         if (idle_ticks == 0 || max_packets == 0)
1067                 /* interrupt after one packet if no mitigation */
1068                 dd->ipath_rhdrhead_intr_off =
1069                         1ULL << IBA7220_HDRHEAD_PKTINT_SHIFT;
1070         else
1071                 /* Turn off RcvHdrHead interrupts if using mitigation */
1072                 dd->ipath_rhdrhead_intr_off = 0ULL;
1073
1074         /* refresh kernel RcvHdrHead registers... */
1075         ipath_write_ureg(dd, ur_rcvhdrhead,
1076                          dd->ipath_rhdrhead_intr_off |
1077                          dd->ipath_pd[0]->port_head, 0);
1078
1079         dd->ipath_jint_max_packets = max_packets;
1080         dd->ipath_jint_idle_ticks = idle_ticks;
1081         ipath_write_kreg(dd, dd->ipath_kregs->kr_jintreload,
1082                          ((u64) max_packets << INFINIPATH_JINT_PACKETSHIFT) |
1083                          idle_ticks);
1084 }
1085
1086 /**
1087  * ipath_7220_quiet_serdes - set serdes to txidle
1088  * @dd: the infinipath device
1089  * Called when driver is being unloaded
1090  */
1091 static void ipath_7220_quiet_serdes(struct ipath_devdata *dd)
1092 {
1093         u64 val;
1094         if (dd->ibsymdelta || dd->iblnkerrdelta ||
1095             dd->ibdeltainprog) {
1096                 u64 diagc;
1097                 /* enable counter writes */
1098                 diagc = ipath_read_kreg64(dd, dd->ipath_kregs->kr_hwdiagctrl);
1099                 ipath_write_kreg(dd, dd->ipath_kregs->kr_hwdiagctrl,
1100                                  diagc | INFINIPATH_DC_COUNTERWREN);
1101
1102                 if (dd->ibsymdelta || dd->ibdeltainprog) {
1103                         val = ipath_read_creg32(dd,
1104                                         dd->ipath_cregs->cr_ibsymbolerrcnt);
1105                         if (dd->ibdeltainprog)
1106                                 val -= val - dd->ibsymsnap;
1107                         val -= dd->ibsymdelta;
1108                         ipath_write_creg(dd,
1109                                   dd->ipath_cregs->cr_ibsymbolerrcnt, val);
1110                 }
1111                 if (dd->iblnkerrdelta || dd->ibdeltainprog) {
1112                         val = ipath_read_creg32(dd,
1113                                         dd->ipath_cregs->cr_iblinkerrrecovcnt);
1114                         if (dd->ibdeltainprog)
1115                                 val -= val - dd->iblnkerrsnap;
1116                         val -= dd->iblnkerrdelta;
1117                         ipath_write_creg(dd,
1118                                    dd->ipath_cregs->cr_iblinkerrrecovcnt, val);
1119              }
1120
1121              /* and disable counter writes */
1122              ipath_write_kreg(dd, dd->ipath_kregs->kr_hwdiagctrl, diagc);
1123         }
1124
1125         dd->ipath_flags &= ~IPATH_IB_AUTONEG_INPROG;
1126         wake_up(&dd->ipath_autoneg_wait);
1127         cancel_delayed_work(&dd->ipath_autoneg_work);
1128         flush_scheduled_work();
1129         ipath_shutdown_relock_poll(dd);
1130         val = ipath_read_kreg64(dd, dd->ipath_kregs->kr_xgxsconfig);
1131         val |= INFINIPATH_XGXS_RESET;
1132         ipath_write_kreg(dd, dd->ipath_kregs->kr_xgxsconfig, val);
1133 }
1134
1135 static int ipath_7220_intconfig(struct ipath_devdata *dd)
1136 {
1137         ipath_7220_config_jint(dd, dd->ipath_jint_idle_ticks,
1138                                dd->ipath_jint_max_packets);
1139         return 0;
1140 }
1141
1142 /**
1143  * ipath_setup_7220_setextled - set the state of the two external LEDs
1144  * @dd: the infinipath device
1145  * @lst: the L state
1146  * @ltst: the LT state
1147  *
1148  * These LEDs indicate the physical and logical state of IB link.
1149  * For this chip (at least with recommended board pinouts), LED1
1150  * is Yellow (logical state) and LED2 is Green (physical state),
1151  *
1152  * Note:  We try to match the Mellanox HCA LED behavior as best
1153  * we can.  Green indicates physical link state is OK (something is
1154  * plugged in, and we can train).
1155  * Amber indicates the link is logically up (ACTIVE).
1156  * Mellanox further blinks the amber LED to indicate data packet
1157  * activity, but we have no hardware support for that, so it would
1158  * require waking up every 10-20 msecs and checking the counters
1159  * on the chip, and then turning the LED off if appropriate.  That's
1160  * visible overhead, so not something we will do.
1161  *
1162  */
1163 static void ipath_setup_7220_setextled(struct ipath_devdata *dd, u64 lst,
1164                                        u64 ltst)
1165 {
1166         u64 extctl, ledblink = 0;
1167         unsigned long flags = 0;
1168
1169         /* the diags use the LED to indicate diag info, so we leave
1170          * the external LED alone when the diags are running */
1171         if (ipath_diag_inuse)
1172                 return;
1173
1174         /* Allow override of LED display for, e.g. Locating system in rack */
1175         if (dd->ipath_led_override) {
1176                 ltst = (dd->ipath_led_override & IPATH_LED_PHYS)
1177                         ? INFINIPATH_IBCS_LT_STATE_LINKUP
1178                         : INFINIPATH_IBCS_LT_STATE_DISABLED;
1179                 lst = (dd->ipath_led_override & IPATH_LED_LOG)
1180                         ? INFINIPATH_IBCS_L_STATE_ACTIVE
1181                         : INFINIPATH_IBCS_L_STATE_DOWN;
1182         }
1183
1184         spin_lock_irqsave(&dd->ipath_gpio_lock, flags);
1185         extctl = dd->ipath_extctrl & ~(INFINIPATH_EXTC_LED1PRIPORT_ON |
1186                                        INFINIPATH_EXTC_LED2PRIPORT_ON);
1187         if (ltst == INFINIPATH_IBCS_LT_STATE_LINKUP) {
1188                 extctl |= INFINIPATH_EXTC_LED1PRIPORT_ON;
1189                 /*
1190                  * counts are in chip clock (4ns) periods.
1191                  * This is 1/16 sec (66.6ms) on,
1192                  * 3/16 sec (187.5 ms) off, with packets rcvd
1193                  */
1194                 ledblink = ((66600*1000UL/4) << IBA7220_LEDBLINK_ON_SHIFT)
1195                         | ((187500*1000UL/4) << IBA7220_LEDBLINK_OFF_SHIFT);
1196         }
1197         if (lst == INFINIPATH_IBCS_L_STATE_ACTIVE)
1198                 extctl |= INFINIPATH_EXTC_LED2PRIPORT_ON;
1199         dd->ipath_extctrl = extctl;
1200         ipath_write_kreg(dd, dd->ipath_kregs->kr_extctrl, extctl);
1201         spin_unlock_irqrestore(&dd->ipath_gpio_lock, flags);
1202
1203         if (ledblink) /* blink the LED on packet receive */
1204                 ipath_write_kreg(dd, dd->ipath_kregs->kr_rcvpktledcnt,
1205                         ledblink);
1206 }
1207
1208 /*
1209  * Similar to pci_intx(pdev, 1), except that we make sure
1210  * msi is off...
1211  */
1212 static void ipath_enable_intx(struct pci_dev *pdev)
1213 {
1214         u16 cw, new;
1215         int pos;
1216
1217         /* first, turn on INTx */
1218         pci_read_config_word(pdev, PCI_COMMAND, &cw);
1219         new = cw & ~PCI_COMMAND_INTX_DISABLE;
1220         if (new != cw)
1221                 pci_write_config_word(pdev, PCI_COMMAND, new);
1222
1223         /* then turn off MSI */
1224         pos = pci_find_capability(pdev, PCI_CAP_ID_MSI);
1225         if (pos) {
1226                 pci_read_config_word(pdev, pos + PCI_MSI_FLAGS, &cw);
1227                 new = cw & ~PCI_MSI_FLAGS_ENABLE;
1228                 if (new != cw)
1229                         pci_write_config_word(pdev, pos + PCI_MSI_FLAGS, new);
1230         }
1231 }
1232
1233 static int ipath_msi_enabled(struct pci_dev *pdev)
1234 {
1235         int pos, ret = 0;
1236
1237         pos = pci_find_capability(pdev, PCI_CAP_ID_MSI);
1238         if (pos) {
1239                 u16 cw;
1240
1241                 pci_read_config_word(pdev, pos + PCI_MSI_FLAGS, &cw);
1242                 ret = !!(cw & PCI_MSI_FLAGS_ENABLE);
1243         }
1244         return ret;
1245 }
1246
1247 /*
1248  * disable msi interrupt if enabled, and clear the flag.
1249  * flag is used primarily for the fallback to INTx, but
1250  * is also used in reinit after reset as a flag.
1251  */
1252 static void ipath_7220_nomsi(struct ipath_devdata *dd)
1253 {
1254         dd->ipath_msi_lo = 0;
1255
1256         if (ipath_msi_enabled(dd->pcidev)) {
1257                 /*
1258                  * free, but don't zero; later kernels require
1259                  * it be freed before disable_msi, so the intx
1260                  * setup has to request it again.
1261                  */
1262                  if (dd->ipath_irq)
1263                         free_irq(dd->ipath_irq, dd);
1264                 pci_disable_msi(dd->pcidev);
1265         }
1266 }
1267
1268 /*
1269  * ipath_setup_7220_cleanup - clean up any per-chip chip-specific stuff
1270  * @dd: the infinipath device
1271  *
1272  * Nothing but msi interrupt cleanup for now.
1273  *
1274  * This is called during driver unload.
1275  */
1276 static void ipath_setup_7220_cleanup(struct ipath_devdata *dd)
1277 {
1278         ipath_7220_nomsi(dd);
1279 }
1280
1281
1282 static void ipath_7220_pcie_params(struct ipath_devdata *dd, u32 boardrev)
1283 {
1284         u16 linkstat, minwidth, speed;
1285         int pos;
1286
1287         pos = pci_find_capability(dd->pcidev, PCI_CAP_ID_EXP);
1288         if (!pos) {
1289                 ipath_dev_err(dd, "Can't find PCI Express capability!\n");
1290                 goto bail;
1291         }
1292
1293         pci_read_config_word(dd->pcidev, pos + PCI_EXP_LNKSTA,
1294                              &linkstat);
1295         /*
1296          * speed is bits 0-4, linkwidth is bits 4-8
1297          * no defines for them in headers
1298          */
1299         speed = linkstat & 0xf;
1300         linkstat >>= 4;
1301         linkstat &= 0x1f;
1302         dd->ipath_lbus_width = linkstat;
1303         switch (boardrev) {
1304         case 0:
1305         case 2:
1306         case 10:
1307         case 12:
1308                 minwidth = 16; /* x16 capable boards */
1309                 break;
1310         default:
1311                 minwidth = 8; /* x8 capable boards */
1312                 break;
1313         }
1314
1315         switch (speed) {
1316         case 1:
1317                 dd->ipath_lbus_speed = 2500; /* Gen1, 2.5GHz */
1318                 break;
1319         case 2:
1320                 dd->ipath_lbus_speed = 5000; /* Gen1, 5GHz */
1321                 break;
1322         default: /* not defined, assume gen1 */
1323                 dd->ipath_lbus_speed = 2500;
1324                 break;
1325         }
1326
1327         if (linkstat < minwidth)
1328                 ipath_dev_err(dd,
1329                         "PCIe width %u (x%u HCA), performance "
1330                         "reduced\n", linkstat, minwidth);
1331         else
1332                 ipath_cdbg(VERBOSE, "PCIe speed %u width %u (x%u HCA)\n",
1333                         dd->ipath_lbus_speed, linkstat, minwidth);
1334
1335         if (speed != 1)
1336                 ipath_dev_err(dd,
1337                         "PCIe linkspeed %u is incorrect; "
1338                         "should be 1 (2500)!\n", speed);
1339
1340 bail:
1341         /* fill in string, even on errors */
1342         snprintf(dd->ipath_lbus_info, sizeof(dd->ipath_lbus_info),
1343                 "PCIe,%uMHz,x%u\n",
1344                 dd->ipath_lbus_speed,
1345                 dd->ipath_lbus_width);
1346         return;
1347 }
1348
1349
1350 /**
1351  * ipath_setup_7220_config - setup PCIe config related stuff
1352  * @dd: the infinipath device
1353  * @pdev: the PCI device
1354  *
1355  * The pci_enable_msi() call will fail on systems with MSI quirks
1356  * such as those with AMD8131, even if the device of interest is not
1357  * attached to that device, (in the 2.6.13 - 2.6.15 kernels, at least, fixed
1358  * late in 2.6.16).
1359  * All that can be done is to edit the kernel source to remove the quirk
1360  * check until that is fixed.
1361  * We do not need to call enable_msi() for our HyperTransport chip,
1362  * even though it uses MSI, and we want to avoid the quirk warning, so
1363  * So we call enable_msi only for PCIe.  If we do end up needing
1364  * pci_enable_msi at some point in the future for HT, we'll move the
1365  * call back into the main init_one code.
1366  * We save the msi lo and hi values, so we can restore them after
1367  * chip reset (the kernel PCI infrastructure doesn't yet handle that
1368  * correctly).
1369  */
1370 static int ipath_setup_7220_config(struct ipath_devdata *dd,
1371                                    struct pci_dev *pdev)
1372 {
1373         int pos, ret = -1;
1374         u32 boardrev;
1375
1376         dd->ipath_msi_lo = 0;   /* used as a flag during reset processing */
1377
1378         pos = pci_find_capability(pdev, PCI_CAP_ID_MSI);
1379         if (!strcmp(int_type, "force_msi") || !strcmp(int_type, "auto"))
1380                 ret = pci_enable_msi(pdev);
1381         if (ret) {
1382                 if (!strcmp(int_type, "force_msi")) {
1383                         ipath_dev_err(dd, "pci_enable_msi failed: %d, "
1384                                       "force_msi is on, so not continuing.\n",
1385                                       ret);
1386                         return ret;
1387                 }
1388
1389                 ipath_enable_intx(pdev);
1390                 if (!strcmp(int_type, "auto"))
1391                         ipath_dev_err(dd, "pci_enable_msi failed: %d, "
1392                                       "falling back to INTx\n", ret);
1393         } else if (pos) {
1394                 u16 control;
1395                 pci_read_config_dword(pdev, pos + PCI_MSI_ADDRESS_LO,
1396                                       &dd->ipath_msi_lo);
1397                 pci_read_config_dword(pdev, pos + PCI_MSI_ADDRESS_HI,
1398                                       &dd->ipath_msi_hi);
1399                 pci_read_config_word(pdev, pos + PCI_MSI_FLAGS,
1400                                      &control);
1401                 /* now save the data (vector) info */
1402                 pci_read_config_word(pdev,
1403                                      pos + ((control & PCI_MSI_FLAGS_64BIT)
1404                                             ? PCI_MSI_DATA_64 :
1405                                             PCI_MSI_DATA_32),
1406                                      &dd->ipath_msi_data);
1407         } else
1408                 ipath_dev_err(dd, "Can't find MSI capability, "
1409                               "can't save MSI settings for reset\n");
1410
1411         dd->ipath_irq = pdev->irq;
1412
1413         /*
1414          * We save the cachelinesize also, although it doesn't
1415          * really matter.
1416          */
1417         pci_read_config_byte(pdev, PCI_CACHE_LINE_SIZE,
1418                              &dd->ipath_pci_cacheline);
1419
1420         /*
1421          * this function called early, ipath_boardrev not set yet.  Can't
1422          * use ipath_read_kreg64() yet, too early in init, so use readq()
1423          */
1424         boardrev = (readq(&dd->ipath_kregbase[dd->ipath_kregs->kr_revision])
1425                  >> INFINIPATH_R_BOARDID_SHIFT) & INFINIPATH_R_BOARDID_MASK;
1426
1427         ipath_7220_pcie_params(dd, boardrev);
1428
1429         dd->ipath_flags |= IPATH_NODMA_RTAIL | IPATH_HAS_SEND_DMA |
1430                 IPATH_HAS_PBC_CNT | IPATH_HAS_THRESH_UPDATE;
1431         dd->ipath_pioupd_thresh = 4U; /* set default update threshold */
1432         return 0;
1433 }
1434
1435 static void ipath_init_7220_variables(struct ipath_devdata *dd)
1436 {
1437         /*
1438          * setup the register offsets, since they are different for each
1439          * chip
1440          */
1441         dd->ipath_kregs = &ipath_7220_kregs;
1442         dd->ipath_cregs = &ipath_7220_cregs;
1443
1444         /*
1445          * bits for selecting i2c direction and values,
1446          * used for I2C serial flash
1447          */
1448         dd->ipath_gpio_sda_num = _IPATH_GPIO_SDA_NUM;
1449         dd->ipath_gpio_scl_num = _IPATH_GPIO_SCL_NUM;
1450         dd->ipath_gpio_sda = IPATH_GPIO_SDA;
1451         dd->ipath_gpio_scl = IPATH_GPIO_SCL;
1452
1453         /*
1454          * Fill in data for field-values that change in IBA7220.
1455          * We dynamically specify only the mask for LINKTRAININGSTATE
1456          * and only the shift for LINKSTATE, as they are the only ones
1457          * that change.  Also precalculate the 3 link states of interest
1458          * and the combined mask.
1459          */
1460         dd->ibcs_ls_shift = IBA7220_IBCS_LINKSTATE_SHIFT;
1461         dd->ibcs_lts_mask = IBA7220_IBCS_LINKTRAININGSTATE_MASK;
1462         dd->ibcs_mask = (INFINIPATH_IBCS_LINKSTATE_MASK <<
1463                 dd->ibcs_ls_shift) | dd->ibcs_lts_mask;
1464         dd->ib_init = (INFINIPATH_IBCS_LT_STATE_LINKUP <<
1465                 INFINIPATH_IBCS_LINKTRAININGSTATE_SHIFT) |
1466                 (INFINIPATH_IBCS_L_STATE_INIT << dd->ibcs_ls_shift);
1467         dd->ib_arm = (INFINIPATH_IBCS_LT_STATE_LINKUP <<
1468                 INFINIPATH_IBCS_LINKTRAININGSTATE_SHIFT) |
1469                 (INFINIPATH_IBCS_L_STATE_ARM << dd->ibcs_ls_shift);
1470         dd->ib_active = (INFINIPATH_IBCS_LT_STATE_LINKUP <<
1471                 INFINIPATH_IBCS_LINKTRAININGSTATE_SHIFT) |
1472                 (INFINIPATH_IBCS_L_STATE_ACTIVE << dd->ibcs_ls_shift);
1473
1474         /*
1475          * Fill in data for ibcc field-values that change in IBA7220.
1476          * We dynamically specify only the mask for LINKINITCMD
1477          * and only the shift for LINKCMD and MAXPKTLEN, as they are
1478          * the only ones that change.
1479          */
1480         dd->ibcc_lic_mask = IBA7220_IBCC_LINKINITCMD_MASK;
1481         dd->ibcc_lc_shift = IBA7220_IBCC_LINKCMD_SHIFT;
1482         dd->ibcc_mpl_shift = IBA7220_IBCC_MAXPKTLEN_SHIFT;
1483
1484         /* Fill in shifts for RcvCtrl. */
1485         dd->ipath_r_portenable_shift = INFINIPATH_R_PORTENABLE_SHIFT;
1486         dd->ipath_r_intravail_shift = IBA7220_R_INTRAVAIL_SHIFT;
1487         dd->ipath_r_tailupd_shift = IBA7220_R_TAILUPD_SHIFT;
1488         dd->ipath_r_portcfg_shift = IBA7220_R_PORTCFG_SHIFT;
1489
1490         /* variables for sanity checking interrupt and errors */
1491         dd->ipath_hwe_bitsextant =
1492                 (INFINIPATH_HWE_RXEMEMPARITYERR_MASK <<
1493                  INFINIPATH_HWE_RXEMEMPARITYERR_SHIFT) |
1494                 (INFINIPATH_HWE_TXEMEMPARITYERR_MASK <<
1495                  INFINIPATH_HWE_TXEMEMPARITYERR_SHIFT) |
1496                 (INFINIPATH_HWE_PCIEMEMPARITYERR_MASK <<
1497                  INFINIPATH_HWE_PCIEMEMPARITYERR_SHIFT) |
1498                 INFINIPATH_HWE_PCIE1PLLFAILED |
1499                 INFINIPATH_HWE_PCIE0PLLFAILED |
1500                 INFINIPATH_HWE_PCIEPOISONEDTLP |
1501                 INFINIPATH_HWE_PCIECPLTIMEOUT |
1502                 INFINIPATH_HWE_PCIEBUSPARITYXTLH |
1503                 INFINIPATH_HWE_PCIEBUSPARITYXADM |
1504                 INFINIPATH_HWE_PCIEBUSPARITYRADM |
1505                 INFINIPATH_HWE_MEMBISTFAILED |
1506                 INFINIPATH_HWE_COREPLL_FBSLIP |
1507                 INFINIPATH_HWE_COREPLL_RFSLIP |
1508                 INFINIPATH_HWE_SERDESPLLFAILED |
1509                 INFINIPATH_HWE_IBCBUSTOSPCPARITYERR |
1510                 INFINIPATH_HWE_IBCBUSFRSPCPARITYERR |
1511                 INFINIPATH_HWE_PCIECPLDATAQUEUEERR |
1512                 INFINIPATH_HWE_PCIECPLHDRQUEUEERR |
1513                 INFINIPATH_HWE_SDMAMEMREADERR |
1514                 INFINIPATH_HWE_CLK_UC_PLLNOTLOCKED |
1515                 INFINIPATH_HWE_PCIESERDESQ0PCLKNOTDETECT |
1516                 INFINIPATH_HWE_PCIESERDESQ1PCLKNOTDETECT |
1517                 INFINIPATH_HWE_PCIESERDESQ2PCLKNOTDETECT |
1518                 INFINIPATH_HWE_PCIESERDESQ3PCLKNOTDETECT |
1519                 INFINIPATH_HWE_DDSRXEQMEMORYPARITYERR |
1520                 INFINIPATH_HWE_IB_UC_MEMORYPARITYERR |
1521                 INFINIPATH_HWE_PCIE_UC_OCT0MEMORYPARITYERR |
1522                 INFINIPATH_HWE_PCIE_UC_OCT1MEMORYPARITYERR;
1523         dd->ipath_i_bitsextant =
1524                 INFINIPATH_I_SDMAINT | INFINIPATH_I_SDMADISABLED |
1525                 (INFINIPATH_I_RCVURG_MASK << INFINIPATH_I_RCVURG_SHIFT) |
1526                 (INFINIPATH_I_RCVAVAIL_MASK <<
1527                  INFINIPATH_I_RCVAVAIL_SHIFT) |
1528                 INFINIPATH_I_ERROR | INFINIPATH_I_SPIOSENT |
1529                 INFINIPATH_I_SPIOBUFAVAIL | INFINIPATH_I_GPIO |
1530                 INFINIPATH_I_JINT | INFINIPATH_I_SERDESTRIMDONE;
1531         dd->ipath_e_bitsextant =
1532                 INFINIPATH_E_RFORMATERR | INFINIPATH_E_RVCRC |
1533                 INFINIPATH_E_RICRC | INFINIPATH_E_RMINPKTLEN |
1534                 INFINIPATH_E_RMAXPKTLEN | INFINIPATH_E_RLONGPKTLEN |
1535                 INFINIPATH_E_RSHORTPKTLEN | INFINIPATH_E_RUNEXPCHAR |
1536                 INFINIPATH_E_RUNSUPVL | INFINIPATH_E_REBP |
1537                 INFINIPATH_E_RIBFLOW | INFINIPATH_E_RBADVERSION |
1538                 INFINIPATH_E_RRCVEGRFULL | INFINIPATH_E_RRCVHDRFULL |
1539                 INFINIPATH_E_RBADTID | INFINIPATH_E_RHDRLEN |
1540                 INFINIPATH_E_RHDR | INFINIPATH_E_RIBLOSTLINK |
1541                 INFINIPATH_E_SENDSPECIALTRIGGER |
1542                 INFINIPATH_E_SDMADISABLED | INFINIPATH_E_SMINPKTLEN |
1543                 INFINIPATH_E_SMAXPKTLEN | INFINIPATH_E_SUNDERRUN |
1544                 INFINIPATH_E_SPKTLEN | INFINIPATH_E_SDROPPEDSMPPKT |
1545                 INFINIPATH_E_SDROPPEDDATAPKT |
1546                 INFINIPATH_E_SPIOARMLAUNCH | INFINIPATH_E_SUNEXPERRPKTNUM |
1547                 INFINIPATH_E_SUNSUPVL | INFINIPATH_E_SENDBUFMISUSE |
1548                 INFINIPATH_E_SDMAGENMISMATCH | INFINIPATH_E_SDMAOUTOFBOUND |
1549                 INFINIPATH_E_SDMATAILOUTOFBOUND | INFINIPATH_E_SDMABASE |
1550                 INFINIPATH_E_SDMA1STDESC | INFINIPATH_E_SDMARPYTAG |
1551                 INFINIPATH_E_SDMADWEN | INFINIPATH_E_SDMAMISSINGDW |
1552                 INFINIPATH_E_SDMAUNEXPDATA |
1553                 INFINIPATH_E_IBSTATUSCHANGED | INFINIPATH_E_INVALIDADDR |
1554                 INFINIPATH_E_RESET | INFINIPATH_E_HARDWARE |
1555                 INFINIPATH_E_SDMADESCADDRMISALIGN |
1556                 INFINIPATH_E_INVALIDEEPCMD;
1557
1558         dd->ipath_i_rcvavail_mask = INFINIPATH_I_RCVAVAIL_MASK;
1559         dd->ipath_i_rcvurg_mask = INFINIPATH_I_RCVURG_MASK;
1560         dd->ipath_i_rcvavail_shift = INFINIPATH_I_RCVAVAIL_SHIFT;
1561         dd->ipath_i_rcvurg_shift = INFINIPATH_I_RCVURG_SHIFT;
1562         dd->ipath_flags |= IPATH_INTREG_64 | IPATH_HAS_MULT_IB_SPEED
1563                 | IPATH_HAS_LINK_LATENCY;
1564
1565         /*
1566          * EEPROM error log 0 is TXE Parity errors. 1 is RXE Parity.
1567          * 2 is Some Misc, 3 is reserved for future.
1568          */
1569         dd->ipath_eep_st_masks[0].hwerrs_to_log =
1570                 INFINIPATH_HWE_TXEMEMPARITYERR_MASK <<
1571                 INFINIPATH_HWE_TXEMEMPARITYERR_SHIFT;
1572
1573         dd->ipath_eep_st_masks[1].hwerrs_to_log =
1574                 INFINIPATH_HWE_RXEMEMPARITYERR_MASK <<
1575                 INFINIPATH_HWE_RXEMEMPARITYERR_SHIFT;
1576
1577         dd->ipath_eep_st_masks[2].errs_to_log = INFINIPATH_E_RESET;
1578
1579         ipath_linkrecovery = 0;
1580
1581         init_waitqueue_head(&dd->ipath_autoneg_wait);
1582         INIT_DELAYED_WORK(&dd->ipath_autoneg_work,  autoneg_work);
1583
1584         dd->ipath_link_width_supported = IB_WIDTH_1X | IB_WIDTH_4X;
1585         dd->ipath_link_speed_supported = IPATH_IB_SDR | IPATH_IB_DDR;
1586
1587         dd->ipath_link_width_enabled = dd->ipath_link_width_supported;
1588         dd->ipath_link_speed_enabled = dd->ipath_link_speed_supported;
1589         /*
1590          * set the initial values to reasonable default, will be set
1591          * for real when link is up.
1592          */
1593         dd->ipath_link_width_active = IB_WIDTH_4X;
1594         dd->ipath_link_speed_active = IPATH_IB_SDR;
1595         dd->delay_mult = rate_to_delay[0][1];
1596 }
1597
1598
1599 /*
1600  * Setup the MSI stuff again after a reset.  I'd like to just call
1601  * pci_enable_msi() and request_irq() again, but when I do that,
1602  * the MSI enable bit doesn't get set in the command word, and
1603  * we switch to to a different interrupt vector, which is confusing,
1604  * so I instead just do it all inline.  Perhaps somehow can tie this
1605  * into the PCIe hotplug support at some point
1606  * Note, because I'm doing it all here, I don't call pci_disable_msi()
1607  * or free_irq() at the start of ipath_setup_7220_reset().
1608  */
1609 static int ipath_reinit_msi(struct ipath_devdata *dd)
1610 {
1611         int ret = 0;
1612
1613         int pos;
1614         u16 control;
1615         if (!dd->ipath_msi_lo) /* Using intX, or init problem */
1616                 goto bail;
1617
1618         pos = pci_find_capability(dd->pcidev, PCI_CAP_ID_MSI);
1619         if (!pos) {
1620                 ipath_dev_err(dd, "Can't find MSI capability, "
1621                               "can't restore MSI settings\n");
1622                 goto bail;
1623         }
1624         ipath_cdbg(VERBOSE, "Writing msi_lo 0x%x to config offset 0x%x\n",
1625                    dd->ipath_msi_lo, pos + PCI_MSI_ADDRESS_LO);
1626         pci_write_config_dword(dd->pcidev, pos + PCI_MSI_ADDRESS_LO,
1627                                dd->ipath_msi_lo);
1628         ipath_cdbg(VERBOSE, "Writing msi_lo 0x%x to config offset 0x%x\n",
1629                    dd->ipath_msi_hi, pos + PCI_MSI_ADDRESS_HI);
1630         pci_write_config_dword(dd->pcidev, pos + PCI_MSI_ADDRESS_HI,
1631                                dd->ipath_msi_hi);
1632         pci_read_config_word(dd->pcidev, pos + PCI_MSI_FLAGS, &control);
1633         if (!(control & PCI_MSI_FLAGS_ENABLE)) {
1634                 ipath_cdbg(VERBOSE, "MSI control at off %x was %x, "
1635                            "setting MSI enable (%x)\n", pos + PCI_MSI_FLAGS,
1636                            control, control | PCI_MSI_FLAGS_ENABLE);
1637                 control |= PCI_MSI_FLAGS_ENABLE;
1638                 pci_write_config_word(dd->pcidev, pos + PCI_MSI_FLAGS,
1639                                       control);
1640         }
1641         /* now rewrite the data (vector) info */
1642         pci_write_config_word(dd->pcidev, pos +
1643                               ((control & PCI_MSI_FLAGS_64BIT) ? 12 : 8),
1644                               dd->ipath_msi_data);
1645         ret = 1;
1646
1647 bail:
1648         if (!ret) {
1649                 ipath_dbg("Using INTx, MSI disabled or not configured\n");
1650                 ipath_enable_intx(dd->pcidev);
1651                 ret = 1;
1652         }
1653         /*
1654          * We restore the cachelinesize also, although it doesn't really
1655          * matter.
1656          */
1657         pci_write_config_byte(dd->pcidev, PCI_CACHE_LINE_SIZE,
1658                               dd->ipath_pci_cacheline);
1659         /* and now set the pci master bit again */
1660         pci_set_master(dd->pcidev);
1661
1662         return ret;
1663 }
1664
1665 /*
1666  * This routine sleeps, so it can only be called from user context, not
1667  * from interrupt context.  If we need interrupt context, we can split
1668  * it into two routines.
1669  */
1670 static int ipath_setup_7220_reset(struct ipath_devdata *dd)
1671 {
1672         u64 val;
1673         int i;
1674         int ret;
1675         u16 cmdval;
1676
1677         pci_read_config_word(dd->pcidev, PCI_COMMAND, &cmdval);
1678
1679         /* Use dev_err so it shows up in logs, etc. */
1680         ipath_dev_err(dd, "Resetting InfiniPath unit %u\n", dd->ipath_unit);
1681
1682         /* keep chip from being accessed in a few places */
1683         dd->ipath_flags &= ~(IPATH_INITTED | IPATH_PRESENT);
1684         val = dd->ipath_control | INFINIPATH_C_RESET;
1685         ipath_write_kreg(dd, dd->ipath_kregs->kr_control, val);
1686         mb();
1687
1688         for (i = 1; i <= 5; i++) {
1689                 int r;
1690
1691                 /*
1692                  * Allow MBIST, etc. to complete; longer on each retry.
1693                  * We sometimes get machine checks from bus timeout if no
1694                  * response, so for now, make it *really* long.
1695                  */
1696                 msleep(1000 + (1 + i) * 2000);
1697                 r = pci_write_config_dword(dd->pcidev, PCI_BASE_ADDRESS_0,
1698                                            dd->ipath_pcibar0);
1699                 if (r)
1700                         ipath_dev_err(dd, "rewrite of BAR0 failed: %d\n", r);
1701                 r = pci_write_config_dword(dd->pcidev, PCI_BASE_ADDRESS_1,
1702                                            dd->ipath_pcibar1);
1703                 if (r)
1704                         ipath_dev_err(dd, "rewrite of BAR1 failed: %d\n", r);
1705                 /* now re-enable memory access */
1706                 pci_write_config_word(dd->pcidev, PCI_COMMAND, cmdval);
1707                 r = pci_enable_device(dd->pcidev);
1708                 if (r)
1709                         ipath_dev_err(dd, "pci_enable_device failed after "
1710                                       "reset: %d\n", r);
1711                 /*
1712                  * whether it fully enabled or not, mark as present,
1713                  * again (but not INITTED)
1714                  */
1715                 dd->ipath_flags |= IPATH_PRESENT;
1716                 val = ipath_read_kreg64(dd, dd->ipath_kregs->kr_revision);
1717                 if (val == dd->ipath_revision) {
1718                         ipath_cdbg(VERBOSE, "Got matching revision "
1719                                    "register %llx on try %d\n",
1720                                    (unsigned long long) val, i);
1721                         ret = ipath_reinit_msi(dd);
1722                         goto bail;
1723                 }
1724                 /* Probably getting -1 back */
1725                 ipath_dbg("Didn't get expected revision register, "
1726                           "got %llx, try %d\n", (unsigned long long) val,
1727                           i + 1);
1728         }
1729         ret = 0; /* failed */
1730
1731 bail:
1732         if (ret)
1733                 ipath_7220_pcie_params(dd, dd->ipath_boardrev);
1734
1735         return ret;
1736 }
1737
1738 /**
1739  * ipath_7220_put_tid - write a TID to the chip
1740  * @dd: the infinipath device
1741  * @tidptr: pointer to the expected TID (in chip) to udpate
1742  * @tidtype: 0 for eager, 1 for expected
1743  * @pa: physical address of in memory buffer; ipath_tidinvalid if freeing
1744  *
1745  * This exists as a separate routine to allow for selection of the
1746  * appropriate "flavor". The static calls in cleanup just use the
1747  * revision-agnostic form, as they are not performance critical.
1748  */
1749 static void ipath_7220_put_tid(struct ipath_devdata *dd, u64 __iomem *tidptr,
1750                              u32 type, unsigned long pa)
1751 {
1752         if (pa != dd->ipath_tidinvalid) {
1753                 u64 chippa = pa >> IBA7220_TID_PA_SHIFT;
1754
1755                 /* paranoia checks */
1756                 if (pa != (chippa << IBA7220_TID_PA_SHIFT)) {
1757                         dev_info(&dd->pcidev->dev, "BUG: physaddr %lx "
1758                                  "not 2KB aligned!\n", pa);
1759                         return;
1760                 }
1761                 if (chippa >= (1UL << IBA7220_TID_SZ_SHIFT)) {
1762                         ipath_dev_err(dd,
1763                                       "BUG: Physical page address 0x%lx "
1764                                       "larger than supported\n", pa);
1765                         return;
1766                 }
1767
1768                 if (type == RCVHQ_RCV_TYPE_EAGER)
1769                         chippa |= dd->ipath_tidtemplate;
1770                 else /* for now, always full 4KB page */
1771                         chippa |= IBA7220_TID_SZ_4K;
1772                 writeq(chippa, tidptr);
1773         } else
1774                 writeq(pa, tidptr);
1775         mmiowb();
1776 }
1777
1778 /**
1779  * ipath_7220_clear_tid - clear all TID entries for a port, expected and eager
1780  * @dd: the infinipath device
1781  * @port: the port
1782  *
1783  * clear all TID entries for a port, expected and eager.
1784  * Used from ipath_close().  On this chip, TIDs are only 32 bits,
1785  * not 64, but they are still on 64 bit boundaries, so tidbase
1786  * is declared as u64 * for the pointer math, even though we write 32 bits
1787  */
1788 static void ipath_7220_clear_tids(struct ipath_devdata *dd, unsigned port)
1789 {
1790         u64 __iomem *tidbase;
1791         unsigned long tidinv;
1792         int i;
1793
1794         if (!dd->ipath_kregbase)
1795                 return;
1796
1797         ipath_cdbg(VERBOSE, "Invalidate TIDs for port %u\n", port);
1798
1799         tidinv = dd->ipath_tidinvalid;
1800         tidbase = (u64 __iomem *)
1801                 ((char __iomem *)(dd->ipath_kregbase) +
1802                  dd->ipath_rcvtidbase +
1803                  port * dd->ipath_rcvtidcnt * sizeof(*tidbase));
1804
1805         for (i = 0; i < dd->ipath_rcvtidcnt; i++)
1806                 ipath_7220_put_tid(dd, &tidbase[i], RCVHQ_RCV_TYPE_EXPECTED,
1807                                    tidinv);
1808
1809         tidbase = (u64 __iomem *)
1810                 ((char __iomem *)(dd->ipath_kregbase) +
1811                  dd->ipath_rcvegrbase + port_egrtid_idx(dd, port)
1812                  * sizeof(*tidbase));
1813
1814         for (i = port ? dd->ipath_rcvegrcnt : dd->ipath_p0_rcvegrcnt; i; i--)
1815                 ipath_7220_put_tid(dd, &tidbase[i-1], RCVHQ_RCV_TYPE_EAGER,
1816                         tidinv);
1817 }
1818
1819 /**
1820  * ipath_7220_tidtemplate - setup constants for TID updates
1821  * @dd: the infinipath device
1822  *
1823  * We setup stuff that we use a lot, to avoid calculating each time
1824  */
1825 static void ipath_7220_tidtemplate(struct ipath_devdata *dd)
1826 {
1827         /* For now, we always allocate 4KB buffers (at init) so we can
1828          * receive max size packets.  We may want a module parameter to
1829          * specify 2KB or 4KB and/or make be per port instead of per device
1830          * for those who want to reduce memory footprint.  Note that the
1831          * ipath_rcvhdrentsize size must be large enough to hold the largest
1832          * IB header (currently 96 bytes) that we expect to handle (plus of
1833          * course the 2 dwords of RHF).
1834          */
1835         if (dd->ipath_rcvegrbufsize == 2048)
1836                 dd->ipath_tidtemplate = IBA7220_TID_SZ_2K;
1837         else if (dd->ipath_rcvegrbufsize == 4096)
1838                 dd->ipath_tidtemplate = IBA7220_TID_SZ_4K;
1839         else {
1840                 dev_info(&dd->pcidev->dev, "BUG: unsupported egrbufsize "
1841                          "%u, using %u\n", dd->ipath_rcvegrbufsize,
1842                          4096);
1843                 dd->ipath_tidtemplate = IBA7220_TID_SZ_4K;
1844         }
1845         dd->ipath_tidinvalid = 0;
1846 }
1847
1848 static int ipath_7220_early_init(struct ipath_devdata *dd)
1849 {
1850         u32 i, s;
1851
1852         if (strcmp(int_type, "auto") &&
1853             strcmp(int_type, "force_msi") &&
1854             strcmp(int_type, "force_intx")) {
1855                 ipath_dev_err(dd, "Invalid interrupt_type: '%s', expecting "
1856                               "auto, force_msi or force_intx\n", int_type);
1857                 return -EINVAL;
1858         }
1859
1860         /*
1861          * Control[4] has been added to change the arbitration within
1862          * the SDMA engine between favoring data fetches over descriptor
1863          * fetches.  ipath_sdma_fetch_arb==0 gives data fetches priority.
1864          */
1865         if (ipath_sdma_fetch_arb && (dd->ipath_minrev > 1))
1866                 dd->ipath_control |= 1<<4;
1867
1868         dd->ipath_flags |= IPATH_4BYTE_TID;
1869
1870         /*
1871          * For openfabrics, we need to be able to handle an IB header of
1872          * 24 dwords.  HT chip has arbitrary sized receive buffers, so we
1873          * made them the same size as the PIO buffers.  This chip does not
1874          * handle arbitrary size buffers, so we need the header large enough
1875          * to handle largest IB header, but still have room for a 2KB MTU
1876          * standard IB packet.
1877          */
1878         dd->ipath_rcvhdrentsize = 24;
1879         dd->ipath_rcvhdrsize = IPATH_DFLT_RCVHDRSIZE;
1880         dd->ipath_rhf_offset =
1881                 dd->ipath_rcvhdrentsize - sizeof(u64) / sizeof(u32);
1882
1883         dd->ipath_rcvegrbufsize = ipath_mtu4096 ? 4096 : 2048;
1884         /*
1885          * the min() check here is currently a nop, but it may not always
1886          * be, depending on just how we do ipath_rcvegrbufsize
1887          */
1888         dd->ipath_ibmaxlen = min(ipath_mtu4096 ? dd->ipath_piosize4k :
1889                                  dd->ipath_piosize2k,
1890                                  dd->ipath_rcvegrbufsize +
1891                                  (dd->ipath_rcvhdrentsize << 2));
1892         dd->ipath_init_ibmaxlen = dd->ipath_ibmaxlen;
1893
1894         ipath_7220_config_jint(dd, INFINIPATH_JINT_DEFAULT_IDLE_TICKS,
1895                                INFINIPATH_JINT_DEFAULT_MAX_PACKETS);
1896
1897         if (dd->ipath_boardrev) /* no eeprom on emulator */
1898                 ipath_get_eeprom_info(dd);
1899
1900         /* start of code to check and print procmon */
1901         s = ipath_read_kreg32(dd, IPATH_KREG_OFFSET(ProcMon));
1902         s &= ~(1U<<31); /* clear done bit */
1903         s |= 1U<<14; /* clear counter (write 1 to clear) */
1904         ipath_write_kreg(dd, IPATH_KREG_OFFSET(ProcMon), s);
1905         /* make sure clear_counter low long enough before start */
1906         ipath_read_kreg32(dd, dd->ipath_kregs->kr_scratch);
1907         ipath_read_kreg32(dd, dd->ipath_kregs->kr_scratch);
1908
1909         s &= ~(1U<<14); /* allow counter to count (before starting) */
1910         ipath_write_kreg(dd, IPATH_KREG_OFFSET(ProcMon), s);
1911         ipath_read_kreg32(dd, dd->ipath_kregs->kr_scratch);
1912         ipath_read_kreg32(dd, dd->ipath_kregs->kr_scratch);
1913         s = ipath_read_kreg32(dd, IPATH_KREG_OFFSET(ProcMon));
1914
1915         s |= 1U<<15; /* start the counter */
1916         s &= ~(1U<<31); /* clear done bit */
1917         s &= ~0x7ffU; /* clear frequency bits */
1918         s |= 0xe29; /* set frequency bits, in case cleared */
1919         ipath_write_kreg(dd, IPATH_KREG_OFFSET(ProcMon), s);
1920
1921         s = 0;
1922         for (i = 500; i > 0 && !(s&(1ULL<<31)); i--) {
1923                 ipath_read_kreg32(dd, dd->ipath_kregs->kr_scratch);
1924                 s = ipath_read_kreg32(dd, IPATH_KREG_OFFSET(ProcMon));
1925         }
1926         if (!(s&(1U<<31)))
1927                 ipath_dev_err(dd, "ProcMon register not valid: 0x%x\n", s);
1928         else
1929                 ipath_dbg("ProcMon=0x%x, count=0x%x\n", s, (s>>16)&0x1ff);
1930
1931         return 0;
1932 }
1933
1934 /**
1935  * ipath_init_7220_get_base_info - set chip-specific flags for user code
1936  * @pd: the infinipath port
1937  * @kbase: ipath_base_info pointer
1938  *
1939  * We set the PCIE flag because the lower bandwidth on PCIe vs
1940  * HyperTransport can affect some user packet algorithims.
1941  */
1942 static int ipath_7220_get_base_info(struct ipath_portdata *pd, void *kbase)
1943 {
1944         struct ipath_base_info *kinfo = kbase;
1945
1946         kinfo->spi_runtime_flags |=
1947                 IPATH_RUNTIME_PCIE | IPATH_RUNTIME_NODMA_RTAIL |
1948                 IPATH_RUNTIME_SDMA;
1949
1950         return 0;
1951 }
1952
1953 static void ipath_7220_free_irq(struct ipath_devdata *dd)
1954 {
1955         free_irq(dd->ipath_irq, dd);
1956         dd->ipath_irq = 0;
1957 }
1958
1959 static struct ipath_message_header *
1960 ipath_7220_get_msgheader(struct ipath_devdata *dd, __le32 *rhf_addr)
1961 {
1962         u32 offset = ipath_hdrget_offset(rhf_addr);
1963
1964         return (struct ipath_message_header *)
1965                 (rhf_addr - dd->ipath_rhf_offset + offset);
1966 }
1967
1968 static void ipath_7220_config_ports(struct ipath_devdata *dd, ushort cfgports)
1969 {
1970         u32 nchipports;
1971
1972         nchipports = ipath_read_kreg32(dd, dd->ipath_kregs->kr_portcnt);
1973         if (!cfgports) {
1974                 int ncpus = num_online_cpus();
1975
1976                 if (ncpus <= 4)
1977                         dd->ipath_portcnt = 5;
1978                 else if (ncpus <= 8)
1979                         dd->ipath_portcnt = 9;
1980                 if (dd->ipath_portcnt)
1981                         ipath_dbg("Auto-configured for %u ports, %d cpus "
1982                                 "online\n", dd->ipath_portcnt, ncpus);
1983         } else if (cfgports <= nchipports)
1984                 dd->ipath_portcnt = cfgports;
1985         if (!dd->ipath_portcnt) /* none of the above, set to max */
1986                 dd->ipath_portcnt = nchipports;
1987         /*
1988          * chip can be configured for 5, 9, or 17 ports, and choice
1989          * affects number of eager TIDs per port (1K, 2K, 4K).
1990          */
1991         if (dd->ipath_portcnt > 9)
1992                 dd->ipath_rcvctrl |= 2ULL << IBA7220_R_PORTCFG_SHIFT;
1993         else if (dd->ipath_portcnt > 5)
1994                 dd->ipath_rcvctrl |= 1ULL << IBA7220_R_PORTCFG_SHIFT;
1995         /* else configure for default 5 receive ports */
1996         ipath_write_kreg(dd, dd->ipath_kregs->kr_rcvctrl,
1997                          dd->ipath_rcvctrl);
1998         dd->ipath_p0_rcvegrcnt = 2048; /* always */
1999         if (dd->ipath_flags & IPATH_HAS_SEND_DMA)
2000                 dd->ipath_pioreserved = 3; /* kpiobufs used for PIO */
2001 }
2002
2003
2004 static int ipath_7220_get_ib_cfg(struct ipath_devdata *dd, int which)
2005 {
2006         int lsb, ret = 0;
2007         u64 maskr; /* right-justified mask */
2008
2009         switch (which) {
2010         case IPATH_IB_CFG_HRTBT: /* Get Heartbeat off/enable/auto */
2011                 lsb = IBA7220_IBC_HRTBT_SHIFT;
2012                 maskr = IBA7220_IBC_HRTBT_MASK;
2013                 break;
2014
2015         case IPATH_IB_CFG_LWID_ENB: /* Get allowed Link-width */
2016                 ret = dd->ipath_link_width_enabled;
2017                 goto done;
2018
2019         case IPATH_IB_CFG_LWID: /* Get currently active Link-width */
2020                 ret = dd->ipath_link_width_active;
2021                 goto done;
2022
2023         case IPATH_IB_CFG_SPD_ENB: /* Get allowed Link speeds */
2024                 ret = dd->ipath_link_speed_enabled;
2025                 goto done;
2026
2027         case IPATH_IB_CFG_SPD: /* Get current Link spd */
2028                 ret = dd->ipath_link_speed_active;
2029                 goto done;
2030
2031         case IPATH_IB_CFG_RXPOL_ENB: /* Get Auto-RX-polarity enable */
2032                 lsb = IBA7220_IBC_RXPOL_SHIFT;
2033                 maskr = IBA7220_IBC_RXPOL_MASK;
2034                 break;
2035
2036         case IPATH_IB_CFG_LREV_ENB: /* Get Auto-Lane-reversal enable */
2037                 lsb = IBA7220_IBC_LREV_SHIFT;
2038                 maskr = IBA7220_IBC_LREV_MASK;
2039                 break;
2040
2041         case IPATH_IB_CFG_LINKLATENCY:
2042                 ret = ipath_read_kreg64(dd, dd->ipath_kregs->kr_ibcddrstatus)
2043                         & IBA7220_DDRSTAT_LINKLAT_MASK;
2044                 goto done;
2045
2046         default:
2047                 ret = -ENOTSUPP;
2048                 goto done;
2049         }
2050         ret = (int)((dd->ipath_ibcddrctrl >> lsb) & maskr);
2051 done:
2052         return ret;
2053 }
2054
2055 static int ipath_7220_set_ib_cfg(struct ipath_devdata *dd, int which, u32 val)
2056 {
2057         int lsb, ret = 0, setforce = 0;
2058         u64 maskr; /* right-justified mask */
2059
2060         switch (which) {
2061         case IPATH_IB_CFG_LIDLMC:
2062                 /*
2063                  * Set LID and LMC. Combined to avoid possible hazard
2064                  * caller puts LMC in 16MSbits, DLID in 16LSbits of val
2065                  */
2066                 lsb = IBA7220_IBC_DLIDLMC_SHIFT;
2067                 maskr = IBA7220_IBC_DLIDLMC_MASK;
2068                 break;
2069
2070         case IPATH_IB_CFG_HRTBT: /* set Heartbeat off/enable/auto */
2071                 if (val & IPATH_IB_HRTBT_ON &&
2072                         (dd->ipath_flags & IPATH_NO_HRTBT))
2073                         goto bail;
2074                 lsb = IBA7220_IBC_HRTBT_SHIFT;
2075                 maskr = IBA7220_IBC_HRTBT_MASK;
2076                 break;
2077
2078         case IPATH_IB_CFG_LWID_ENB: /* set allowed Link-width */
2079                 /*
2080                  * As with speed, only write the actual register if
2081                  * the link is currently down, otherwise takes effect
2082                  * on next link change.
2083                  */
2084                 dd->ipath_link_width_enabled = val;
2085                 if ((dd->ipath_flags & (IPATH_LINKDOWN|IPATH_LINKINIT)) !=
2086                         IPATH_LINKDOWN)
2087                         goto bail;
2088                 /*
2089                  * We set the IPATH_IB_FORCE_NOTIFY bit so updown
2090                  * will get called because we want update
2091                  * link_width_active, and the change may not take
2092                  * effect for some time (if we are in POLL), so this
2093                  * flag will force the updown routine to be called
2094                  * on the next ibstatuschange down interrupt, even
2095                  * if it's not an down->up transition.
2096                  */
2097                 val--; /* convert from IB to chip */
2098                 maskr = IBA7220_IBC_WIDTH_MASK;
2099                 lsb = IBA7220_IBC_WIDTH_SHIFT;
2100                 setforce = 1;
2101                 dd->ipath_flags |= IPATH_IB_FORCE_NOTIFY;
2102                 break;
2103
2104         case IPATH_IB_CFG_SPD_ENB: /* set allowed Link speeds */
2105                 /*
2106                  * If we turn off IB1.2, need to preset SerDes defaults,
2107                  * but not right now. Set a flag for the next time
2108                  * we command the link down.  As with width, only write the
2109                  * actual register if the link is currently down, otherwise
2110                  * takes effect on next link change.  Since setting is being
2111                  * explictly requested (via MAD or sysfs), clear autoneg
2112                  * failure status if speed autoneg is enabled.
2113                  */
2114                 dd->ipath_link_speed_enabled = val;
2115                 if (dd->ipath_ibcddrctrl & IBA7220_IBC_IBTA_1_2_MASK &&
2116                     !(val & (val - 1)))
2117                         dd->ipath_presets_needed = 1;
2118                 if ((dd->ipath_flags & (IPATH_LINKDOWN|IPATH_LINKINIT)) !=
2119                         IPATH_LINKDOWN)
2120                         goto bail;
2121                 /*
2122                  * We set the IPATH_IB_FORCE_NOTIFY bit so updown
2123                  * will get called because we want update
2124                  * link_speed_active, and the change may not take
2125                  * effect for some time (if we are in POLL), so this
2126                  * flag will force the updown routine to be called
2127                  * on the next ibstatuschange down interrupt, even
2128                  * if it's not an down->up transition.  When setting
2129                  * speed autoneg, clear AUTONEG_FAILED.
2130                  */
2131                 if (val == (IPATH_IB_SDR | IPATH_IB_DDR)) {
2132                         val = IBA7220_IBC_SPEED_AUTONEG_MASK |
2133                                 IBA7220_IBC_IBTA_1_2_MASK;
2134                         dd->ipath_flags &= ~IPATH_IB_AUTONEG_FAILED;
2135                 } else
2136                         val = val == IPATH_IB_DDR ?  IBA7220_IBC_SPEED_DDR
2137                                 : IBA7220_IBC_SPEED_SDR;
2138                 maskr = IBA7220_IBC_SPEED_AUTONEG_MASK |
2139                         IBA7220_IBC_IBTA_1_2_MASK;
2140                 lsb = 0; /* speed bits are low bits */
2141                 setforce = 1;
2142                 break;
2143
2144         case IPATH_IB_CFG_RXPOL_ENB: /* set Auto-RX-polarity enable */
2145                 lsb = IBA7220_IBC_RXPOL_SHIFT;
2146                 maskr = IBA7220_IBC_RXPOL_MASK;
2147                 break;
2148
2149         case IPATH_IB_CFG_LREV_ENB: /* set Auto-Lane-reversal enable */
2150                 lsb = IBA7220_IBC_LREV_SHIFT;
2151                 maskr = IBA7220_IBC_LREV_MASK;
2152                 break;
2153
2154         default:
2155                 ret = -ENOTSUPP;
2156                 goto bail;
2157         }
2158         dd->ipath_ibcddrctrl &= ~(maskr << lsb);
2159         dd->ipath_ibcddrctrl |= (((u64) val & maskr) << lsb);
2160         ipath_write_kreg(dd, dd->ipath_kregs->kr_ibcddrctrl,
2161                          dd->ipath_ibcddrctrl);
2162         if (setforce)
2163                 dd->ipath_flags |= IPATH_IB_FORCE_NOTIFY;
2164 bail:
2165         return ret;
2166 }
2167
2168 static void ipath_7220_read_counters(struct ipath_devdata *dd,
2169                                      struct infinipath_counters *cntrs)
2170 {
2171         u64 *counters = (u64 *) cntrs;
2172         int i;
2173
2174         for (i = 0; i < sizeof(*cntrs) / sizeof(u64); i++)
2175                 counters[i] = ipath_snap_cntr(dd, i);
2176 }
2177
2178 /* if we are using MSI, try to fallback to INTx */
2179 static int ipath_7220_intr_fallback(struct ipath_devdata *dd)
2180 {
2181         if (dd->ipath_msi_lo) {
2182                 dev_info(&dd->pcidev->dev, "MSI interrupt not detected,"
2183                         " trying INTx interrupts\n");
2184                 ipath_7220_nomsi(dd);
2185                 ipath_enable_intx(dd->pcidev);
2186                 /*
2187                  * some newer kernels require free_irq before disable_msi,
2188                  * and irq can be changed during disable and intx enable
2189                  * and we need to therefore use the pcidev->irq value,
2190                  * not our saved MSI value.
2191                  */
2192                 dd->ipath_irq = dd->pcidev->irq;
2193                 if (request_irq(dd->ipath_irq, ipath_intr, IRQF_SHARED,
2194                         IPATH_DRV_NAME, dd))
2195                         ipath_dev_err(dd,
2196                                 "Could not re-request_irq for INTx\n");
2197                 return 1;
2198         }
2199         return 0;
2200 }
2201
2202 /*
2203  * reset the XGXS (between serdes and IBC).  Slightly less intrusive
2204  * than resetting the IBC or external link state, and useful in some
2205  * cases to cause some retraining.  To do this right, we reset IBC
2206  * as well.
2207  */
2208 static void ipath_7220_xgxs_reset(struct ipath_devdata *dd)
2209 {
2210         u64 val, prev_val;
2211
2212         prev_val = ipath_read_kreg64(dd, dd->ipath_kregs->kr_xgxsconfig);
2213         val = prev_val | INFINIPATH_XGXS_RESET;
2214         prev_val &= ~INFINIPATH_XGXS_RESET; /* be sure */
2215         ipath_write_kreg(dd, dd->ipath_kregs->kr_control,
2216                          dd->ipath_control & ~INFINIPATH_C_LINKENABLE);
2217         ipath_write_kreg(dd, dd->ipath_kregs->kr_xgxsconfig, val);
2218         ipath_read_kreg32(dd, dd->ipath_kregs->kr_scratch);
2219         ipath_write_kreg(dd, dd->ipath_kregs->kr_xgxsconfig, prev_val);
2220         ipath_write_kreg(dd, dd->ipath_kregs->kr_control,
2221                          dd->ipath_control);
2222 }
2223
2224
2225 /* Still needs cleanup, too much hardwired stuff */
2226 static void autoneg_send(struct ipath_devdata *dd,
2227         u32 *hdr, u32 dcnt, u32 *data)
2228 {
2229         int i;
2230         u64 cnt;
2231         u32 __iomem *piobuf;
2232         u32 pnum;
2233
2234         i = 0;
2235         cnt = 7 + dcnt + 1; /* 7 dword header, dword data, icrc */
2236         while (!(piobuf = ipath_getpiobuf(dd, cnt, &pnum))) {
2237                 if (i++ > 15) {
2238                         ipath_dbg("Couldn't get pio buffer for send\n");
2239                         return;
2240                 }
2241                 udelay(2);
2242         }
2243         if (dd->ipath_flags&IPATH_HAS_PBC_CNT)
2244                 cnt |= 0x80000000UL<<32; /* mark as VL15 */
2245         writeq(cnt, piobuf);
2246         ipath_flush_wc();
2247         __iowrite32_copy(piobuf + 2, hdr, 7);
2248         __iowrite32_copy(piobuf + 9, data, dcnt);
2249         ipath_flush_wc();
2250 }
2251
2252 /*
2253  * _start packet gets sent twice at start, _done gets sent twice at end
2254  */
2255 static void ipath_autoneg_send(struct ipath_devdata *dd, int which)
2256 {
2257         static u32 swapped;
2258         u32 dw, i, hcnt, dcnt, *data;
2259         static u32 hdr[7] = { 0xf002ffff, 0x48ffff, 0x6400abba };
2260         static u32 madpayload_start[0x40] = {
2261                 0x1810103, 0x1, 0x0, 0x0, 0x2c90000, 0x2c9, 0x0, 0x0,
2262                 0xffffffff, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
2263                 0x1, 0x1388, 0x15e, 0x1, /* rest 0's */
2264                 };
2265         static u32 madpayload_done[0x40] = {
2266                 0x1810103, 0x1, 0x0, 0x0, 0x2c90000, 0x2c9, 0x0, 0x0,
2267                 0xffffffff, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
2268                 0x40000001, 0x1388, 0x15e, /* rest 0's */
2269                 };
2270         dcnt = ARRAY_SIZE(madpayload_start);
2271         hcnt = ARRAY_SIZE(hdr);
2272         if (!swapped) {
2273                 /* for maintainability, do it at runtime */
2274                 for (i = 0; i < hcnt; i++) {
2275                         dw = (__force u32) cpu_to_be32(hdr[i]);
2276                         hdr[i] = dw;
2277                 }
2278                 for (i = 0; i < dcnt; i++) {
2279                         dw = (__force u32) cpu_to_be32(madpayload_start[i]);
2280                         madpayload_start[i] = dw;
2281                         dw = (__force u32) cpu_to_be32(madpayload_done[i]);
2282                         madpayload_done[i] = dw;
2283                 }
2284                 swapped = 1;
2285         }
2286
2287         data = which ? madpayload_done : madpayload_start;
2288         ipath_cdbg(PKT, "Sending %s special MADs\n", which?"done":"start");
2289
2290         autoneg_send(dd, hdr, dcnt, data);
2291         ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch);
2292         udelay(2);
2293         autoneg_send(dd, hdr, dcnt, data);
2294         ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch);
2295         udelay(2);
2296 }
2297
2298
2299
2300 /*
2301  * Do the absolute minimum to cause an IB speed change, and make it
2302  * ready, but don't actually trigger the change.   The caller will
2303  * do that when ready (if link is in Polling training state, it will
2304  * happen immediately, otherwise when link next goes down)
2305  *
2306  * This routine should only be used as part of the DDR autonegotation
2307  * code for devices that are not compliant with IB 1.2 (or code that
2308  * fixes things up for same).
2309  *
2310  * When link has gone down, and autoneg enabled, or autoneg has
2311  * failed and we give up until next time we set both speeds, and
2312  * then we want IBTA enabled as well as "use max enabled speed.
2313  */
2314 static void set_speed_fast(struct ipath_devdata *dd, u32 speed)
2315 {
2316         dd->ipath_ibcddrctrl &= ~(IBA7220_IBC_SPEED_AUTONEG_MASK |
2317                 IBA7220_IBC_IBTA_1_2_MASK |
2318                 (IBA7220_IBC_WIDTH_MASK << IBA7220_IBC_WIDTH_SHIFT));
2319
2320         if (speed == (IPATH_IB_SDR | IPATH_IB_DDR))
2321                 dd->ipath_ibcddrctrl |= IBA7220_IBC_SPEED_AUTONEG_MASK |
2322                         IBA7220_IBC_IBTA_1_2_MASK;
2323         else
2324                 dd->ipath_ibcddrctrl |= speed == IPATH_IB_DDR ?
2325                         IBA7220_IBC_SPEED_DDR : IBA7220_IBC_SPEED_SDR;
2326
2327         /*
2328          * Convert from IB-style 1 = 1x, 2 = 4x, 3 = auto
2329          * to chip-centric       0 = 1x, 1 = 4x, 2 = auto
2330          */
2331         dd->ipath_ibcddrctrl |= (u64)(dd->ipath_link_width_enabled - 1) <<
2332                 IBA7220_IBC_WIDTH_SHIFT;
2333         ipath_write_kreg(dd, dd->ipath_kregs->kr_ibcddrctrl,
2334                         dd->ipath_ibcddrctrl);
2335         ipath_cdbg(VERBOSE, "setup for IB speed (%x) done\n", speed);
2336 }
2337
2338
2339 /*
2340  * this routine is only used when we are not talking to another
2341  * IB 1.2-compliant device that we think can do DDR.
2342  * (This includes all existing switch chips as of Oct 2007.)
2343  * 1.2-compliant devices go directly to DDR prior to reaching INIT
2344  */
2345 static void try_auto_neg(struct ipath_devdata *dd)
2346 {
2347         /*
2348          * required for older non-IB1.2 DDR switches.  Newer
2349          * non-IB-compliant switches don't need it, but so far,
2350          * aren't bothered by it either.  "Magic constant"
2351          */
2352         ipath_write_kreg(dd, IPATH_KREG_OFFSET(IBNCModeCtrl),
2353                 0x3b9dc07);
2354         dd->ipath_flags |= IPATH_IB_AUTONEG_INPROG;
2355         ipath_autoneg_send(dd, 0);
2356         set_speed_fast(dd, IPATH_IB_DDR);
2357         ipath_toggle_rclkrls(dd);
2358         /* 2 msec is minimum length of a poll cycle */
2359         schedule_delayed_work(&dd->ipath_autoneg_work,
2360                 msecs_to_jiffies(2));
2361 }
2362
2363
2364 static int ipath_7220_ib_updown(struct ipath_devdata *dd, int ibup, u64 ibcs)
2365 {
2366         int ret = 0, symadj = 0;
2367         u32 ltstate = ipath_ib_linkstate(dd, ibcs);
2368
2369         dd->ipath_link_width_active =
2370                 ((ibcs >> IBA7220_IBCS_LINKWIDTH_SHIFT) & 1) ?
2371                     IB_WIDTH_4X : IB_WIDTH_1X;
2372         dd->ipath_link_speed_active =
2373                 ((ibcs >> IBA7220_IBCS_LINKSPEED_SHIFT) & 1) ?
2374                     IPATH_IB_DDR : IPATH_IB_SDR;
2375
2376         if (!ibup) {
2377                 /*
2378                  * when link goes down we don't want aeq running, so it
2379                  * won't't interfere with IBC training, etc., and we need
2380                  * to go back to the static SerDes preset values
2381                  */
2382                 if (dd->ipath_x1_fix_tries &&
2383                          ltstate <= INFINIPATH_IBCS_LT_STATE_SLEEPQUIET &&
2384                         ltstate != INFINIPATH_IBCS_LT_STATE_LINKUP)
2385                         dd->ipath_x1_fix_tries = 0;
2386                 if (!(dd->ipath_flags & (IPATH_IB_AUTONEG_FAILED |
2387                         IPATH_IB_AUTONEG_INPROG)))
2388                         set_speed_fast(dd, dd->ipath_link_speed_enabled);
2389                 if (!(dd->ipath_flags & IPATH_IB_AUTONEG_INPROG)) {
2390                         ipath_cdbg(VERBOSE, "Setting RXEQ defaults\n");
2391                         ipath_sd7220_presets(dd);
2392                 }
2393                 /* this might better in ipath_sd7220_presets() */
2394                 ipath_set_relock_poll(dd, ibup);
2395         } else {
2396                 if (ipath_compat_ddr_negotiate &&
2397                     !(dd->ipath_flags & (IPATH_IB_AUTONEG_FAILED |
2398                         IPATH_IB_AUTONEG_INPROG)) &&
2399                         dd->ipath_link_speed_active == IPATH_IB_SDR &&
2400                         (dd->ipath_link_speed_enabled &
2401                             (IPATH_IB_DDR | IPATH_IB_SDR)) ==
2402                             (IPATH_IB_DDR | IPATH_IB_SDR) &&
2403                         dd->ipath_autoneg_tries < IPATH_AUTONEG_TRIES) {
2404                         /* we are SDR, and DDR auto-negotiation enabled */
2405                         ++dd->ipath_autoneg_tries;
2406                         ipath_dbg("DDR negotiation try, %u/%u\n",
2407                                 dd->ipath_autoneg_tries,
2408                                 IPATH_AUTONEG_TRIES);
2409                         if (!dd->ibdeltainprog) {
2410                                 dd->ibdeltainprog = 1;
2411                                 dd->ibsymsnap = ipath_read_creg32(dd,
2412                                         dd->ipath_cregs->cr_ibsymbolerrcnt);
2413                                 dd->iblnkerrsnap = ipath_read_creg32(dd,
2414                                         dd->ipath_cregs->cr_iblinkerrrecovcnt);
2415                         }
2416                         try_auto_neg(dd);
2417                         ret = 1; /* no other IB status change processing */
2418                 } else if ((dd->ipath_flags & IPATH_IB_AUTONEG_INPROG)
2419                         && dd->ipath_link_speed_active == IPATH_IB_SDR) {
2420                         ipath_autoneg_send(dd, 1);
2421                         set_speed_fast(dd, IPATH_IB_DDR);
2422                         udelay(2);
2423                         ipath_toggle_rclkrls(dd);
2424                         ret = 1; /* no other IB status change processing */
2425                 } else {
2426                         if ((dd->ipath_flags & IPATH_IB_AUTONEG_INPROG) &&
2427                                 (dd->ipath_link_speed_active & IPATH_IB_DDR)) {
2428                                 ipath_dbg("Got to INIT with DDR autoneg\n");
2429                                 dd->ipath_flags &= ~(IPATH_IB_AUTONEG_INPROG
2430                                         | IPATH_IB_AUTONEG_FAILED);
2431                                 dd->ipath_autoneg_tries = 0;
2432                                 /* re-enable SDR, for next link down */
2433                                 set_speed_fast(dd,
2434                                         dd->ipath_link_speed_enabled);
2435                                 wake_up(&dd->ipath_autoneg_wait);
2436                                 symadj = 1;
2437                         } else if (dd->ipath_flags & IPATH_IB_AUTONEG_FAILED) {
2438                                 /*
2439                                  * clear autoneg failure flag, and do setup
2440                                  * so we'll try next time link goes down and
2441                                  * back to INIT (possibly connected to different
2442                                  * device).
2443                                  */
2444                                 ipath_dbg("INIT %sDR after autoneg failure\n",
2445                                         (dd->ipath_link_speed_active &
2446                                           IPATH_IB_DDR) ? "D" : "S");
2447                                 dd->ipath_flags &= ~IPATH_IB_AUTONEG_FAILED;
2448                                 dd->ipath_ibcddrctrl |=
2449                                         IBA7220_IBC_IBTA_1_2_MASK;
2450                                 ipath_write_kreg(dd,
2451                                         IPATH_KREG_OFFSET(IBNCModeCtrl), 0);
2452                                 symadj = 1;
2453                         }
2454                 }
2455                 /*
2456                  * if we are in 1X on rev1 only, and are in autoneg width,
2457                  * it could be due to an xgxs problem, so if we haven't
2458                  * already tried, try twice to get to 4X; if we
2459                  * tried, and couldn't, report it, since it will
2460                  * probably not be what is desired.
2461                  */
2462                 if (dd->ipath_minrev == 1 &&
2463                     (dd->ipath_link_width_enabled & (IB_WIDTH_1X |
2464                         IB_WIDTH_4X)) == (IB_WIDTH_1X | IB_WIDTH_4X)
2465                         && dd->ipath_link_width_active == IB_WIDTH_1X
2466                         && dd->ipath_x1_fix_tries < 3) {
2467                      if (++dd->ipath_x1_fix_tries == 3) {
2468                                 dev_info(&dd->pcidev->dev,
2469                                         "IB link is in 1X mode\n");
2470                                 if (!(dd->ipath_flags &
2471                                       IPATH_IB_AUTONEG_INPROG))
2472                                         symadj = 1;
2473                      }
2474                         else {
2475                                 ipath_cdbg(VERBOSE, "IB 1X in "
2476                                         "auto-width, try %u to be "
2477                                         "sure it's really 1X; "
2478                                         "ltstate %u\n",
2479                                          dd->ipath_x1_fix_tries,
2480                                          ltstate);
2481                                 dd->ipath_f_xgxs_reset(dd);
2482                                 ret = 1; /* skip other processing */
2483                         }
2484                 } else if (!(dd->ipath_flags & IPATH_IB_AUTONEG_INPROG))
2485                         symadj = 1;
2486
2487                 if (!ret) {
2488                         dd->delay_mult = rate_to_delay
2489                             [(ibcs >> IBA7220_IBCS_LINKSPEED_SHIFT) & 1]
2490                             [(ibcs >> IBA7220_IBCS_LINKWIDTH_SHIFT) & 1];
2491
2492                         ipath_set_relock_poll(dd, ibup);
2493                 }
2494         }
2495
2496         if (symadj) {
2497                 if (dd->ibdeltainprog) {
2498                         dd->ibdeltainprog = 0;
2499                         dd->ibsymdelta += ipath_read_creg32(dd,
2500                                 dd->ipath_cregs->cr_ibsymbolerrcnt) -
2501                                 dd->ibsymsnap;
2502                         dd->iblnkerrdelta += ipath_read_creg32(dd,
2503                                 dd->ipath_cregs->cr_iblinkerrrecovcnt) -
2504                                 dd->iblnkerrsnap;
2505                 }
2506         } else if (!ibup && !dd->ibdeltainprog
2507                    && !(dd->ipath_flags & IPATH_IB_AUTONEG_INPROG)) {
2508                 dd->ibdeltainprog = 1;
2509                 dd->ibsymsnap = ipath_read_creg32(dd,
2510                                      dd->ipath_cregs->cr_ibsymbolerrcnt);
2511                 dd->iblnkerrsnap = ipath_read_creg32(dd,
2512                                      dd->ipath_cregs->cr_iblinkerrrecovcnt);
2513         }
2514
2515         if (!ret)
2516                 ipath_setup_7220_setextled(dd, ipath_ib_linkstate(dd, ibcs),
2517                         ltstate);
2518         return ret;
2519 }
2520
2521
2522 /*
2523  * Handle the empirically determined mechanism for auto-negotiation
2524  * of DDR speed with switches.
2525  */
2526 static void autoneg_work(struct work_struct *work)
2527 {
2528         struct ipath_devdata *dd;
2529         u64 startms;
2530         u32 lastlts, i;
2531
2532         dd = container_of(work, struct ipath_devdata,
2533                 ipath_autoneg_work.work);
2534
2535         startms = jiffies_to_msecs(jiffies);
2536
2537         /*
2538          * busy wait for this first part, it should be at most a
2539          * few hundred usec, since we scheduled ourselves for 2msec.
2540          */
2541         for (i = 0; i < 25; i++) {
2542                 lastlts = ipath_ib_linktrstate(dd, dd->ipath_lastibcstat);
2543                 if (lastlts == INFINIPATH_IBCS_LT_STATE_POLLQUIET) {
2544                         ipath_set_linkstate(dd, IPATH_IB_LINKDOWN_DISABLE);
2545                         break;
2546                 }
2547                 udelay(100);
2548         }
2549
2550         if (!(dd->ipath_flags & IPATH_IB_AUTONEG_INPROG))
2551                 goto done; /* we got there early or told to stop */
2552
2553         /* we expect this to timeout */
2554         if (wait_event_timeout(dd->ipath_autoneg_wait,
2555                 !(dd->ipath_flags & IPATH_IB_AUTONEG_INPROG),
2556                 msecs_to_jiffies(90)))
2557                 goto done;
2558
2559         ipath_toggle_rclkrls(dd);
2560
2561         /* we expect this to timeout */
2562         if (wait_event_timeout(dd->ipath_autoneg_wait,
2563                 !(dd->ipath_flags & IPATH_IB_AUTONEG_INPROG),
2564                 msecs_to_jiffies(1700)))
2565                 goto done;
2566
2567         set_speed_fast(dd, IPATH_IB_SDR);
2568         ipath_toggle_rclkrls(dd);
2569
2570         /*
2571          * wait up to 250 msec for link to train and get to INIT at DDR;
2572          * this should terminate early
2573          */
2574         wait_event_timeout(dd->ipath_autoneg_wait,
2575                 !(dd->ipath_flags & IPATH_IB_AUTONEG_INPROG),
2576                 msecs_to_jiffies(250));
2577 done:
2578         if (dd->ipath_flags & IPATH_IB_AUTONEG_INPROG) {
2579                 ipath_dbg("Did not get to DDR INIT (%x) after %Lu msecs\n",
2580                         ipath_ib_state(dd, dd->ipath_lastibcstat),
2581                         (unsigned long long) jiffies_to_msecs(jiffies)-startms);
2582                 dd->ipath_flags &= ~IPATH_IB_AUTONEG_INPROG;
2583                 if (dd->ipath_autoneg_tries == IPATH_AUTONEG_TRIES) {
2584                         dd->ipath_flags |= IPATH_IB_AUTONEG_FAILED;
2585                         ipath_dbg("Giving up on DDR until next IB "
2586                                 "link Down\n");
2587                         dd->ipath_autoneg_tries = 0;
2588                 }
2589                 set_speed_fast(dd, dd->ipath_link_speed_enabled);
2590         }
2591 }
2592
2593
2594 /**
2595  * ipath_init_iba7220_funcs - set up the chip-specific function pointers
2596  * @dd: the infinipath device
2597  *
2598  * This is global, and is called directly at init to set up the
2599  * chip-specific function pointers for later use.
2600  */
2601 void ipath_init_iba7220_funcs(struct ipath_devdata *dd)
2602 {
2603         dd->ipath_f_intrsetup = ipath_7220_intconfig;
2604         dd->ipath_f_bus = ipath_setup_7220_config;
2605         dd->ipath_f_reset = ipath_setup_7220_reset;
2606         dd->ipath_f_get_boardname = ipath_7220_boardname;
2607         dd->ipath_f_init_hwerrors = ipath_7220_init_hwerrors;
2608         dd->ipath_f_early_init = ipath_7220_early_init;
2609         dd->ipath_f_handle_hwerrors = ipath_7220_handle_hwerrors;
2610         dd->ipath_f_quiet_serdes = ipath_7220_quiet_serdes;
2611         dd->ipath_f_bringup_serdes = ipath_7220_bringup_serdes;
2612         dd->ipath_f_clear_tids = ipath_7220_clear_tids;
2613         dd->ipath_f_put_tid = ipath_7220_put_tid;
2614         dd->ipath_f_cleanup = ipath_setup_7220_cleanup;
2615         dd->ipath_f_setextled = ipath_setup_7220_setextled;
2616         dd->ipath_f_get_base_info = ipath_7220_get_base_info;
2617         dd->ipath_f_free_irq = ipath_7220_free_irq;
2618         dd->ipath_f_tidtemplate = ipath_7220_tidtemplate;
2619         dd->ipath_f_intr_fallback = ipath_7220_intr_fallback;
2620         dd->ipath_f_xgxs_reset = ipath_7220_xgxs_reset;
2621         dd->ipath_f_get_ib_cfg = ipath_7220_get_ib_cfg;
2622         dd->ipath_f_set_ib_cfg = ipath_7220_set_ib_cfg;
2623         dd->ipath_f_config_jint = ipath_7220_config_jint;
2624         dd->ipath_f_config_ports = ipath_7220_config_ports;
2625         dd->ipath_f_read_counters = ipath_7220_read_counters;
2626         dd->ipath_f_get_msgheader = ipath_7220_get_msgheader;
2627         dd->ipath_f_ib_updown = ipath_7220_ib_updown;
2628
2629         /* initialize chip-specific variables */
2630         ipath_init_7220_variables(dd);
2631 }