drivers/net/tg3.c

   1 /*
   2  * tg3.c: Broadcom Tigon3 ethernet driver.
   3  *
   4  * Copyright (C) 2001, 2002, 2003, 2004 David S. Miller (davem@redhat.com)
   5  * Copyright (C) 2001, 2002, 2003 Jeff Garzik (jgarzik@pobox.com)
   6  * Copyright (C) 2004 Sun Microsystems Inc.
   7  * Copyright (C) 2005-2011 Broadcom Corporation.
   8  *
   9  * Firmware is:
  10  *      Derived from proprietary unpublished source code,
  11  *      Copyright (C) 2000-2003 Broadcom Corporation.
  12  *
  13  *      Permission is hereby granted for the distribution of this firmware
  14  *      data in hexadecimal or equivalent format, provided this copyright
  15  *      notice is accompanying it.
  16  */
  17
  18
  19 #include <linux/module.h>
  20 #include <linux/moduleparam.h>
  21 #include <linux/stringify.h>
  22 #include <linux/kernel.h>
  23 #include <linux/types.h>
  24 #include <linux/compiler.h>
  25 #include <linux/slab.h>
  26 #include <linux/delay.h>
  27 #include <linux/in.h>
  28 #include <linux/init.h>
  29 #include <linux/ioport.h>
  30 #include <linux/pci.h>
  31 #include <linux/netdevice.h>
  32 #include <linux/etherdevice.h>
  33 #include <linux/skbuff.h>
  34 #include <linux/ethtool.h>
  35 #include <linux/mdio.h>
  36 #include <linux/mii.h>
  37 #include <linux/phy.h>
  38 #include <linux/brcmphy.h>
  39 #include <linux/if_vlan.h>
  40 #include <linux/ip.h>
  41 #include <linux/tcp.h>
  42 #include <linux/workqueue.h>
  43 #include <linux/prefetch.h>
  44 #include <linux/dma-mapping.h>
  45 #include <linux/firmware.h>
  46
  47 #include <net/checksum.h>
  48 #include <net/ip.h>
  49
  50 #include <asm/system.h>
  51 #include <linux/io.h>
  52 #include <asm/byteorder.h>
  53 #include <linux/uaccess.h>
  54
  55 #ifdef CONFIG_SPARC
  56 #include <asm/idprom.h>
  57 #include <asm/prom.h>
  58 #endif
  59
  60 #define BAR_0   0
  61 #define BAR_2   2
  62
  63 #include "tg3.h"
  64
  65 /* Functions & macros to verify TG3_FLAGS types */
  66
  67 static inline int _tg3_flag(enum TG3_FLAGS flag, unsigned long *bits)
  68 {
  69         return test_bit(flag, bits);
  70 }
  71
  72 static inline void _tg3_flag_set(enum TG3_FLAGS flag, unsigned long *bits)
  73 {
  74         set_bit(flag, bits);
  75 }
  76
  77 static inline void _tg3_flag_clear(enum TG3_FLAGS flag, unsigned long *bits)
  78 {
  79         clear_bit(flag, bits);
  80 }
  81
  82 #define tg3_flag(tp, flag)                              \
  83         _tg3_flag(TG3_FLAG_##flag, (tp)->tg3_flags)
  84 #define tg3_flag_set(tp, flag)                          \
  85         _tg3_flag_set(TG3_FLAG_##flag, (tp)->tg3_flags)
  86 #define tg3_flag_clear(tp, flag)                        \
  87         _tg3_flag_clear(TG3_FLAG_##flag, (tp)->tg3_flags)
  88
  89 #define DRV_MODULE_NAME         "tg3"
  90 #define TG3_MAJ_NUM                     3
  91 #define TG3_MIN_NUM                     118
  92 #define DRV_MODULE_VERSION      \
  93         __stringify(TG3_MAJ_NUM) "." __stringify(TG3_MIN_NUM)
  94 #define DRV_MODULE_RELDATE      "April 22, 2011"
  95
  96 #define TG3_DEF_MAC_MODE        0
  97 #define TG3_DEF_RX_MODE         0
  98 #define TG3_DEF_TX_MODE         0
  99 #define TG3_DEF_MSG_ENABLE        \
 100         (NETIF_MSG_DRV          | \
 101          NETIF_MSG_PROBE        | \
 102          NETIF_MSG_LINK         | \
 103          NETIF_MSG_TIMER        | \
 104          NETIF_MSG_IFDOWN       | \
 105          NETIF_MSG_IFUP         | \
 106          NETIF_MSG_RX_ERR       | \
 107          NETIF_MSG_TX_ERR)
 108
 109 /* length of time before we decide the hardware is borked,
 110  * and dev->tx_timeout() should be called to fix the problem
 111  */
 112
 113 #define TG3_TX_TIMEOUT                  (5 * HZ)
 114
 115 /* hardware minimum and maximum for a single frame's data payload */
 116 #define TG3_MIN_MTU                     60
 117 #define TG3_MAX_MTU(tp) \
 118         (tg3_flag(tp, JUMBO_CAPABLE) ? 9000 : 1500)
 119
 120 /* These numbers seem to be hard coded in the NIC firmware somehow.
 121  * You can't change the ring sizes, but you can change where you place
 122  * them in the NIC onboard memory.
 123  */
 124 #define TG3_RX_STD_RING_SIZE(tp) \
 125         (tg3_flag(tp, LRG_PROD_RING_CAP) ? \
 126          TG3_RX_STD_MAX_SIZE_5717 : TG3_RX_STD_MAX_SIZE_5700)
 127 #define TG3_DEF_RX_RING_PENDING         200
 128 #define TG3_RX_JMB_RING_SIZE(tp) \
 129         (tg3_flag(tp, LRG_PROD_RING_CAP) ? \
 130          TG3_RX_JMB_MAX_SIZE_5717 : TG3_RX_JMB_MAX_SIZE_5700)
 131 #define TG3_DEF_RX_JUMBO_RING_PENDING   100
 132 #define TG3_RSS_INDIR_TBL_SIZE          128
 133
 134 /* Do not place this n-ring entries value into the tp struct itself,
 135  * we really want to expose these constants to GCC so that modulo et
 136  * al.  operations are done with shifts and masks instead of with
 137  * hw multiply/modulo instructions.  Another solution would be to
 138  * replace things like '% foo' with '& (foo - 1)'.
 139  */
 140
 141 #define TG3_TX_RING_SIZE                512
 142 #define TG3_DEF_TX_RING_PENDING         (TG3_TX_RING_SIZE - 1)
 143
 144 #define TG3_RX_STD_RING_BYTES(tp) \
 145         (sizeof(struct tg3_rx_buffer_desc) * TG3_RX_STD_RING_SIZE(tp))
 146 #define TG3_RX_JMB_RING_BYTES(tp) \
 147         (sizeof(struct tg3_ext_rx_buffer_desc) * TG3_RX_JMB_RING_SIZE(tp))
 148 #define TG3_RX_RCB_RING_BYTES(tp) \
 149         (sizeof(struct tg3_rx_buffer_desc) * (tp->rx_ret_ring_mask + 1))
 150 #define TG3_TX_RING_BYTES       (sizeof(struct tg3_tx_buffer_desc) * \
 151                                  TG3_TX_RING_SIZE)
 152 #define NEXT_TX(N)              (((N) + 1) & (TG3_TX_RING_SIZE - 1))
 153
 154 #define TG3_DMA_BYTE_ENAB               64
 155
 156 #define TG3_RX_STD_DMA_SZ               1536
 157 #define TG3_RX_JMB_DMA_SZ               9046
 158
 159 #define TG3_RX_DMA_TO_MAP_SZ(x)         ((x) + TG3_DMA_BYTE_ENAB)
 160
 161 #define TG3_RX_STD_MAP_SZ               TG3_RX_DMA_TO_MAP_SZ(TG3_RX_STD_DMA_SZ)
 162 #define TG3_RX_JMB_MAP_SZ               TG3_RX_DMA_TO_MAP_SZ(TG3_RX_JMB_DMA_SZ)
 163
 164 #define TG3_RX_STD_BUFF_RING_SIZE(tp) \
 165         (sizeof(struct ring_info) * TG3_RX_STD_RING_SIZE(tp))
 166
 167 #define TG3_RX_JMB_BUFF_RING_SIZE(tp) \
 168         (sizeof(struct ring_info) * TG3_RX_JMB_RING_SIZE(tp))
 169
 170 /* Due to a hardware bug, the 5701 can only DMA to memory addresses
 171  * that are at least dword aligned when used in PCIX mode.  The driver
 172  * works around this bug by double copying the packet.  This workaround
 173  * is built into the normal double copy length check for efficiency.
 174  *
 175  * However, the double copy is only necessary on those architectures
 176  * where unaligned memory accesses are inefficient.  For those architectures
 177  * where unaligned memory accesses incur little penalty, we can reintegrate
 178  * the 5701 in the normal rx path.  Doing so saves a device structure
 179  * dereference by hardcoding the double copy threshold in place.
 180  */
 181 #define TG3_RX_COPY_THRESHOLD           256
 182 #if NET_IP_ALIGN == 0 || defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)
 183         #define TG3_RX_COPY_THRESH(tp)  TG3_RX_COPY_THRESHOLD
 184 #else
 185         #define TG3_RX_COPY_THRESH(tp)  ((tp)->rx_copy_thresh)
 186 #endif
 187
 188 /* minimum number of free TX descriptors required to wake up TX process */
 189 #define TG3_TX_WAKEUP_THRESH(tnapi)             ((tnapi)->tx_pending / 4)
 190
 191 #define TG3_RAW_IP_ALIGN 2
 192
 193 #define TG3_FW_UPDATE_TIMEOUT_SEC       5
 194
 195 #define FIRMWARE_TG3            "tigon/tg3.bin"
 196 #define FIRMWARE_TG3TSO         "tigon/tg3_tso.bin"
 197 #define FIRMWARE_TG3TSO5        "tigon/tg3_tso5.bin"
 198
 199 static char version[] __devinitdata =
 200         DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")";
 201
 202 MODULE_AUTHOR("David S. Miller (davem@redhat.com) and Jeff Garzik (jgarzik@pobox.com)");
 203 MODULE_DESCRIPTION("Broadcom Tigon3 ethernet driver");
 204 MODULE_LICENSE("GPL");
 205 MODULE_VERSION(DRV_MODULE_VERSION);
 206 MODULE_FIRMWARE(FIRMWARE_TG3);
 207 MODULE_FIRMWARE(FIRMWARE_TG3TSO);
 208 MODULE_FIRMWARE(FIRMWARE_TG3TSO5);
 209
 210 static int tg3_debug = -1;      /* -1 == use TG3_DEF_MSG_ENABLE as value */
 211 module_param(tg3_debug, int, 0);
 212 MODULE_PARM_DESC(tg3_debug, "Tigon3 bitmapped debugging message enable value");
 213
 214 static DEFINE_PCI_DEVICE_TABLE(tg3_pci_tbl) = {
 215         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5700)},
 216         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5701)},
 217         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702)},
 218         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5703)},
 219         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5704)},
 220         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702FE)},
 221         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705)},
 222         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705_2)},
 223         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705M)},
 224         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705M_2)},
 225         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702X)},
 226         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5703X)},
 227         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5704S)},
 228         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702A3)},
 229         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5703A3)},
 230         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5782)},
 231         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5788)},
 232         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5789)},
 233         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5901)},
 234         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5901_2)},
 235         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5704S_2)},
 236         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705F)},
 237         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5721)},
 238         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5722)},
 239         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5751)},
 240         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5751M)},
 241         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5751F)},
 242         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5752)},
 243         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5752M)},
 244         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5753)},
 245         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5753M)},
 246         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5753F)},
 247         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5754)},
 248         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5754M)},
 249         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5755)},
 250         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5755M)},
 251         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5756)},
 252         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5786)},
 253         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5787)},
 254         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5787M)},
 255         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5787F)},
 256         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5714)},
 257         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5714S)},
 258         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5715)},
 259         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5715S)},
 260         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5780)},
 261         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5780S)},
 262         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5781)},
 263         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5906)},
 264         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5906M)},
 265         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5784)},
 266         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5764)},
 267         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5723)},
 268         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5761)},
 269         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5761E)},
 270         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5761S)},
 271         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5761SE)},
 272         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5785_G)},
 273         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5785_F)},
 274         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57780)},
 275         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57760)},
 276         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57790)},
 277         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57788)},
 278         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5717)},
 279         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5718)},
 280         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57781)},
 281         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57785)},
 282         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57761)},
 283         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57765)},
 284         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57791)},
 285         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57795)},
 286         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5719)},
 287         {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5720)},
 288         {PCI_DEVICE(PCI_VENDOR_ID_SYSKONNECT, PCI_DEVICE_ID_SYSKONNECT_9DXX)},
 289         {PCI_DEVICE(PCI_VENDOR_ID_SYSKONNECT, PCI_DEVICE_ID_SYSKONNECT_9MXX)},
 290         {PCI_DEVICE(PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC1000)},
 291         {PCI_DEVICE(PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC1001)},
 292         {PCI_DEVICE(PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC1003)},
 293         {PCI_DEVICE(PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC9100)},
 294         {PCI_DEVICE(PCI_VENDOR_ID_APPLE, PCI_DEVICE_ID_APPLE_TIGON3)},
 295         {}
 296 };
 297
 298 MODULE_DEVICE_TABLE(pci, tg3_pci_tbl);
 299
 300 static const struct {
 301         const char string[ETH_GSTRING_LEN];
 302 } ethtool_stats_keys[] = {
 303         { "rx_octets" },
 304         { "rx_fragments" },
 305         { "rx_ucast_packets" },
 306         { "rx_mcast_packets" },
 307         { "rx_bcast_packets" },
 308         { "rx_fcs_errors" },
 309         { "rx_align_errors" },
 310         { "rx_xon_pause_rcvd" },
 311         { "rx_xoff_pause_rcvd" },
 312         { "rx_mac_ctrl_rcvd" },
 313         { "rx_xoff_entered" },
 314         { "rx_frame_too_long_errors" },
 315         { "rx_jabbers" },
 316         { "rx_undersize_packets" },
 317         { "rx_in_length_errors" },
 318         { "rx_out_length_errors" },
 319         { "rx_64_or_less_octet_packets" },
 320         { "rx_65_to_127_octet_packets" },
 321         { "rx_128_to_255_octet_packets" },
 322         { "rx_256_to_511_octet_packets" },
 323         { "rx_512_to_1023_octet_packets" },
 324         { "rx_1024_to_1522_octet_packets" },
 325         { "rx_1523_to_2047_octet_packets" },
 326         { "rx_2048_to_4095_octet_packets" },
 327         { "rx_4096_to_8191_octet_packets" },
 328         { "rx_8192_to_9022_octet_packets" },
 329
 330         { "tx_octets" },
 331         { "tx_collisions" },
 332
 333         { "tx_xon_sent" },
 334         { "tx_xoff_sent" },
 335         { "tx_flow_control" },
 336         { "tx_mac_errors" },
 337         { "tx_single_collisions" },
 338         { "tx_mult_collisions" },
 339         { "tx_deferred" },
 340         { "tx_excessive_collisions" },
 341         { "tx_late_collisions" },
 342         { "tx_collide_2times" },
 343         { "tx_collide_3times" },
 344         { "tx_collide_4times" },
 345         { "tx_collide_5times" },
 346         { "tx_collide_6times" },
 347         { "tx_collide_7times" },
 348         { "tx_collide_8times" },
 349         { "tx_collide_9times" },
 350         { "tx_collide_10times" },
 351         { "tx_collide_11times" },
 352         { "tx_collide_12times" },
 353         { "tx_collide_13times" },
 354         { "tx_collide_14times" },
 355         { "tx_collide_15times" },
 356         { "tx_ucast_packets" },
 357         { "tx_mcast_packets" },
 358         { "tx_bcast_packets" },
 359         { "tx_carrier_sense_errors" },
 360         { "tx_discards" },
 361         { "tx_errors" },
 362
 363         { "dma_writeq_full" },
 364         { "dma_write_prioq_full" },
 365         { "rxbds_empty" },
 366         { "rx_discards" },
 367         { "mbuf_lwm_thresh_hit" },
 368         { "rx_errors" },
 369         { "rx_threshold_hit" },
 370
 371         { "dma_readq_full" },
 372         { "dma_read_prioq_full" },
 373         { "tx_comp_queue_full" },
 374
 375         { "ring_set_send_prod_index" },
 376         { "ring_status_update" },
 377         { "nic_irqs" },
 378         { "nic_avoided_irqs" },
 379         { "nic_tx_threshold_hit" }
 380 };
 381
 382 #define TG3_NUM_STATS   ARRAY_SIZE(ethtool_stats_keys)
 383
 384
 385 static const struct {
 386         const char string[ETH_GSTRING_LEN];
 387 } ethtool_test_keys[] = {
 388         { "nvram test     (online) " },
 389         { "link test      (online) " },
 390         { "register test  (offline)" },
 391         { "memory test    (offline)" },
 392         { "loopback test  (offline)" },
 393         { "interrupt test (offline)" },
 394 };
 395
 396 #define TG3_NUM_TEST    ARRAY_SIZE(ethtool_test_keys)
 397
 398
 399 static void tg3_write32(struct tg3 *tp, u32 off, u32 val)
 400 {
 401         writel(val, tp->regs + off);
 402 }
 403
 404 static u32 tg3_read32(struct tg3 *tp, u32 off)
 405 {
 406         return readl(tp->regs + off);
 407 }
 408
 409 static void tg3_ape_write32(struct tg3 *tp, u32 off, u32 val)
 410 {
 411         writel(val, tp->aperegs + off);
 412 }
 413
 414 static u32 tg3_ape_read32(struct tg3 *tp, u32 off)
 415 {
 416         return readl(tp->aperegs + off);
 417 }
 418
 419 static void tg3_write_indirect_reg32(struct tg3 *tp, u32 off, u32 val)
 420 {
 421         unsigned long flags;
 422
 423         spin_lock_irqsave(&tp->indirect_lock, flags);
 424         pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off);
 425         pci_write_config_dword(tp->pdev, TG3PCI_REG_DATA, val);
 426         spin_unlock_irqrestore(&tp->indirect_lock, flags);
 427 }
 428
 429 static void tg3_write_flush_reg32(struct tg3 *tp, u32 off, u32 val)
 430 {
 431         writel(val, tp->regs + off);
 432         readl(tp->regs + off);
 433 }
 434
 435 static u32 tg3_read_indirect_reg32(struct tg3 *tp, u32 off)
 436 {
 437         unsigned long flags;
 438         u32 val;
 439
 440         spin_lock_irqsave(&tp->indirect_lock, flags);
 441         pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off);
 442         pci_read_config_dword(tp->pdev, TG3PCI_REG_DATA, &val);
 443         spin_unlock_irqrestore(&tp->indirect_lock, flags);
 444         return val;
 445 }
 446
 447 static void tg3_write_indirect_mbox(struct tg3 *tp, u32 off, u32 val)
 448 {
 449         unsigned long flags;
 450
 451         if (off == (MAILBOX_RCVRET_CON_IDX_0 + TG3_64BIT_REG_LOW)) {
 452                 pci_write_config_dword(tp->pdev, TG3PCI_RCV_RET_RING_CON_IDX +
 453                                        TG3_64BIT_REG_LOW, val);
 454                 return;
 455         }
 456         if (off == TG3_RX_STD_PROD_IDX_REG) {
 457                 pci_write_config_dword(tp->pdev, TG3PCI_STD_RING_PROD_IDX +
 458                                        TG3_64BIT_REG_LOW, val);
 459                 return;
 460         }
 461
 462         spin_lock_irqsave(&tp->indirect_lock, flags);
 463         pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off + 0x5600);
 464         pci_write_config_dword(tp->pdev, TG3PCI_REG_DATA, val);
 465         spin_unlock_irqrestore(&tp->indirect_lock, flags);
 466
 467         /* In indirect mode when disabling interrupts, we also need
 468          * to clear the interrupt bit in the GRC local ctrl register.
 469          */
 470         if ((off == (MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW)) &&
 471             (val == 0x1)) {
 472                 pci_write_config_dword(tp->pdev, TG3PCI_MISC_LOCAL_CTRL,
 473                                        tp->grc_local_ctrl|GRC_LCLCTRL_CLEARINT);
 474         }
 475 }
 476
 477 static u32 tg3_read_indirect_mbox(struct tg3 *tp, u32 off)
 478 {
 479         unsigned long flags;
 480         u32 val;
 481
 482         spin_lock_irqsave(&tp->indirect_lock, flags);
 483         pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off + 0x5600);
 484         pci_read_config_dword(tp->pdev, TG3PCI_REG_DATA, &val);
 485         spin_unlock_irqrestore(&tp->indirect_lock, flags);
 486         return val;
 487 }
 488
 489 /* usec_wait specifies the wait time in usec when writing to certain registers
 490  * where it is unsafe to read back the register without some delay.
 491  * GRC_LOCAL_CTRL is one example if the GPIOs are toggled to switch power.
 492  * TG3PCI_CLOCK_CTRL is another example if the clock frequencies are changed.
 493  */
 494 static void _tw32_flush(struct tg3 *tp, u32 off, u32 val, u32 usec_wait)
 495 {
 496         if (tg3_flag(tp, PCIX_TARGET_HWBUG) || tg3_flag(tp, ICH_WORKAROUND))
 497                 /* Non-posted methods */
 498                 tp->write32(tp, off, val);
 499         else {
 500                 /* Posted method */
 501                 tg3_write32(tp, off, val);
 502                 if (usec_wait)
 503                         udelay(usec_wait);
 504                 tp->read32(tp, off);
 505         }
 506         /* Wait again after the read for the posted method to guarantee that
 507          * the wait time is met.
 508          */
 509         if (usec_wait)
 510                 udelay(usec_wait);
 511 }
 512
 513 static inline void tw32_mailbox_flush(struct tg3 *tp, u32 off, u32 val)
 514 {
 515         tp->write32_mbox(tp, off, val);
 516         if (!tg3_flag(tp, MBOX_WRITE_REORDER) && !tg3_flag(tp, ICH_WORKAROUND))
 517                 tp->read32_mbox(tp, off);
 518 }
 519
 520 static void tg3_write32_tx_mbox(struct tg3 *tp, u32 off, u32 val)
 521 {
 522         void __iomem *mbox = tp->regs + off;
 523         writel(val, mbox);
 524         if (tg3_flag(tp, TXD_MBOX_HWBUG))
 525                 writel(val, mbox);
 526         if (tg3_flag(tp, MBOX_WRITE_REORDER))
 527                 readl(mbox);
 528 }
 529
 530 static u32 tg3_read32_mbox_5906(struct tg3 *tp, u32 off)
 531 {
 532         return readl(tp->regs + off + GRCMBOX_BASE);
 533 }
 534
 535 static void tg3_write32_mbox_5906(struct tg3 *tp, u32 off, u32 val)
 536 {
 537         writel(val, tp->regs + off + GRCMBOX_BASE);
 538 }
 539
 540 #define tw32_mailbox(reg, val)          tp->write32_mbox(tp, reg, val)
 541 #define tw32_mailbox_f(reg, val)        tw32_mailbox_flush(tp, (reg), (val))
 542 #define tw32_rx_mbox(reg, val)          tp->write32_rx_mbox(tp, reg, val)
 543 #define tw32_tx_mbox(reg, val)          tp->write32_tx_mbox(tp, reg, val)
 544 #define tr32_mailbox(reg)               tp->read32_mbox(tp, reg)
 545
 546 #define tw32(reg, val)                  tp->write32(tp, reg, val)
 547 #define tw32_f(reg, val)                _tw32_flush(tp, (reg), (val), 0)
 548 #define tw32_wait_f(reg, val, us)       _tw32_flush(tp, (reg), (val), (us))
 549 #define tr32(reg)                       tp->read32(tp, reg)
 550
 551 static void tg3_write_mem(struct tg3 *tp, u32 off, u32 val)
 552 {
 553         unsigned long flags;
 554
 555         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) &&
 556             (off >= NIC_SRAM_STATS_BLK) && (off < NIC_SRAM_TX_BUFFER_DESC))
 557                 return;
 558
 559         spin_lock_irqsave(&tp->indirect_lock, flags);
 560         if (tg3_flag(tp, SRAM_USE_CONFIG)) {
 561                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, off);
 562                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_DATA, val);
 563
 564                 /* Always leave this as zero. */
 565                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, 0);
 566         } else {
 567                 tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, off);
 568                 tw32_f(TG3PCI_MEM_WIN_DATA, val);
 569
 570                 /* Always leave this as zero. */
 571                 tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, 0);
 572         }
 573         spin_unlock_irqrestore(&tp->indirect_lock, flags);
 574 }
 575
 576 static void tg3_read_mem(struct tg3 *tp, u32 off, u32 *val)
 577 {
 578         unsigned long flags;
 579
 580         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) &&
 581             (off >= NIC_SRAM_STATS_BLK) && (off < NIC_SRAM_TX_BUFFER_DESC)) {
 582                 *val = 0;
 583                 return;
 584         }
 585
 586         spin_lock_irqsave(&tp->indirect_lock, flags);
 587         if (tg3_flag(tp, SRAM_USE_CONFIG)) {
 588                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, off);
 589                 pci_read_config_dword(tp->pdev, TG3PCI_MEM_WIN_DATA, val);
 590
 591                 /* Always leave this as zero. */
 592                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, 0);
 593         } else {
 594                 tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, off);
 595                 *val = tr32(TG3PCI_MEM_WIN_DATA);
 596
 597                 /* Always leave this as zero. */
 598                 tw32_f(TG3PCI_MEM_WIN_BASE_ADDR, 0);
 599         }
 600         spin_unlock_irqrestore(&tp->indirect_lock, flags);
 601 }
 602
 603 static void tg3_ape_lock_init(struct tg3 *tp)
 604 {
 605         int i;
 606         u32 regbase;
 607
 608         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761)
 609                 regbase = TG3_APE_LOCK_GRANT;
 610         else
 611                 regbase = TG3_APE_PER_LOCK_GRANT;
 612
 613         /* Make sure the driver hasn't any stale locks. */
 614         for (i = 0; i < 8; i++)
 615                 tg3_ape_write32(tp, regbase + 4 * i, APE_LOCK_GRANT_DRIVER);
 616 }
 617
 618 static int tg3_ape_lock(struct tg3 *tp, int locknum)
 619 {
 620         int i, off;
 621         int ret = 0;
 622         u32 status, req, gnt;
 623
 624         if (!tg3_flag(tp, ENABLE_APE))
 625                 return 0;
 626
 627         switch (locknum) {
 628         case TG3_APE_LOCK_GRC:
 629         case TG3_APE_LOCK_MEM:
 630                 break;
 631         default:
 632                 return -EINVAL;
 633         }
 634
 635         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761) {
 636                 req = TG3_APE_LOCK_REQ;
 637                 gnt = TG3_APE_LOCK_GRANT;
 638         } else {
 639                 req = TG3_APE_PER_LOCK_REQ;
 640                 gnt = TG3_APE_PER_LOCK_GRANT;
 641         }
 642
 643         off = 4 * locknum;
 644
 645         tg3_ape_write32(tp, req + off, APE_LOCK_REQ_DRIVER);
 646
 647         /* Wait for up to 1 millisecond to acquire lock. */
 648         for (i = 0; i < 100; i++) {
 649                 status = tg3_ape_read32(tp, gnt + off);
 650                 if (status == APE_LOCK_GRANT_DRIVER)
 651                         break;
 652                 udelay(10);
 653         }
 654
 655         if (status != APE_LOCK_GRANT_DRIVER) {
 656                 /* Revoke the lock request. */
 657                 tg3_ape_write32(tp, gnt + off,
 658                                 APE_LOCK_GRANT_DRIVER);
 659
 660                 ret = -EBUSY;
 661         }
 662
 663         return ret;
 664 }
 665
 666 static void tg3_ape_unlock(struct tg3 *tp, int locknum)
 667 {
 668         u32 gnt;
 669
 670         if (!tg3_flag(tp, ENABLE_APE))
 671                 return;
 672
 673         switch (locknum) {
 674         case TG3_APE_LOCK_GRC:
 675         case TG3_APE_LOCK_MEM:
 676                 break;
 677         default:
 678                 return;
 679         }
 680
 681         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761)
 682                 gnt = TG3_APE_LOCK_GRANT;
 683         else
 684                 gnt = TG3_APE_PER_LOCK_GRANT;
 685
 686         tg3_ape_write32(tp, gnt + 4 * locknum, APE_LOCK_GRANT_DRIVER);
 687 }
 688
 689 static void tg3_disable_ints(struct tg3 *tp)
 690 {
 691         int i;
 692
 693         tw32(TG3PCI_MISC_HOST_CTRL,
 694              (tp->misc_host_ctrl | MISC_HOST_CTRL_MASK_PCI_INT));
 695         for (i = 0; i < tp->irq_max; i++)
 696                 tw32_mailbox_f(tp->napi[i].int_mbox, 0x00000001);
 697 }
 698
 699 static void tg3_enable_ints(struct tg3 *tp)
 700 {
 701         int i;
 702
 703         tp->irq_sync = 0;
 704         wmb();
 705
 706         tw32(TG3PCI_MISC_HOST_CTRL,
 707              (tp->misc_host_ctrl & ~MISC_HOST_CTRL_MASK_PCI_INT));
 708
 709         tp->coal_now = tp->coalesce_mode | HOSTCC_MODE_ENABLE;
 710         for (i = 0; i < tp->irq_cnt; i++) {
 711                 struct tg3_napi *tnapi = &tp->napi[i];
 712
 713                 tw32_mailbox_f(tnapi->int_mbox, tnapi->last_tag << 24);
 714                 if (tg3_flag(tp, 1SHOT_MSI))
 715                         tw32_mailbox_f(tnapi->int_mbox, tnapi->last_tag << 24);
 716
 717                 tp->coal_now |= tnapi->coal_now;
 718         }
 719
 720         /* Force an initial interrupt */
 721         if (!tg3_flag(tp, TAGGED_STATUS) &&
 722             (tp->napi[0].hw_status->status & SD_STATUS_UPDATED))
 723                 tw32(GRC_LOCAL_CTRL, tp->grc_local_ctrl | GRC_LCLCTRL_SETINT);
 724         else
 725                 tw32(HOSTCC_MODE, tp->coal_now);
 726
 727         tp->coal_now &= ~(tp->napi[0].coal_now | tp->napi[1].coal_now);
 728 }
 729
 730 static inline unsigned int tg3_has_work(struct tg3_napi *tnapi)
 731 {
 732         struct tg3 *tp = tnapi->tp;
 733         struct tg3_hw_status *sblk = tnapi->hw_status;
 734         unsigned int work_exists = 0;
 735
 736         /* check for phy events */
 737         if (!(tg3_flag(tp, USE_LINKCHG_REG) || tg3_flag(tp, POLL_SERDES))) {
 738                 if (sblk->status & SD_STATUS_LINK_CHG)
 739                         work_exists = 1;
 740         }
 741         /* check for RX/TX work to do */
 742         if (sblk->idx[0].tx_consumer != tnapi->tx_cons ||
 743             *(tnapi->rx_rcb_prod_idx) != tnapi->rx_rcb_ptr)
 744                 work_exists = 1;
 745
 746         return work_exists;
 747 }
 748
 749 /* tg3_int_reenable
 750  *  similar to tg3_enable_ints, but it accurately determines whether there
 751  *  is new work pending and can return without flushing the PIO write
 752  *  which reenables interrupts
 753  */
 754 static void tg3_int_reenable(struct tg3_napi *tnapi)
 755 {
 756         struct tg3 *tp = tnapi->tp;
 757
 758         tw32_mailbox(tnapi->int_mbox, tnapi->last_tag << 24);
 759         mmiowb();
 760
 761         /* When doing tagged status, this work check is unnecessary.
 762          * The last_tag we write above tells the chip which piece of
 763          * work we've completed.
 764          */
 765         if (!tg3_flag(tp, TAGGED_STATUS) && tg3_has_work(tnapi))
 766                 tw32(HOSTCC_MODE, tp->coalesce_mode |
 767                      HOSTCC_MODE_ENABLE | tnapi->coal_now);
 768 }
 769
 770 static void tg3_switch_clocks(struct tg3 *tp)
 771 {
 772         u32 clock_ctrl;
 773         u32 orig_clock_ctrl;
 774
 775         if (tg3_flag(tp, CPMU_PRESENT) || tg3_flag(tp, 5780_CLASS))
 776                 return;
 777
 778         clock_ctrl = tr32(TG3PCI_CLOCK_CTRL);
 779
 780         orig_clock_ctrl = clock_ctrl;
 781         clock_ctrl &= (CLOCK_CTRL_FORCE_CLKRUN |
 782                        CLOCK_CTRL_CLKRUN_OENABLE |
 783                        0x1f);
 784         tp->pci_clock_ctrl = clock_ctrl;
 785
 786         if (tg3_flag(tp, 5705_PLUS)) {
 787                 if (orig_clock_ctrl & CLOCK_CTRL_625_CORE) {
 788                         tw32_wait_f(TG3PCI_CLOCK_CTRL,
 789                                     clock_ctrl | CLOCK_CTRL_625_CORE, 40);
 790                 }
 791         } else if ((orig_clock_ctrl & CLOCK_CTRL_44MHZ_CORE) != 0) {
 792                 tw32_wait_f(TG3PCI_CLOCK_CTRL,
 793                             clock_ctrl |
 794                             (CLOCK_CTRL_44MHZ_CORE | CLOCK_CTRL_ALTCLK),
 795                             40);
 796                 tw32_wait_f(TG3PCI_CLOCK_CTRL,
 797                             clock_ctrl | (CLOCK_CTRL_ALTCLK),
 798                             40);
 799         }
 800         tw32_wait_f(TG3PCI_CLOCK_CTRL, clock_ctrl, 40);
 801 }
 802
 803 #define PHY_BUSY_LOOPS  5000
 804
 805 static int tg3_readphy(struct tg3 *tp, int reg, u32 *val)
 806 {
 807         u32 frame_val;
 808         unsigned int loops;
 809         int ret;
 810
 811         if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
 812                 tw32_f(MAC_MI_MODE,
 813                      (tp->mi_mode & ~MAC_MI_MODE_AUTO_POLL));
 814                 udelay(80);
 815         }
 816
 817         *val = 0x0;
 818
 819         frame_val  = ((tp->phy_addr << MI_COM_PHY_ADDR_SHIFT) &
 820                       MI_COM_PHY_ADDR_MASK);
 821         frame_val |= ((reg << MI_COM_REG_ADDR_SHIFT) &
 822                       MI_COM_REG_ADDR_MASK);
 823         frame_val |= (MI_COM_CMD_READ | MI_COM_START);
 824
 825         tw32_f(MAC_MI_COM, frame_val);
 826
 827         loops = PHY_BUSY_LOOPS;
 828         while (loops != 0) {
 829                 udelay(10);
 830                 frame_val = tr32(MAC_MI_COM);
 831
 832                 if ((frame_val & MI_COM_BUSY) == 0) {
 833                         udelay(5);
 834                         frame_val = tr32(MAC_MI_COM);
 835                         break;
 836                 }
 837                 loops -= 1;
 838         }
 839
 840         ret = -EBUSY;
 841         if (loops != 0) {
 842                 *val = frame_val & MI_COM_DATA_MASK;
 843                 ret = 0;
 844         }
 845
 846         if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
 847                 tw32_f(MAC_MI_MODE, tp->mi_mode);
 848                 udelay(80);
 849         }
 850
 851         return ret;
 852 }
 853
 854 static int tg3_writephy(struct tg3 *tp, int reg, u32 val)
 855 {
 856         u32 frame_val;
 857         unsigned int loops;
 858         int ret;
 859
 860         if ((tp->phy_flags & TG3_PHYFLG_IS_FET) &&
 861             (reg == MII_TG3_CTRL || reg == MII_TG3_AUX_CTRL))
 862                 return 0;
 863
 864         if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
 865                 tw32_f(MAC_MI_MODE,
 866                      (tp->mi_mode & ~MAC_MI_MODE_AUTO_POLL));
 867                 udelay(80);
 868         }
 869
 870         frame_val  = ((tp->phy_addr << MI_COM_PHY_ADDR_SHIFT) &
 871                       MI_COM_PHY_ADDR_MASK);
 872         frame_val |= ((reg << MI_COM_REG_ADDR_SHIFT) &
 873                       MI_COM_REG_ADDR_MASK);
 874         frame_val |= (val & MI_COM_DATA_MASK);
 875         frame_val |= (MI_COM_CMD_WRITE | MI_COM_START);
 876
 877         tw32_f(MAC_MI_COM, frame_val);
 878
 879         loops = PHY_BUSY_LOOPS;
 880         while (loops != 0) {
 881                 udelay(10);
 882                 frame_val = tr32(MAC_MI_COM);
 883                 if ((frame_val & MI_COM_BUSY) == 0) {
 884                         udelay(5);
 885                         frame_val = tr32(MAC_MI_COM);
 886                         break;
 887                 }
 888                 loops -= 1;
 889         }
 890
 891         ret = -EBUSY;
 892         if (loops != 0)
 893                 ret = 0;
 894
 895         if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
 896                 tw32_f(MAC_MI_MODE, tp->mi_mode);
 897                 udelay(80);
 898         }
 899
 900         return ret;
 901 }
 902
 903 static int tg3_phy_cl45_write(struct tg3 *tp, u32 devad, u32 addr, u32 val)
 904 {
 905         int err;
 906
 907         err = tg3_writephy(tp, MII_TG3_MMD_CTRL, devad);
 908         if (err)
 909                 goto done;
 910
 911         err = tg3_writephy(tp, MII_TG3_MMD_ADDRESS, addr);
 912         if (err)
 913                 goto done;
 914
 915         err = tg3_writephy(tp, MII_TG3_MMD_CTRL,
 916                            MII_TG3_MMD_CTRL_DATA_NOINC | devad);
 917         if (err)
 918                 goto done;
 919
 920         err = tg3_writephy(tp, MII_TG3_MMD_ADDRESS, val);
 921
 922 done:
 923         return err;
 924 }
 925
 926 static int tg3_phy_cl45_read(struct tg3 *tp, u32 devad, u32 addr, u32 *val)
 927 {
 928         int err;
 929
 930         err = tg3_writephy(tp, MII_TG3_MMD_CTRL, devad);
 931         if (err)
 932                 goto done;
 933
 934         err = tg3_writephy(tp, MII_TG3_MMD_ADDRESS, addr);
 935         if (err)
 936                 goto done;
 937
 938         err = tg3_writephy(tp, MII_TG3_MMD_CTRL,
 939                            MII_TG3_MMD_CTRL_DATA_NOINC | devad);
 940         if (err)
 941                 goto done;
 942
 943         err = tg3_readphy(tp, MII_TG3_MMD_ADDRESS, val);
 944
 945 done:
 946         return err;
 947 }
 948
 949 static int tg3_phydsp_read(struct tg3 *tp, u32 reg, u32 *val)
 950 {
 951         int err;
 952
 953         err = tg3_writephy(tp, MII_TG3_DSP_ADDRESS, reg);
 954         if (!err)
 955                 err = tg3_readphy(tp, MII_TG3_DSP_RW_PORT, val);
 956
 957         return err;
 958 }
 959
 960 static int tg3_phydsp_write(struct tg3 *tp, u32 reg, u32 val)
 961 {
 962         int err;
 963
 964         err = tg3_writephy(tp, MII_TG3_DSP_ADDRESS, reg);
 965         if (!err)
 966                 err = tg3_writephy(tp, MII_TG3_DSP_RW_PORT, val);
 967
 968         return err;
 969 }
 970
 971 static int tg3_phy_auxctl_read(struct tg3 *tp, int reg, u32 *val)
 972 {
 973         int err;
 974
 975         err = tg3_writephy(tp, MII_TG3_AUX_CTRL,
 976                            (reg << MII_TG3_AUXCTL_MISC_RDSEL_SHIFT) |
 977                            MII_TG3_AUXCTL_SHDWSEL_MISC);
 978         if (!err)
 979                 err = tg3_readphy(tp, MII_TG3_AUX_CTRL, val);
 980
 981         return err;
 982 }
 983
 984 static int tg3_phy_auxctl_write(struct tg3 *tp, int reg, u32 set)
 985 {
 986         if (reg == MII_TG3_AUXCTL_SHDWSEL_MISC)
 987                 set |= MII_TG3_AUXCTL_MISC_WREN;
 988
 989         return tg3_writephy(tp, MII_TG3_AUX_CTRL, set | reg);
 990 }
 991
 992 #define TG3_PHY_AUXCTL_SMDSP_ENABLE(tp) \
 993         tg3_phy_auxctl_write((tp), MII_TG3_AUXCTL_SHDWSEL_AUXCTL, \
 994                              MII_TG3_AUXCTL_ACTL_SMDSP_ENA | \
 995                              MII_TG3_AUXCTL_ACTL_TX_6DB)
 996
 997 #define TG3_PHY_AUXCTL_SMDSP_DISABLE(tp) \
 998         tg3_phy_auxctl_write((tp), MII_TG3_AUXCTL_SHDWSEL_AUXCTL, \
 999                              MII_TG3_AUXCTL_ACTL_TX_6DB);
1000
1001 static int tg3_bmcr_reset(struct tg3 *tp)
1002 {
1003         u32 phy_control;
1004         int limit, err;
1005
1006         /* OK, reset it, and poll the BMCR_RESET bit until it
1007          * clears or we time out.
1008          */
1009         phy_control = BMCR_RESET;
1010         err = tg3_writephy(tp, MII_BMCR, phy_control);
1011         if (err != 0)
1012                 return -EBUSY;
1013
1014         limit = 5000;
1015         while (limit--) {
1016                 err = tg3_readphy(tp, MII_BMCR, &phy_control);
1017                 if (err != 0)
1018                         return -EBUSY;
1019
1020                 if ((phy_control & BMCR_RESET) == 0) {
1021                         udelay(40);
1022                         break;
1023                 }
1024                 udelay(10);
1025         }
1026         if (limit < 0)
1027                 return -EBUSY;
1028
1029         return 0;
1030 }
1031
1032 static int tg3_mdio_read(struct mii_bus *bp, int mii_id, int reg)
1033 {
1034         struct tg3 *tp = bp->priv;
1035         u32 val;
1036
1037         spin_lock_bh(&tp->lock);
1038
1039         if (tg3_readphy(tp, reg, &val))
1040                 val = -EIO;
1041
1042         spin_unlock_bh(&tp->lock);
1043
1044         return val;
1045 }
1046
1047 static int tg3_mdio_write(struct mii_bus *bp, int mii_id, int reg, u16 val)
1048 {
1049         struct tg3 *tp = bp->priv;
1050         u32 ret = 0;
1051
1052         spin_lock_bh(&tp->lock);
1053
1054         if (tg3_writephy(tp, reg, val))
1055                 ret = -EIO;
1056
1057         spin_unlock_bh(&tp->lock);
1058
1059         return ret;
1060 }
1061
1062 static int tg3_mdio_reset(struct mii_bus *bp)
1063 {
1064         return 0;
1065 }
1066
1067 static void tg3_mdio_config_5785(struct tg3 *tp)
1068 {
1069         u32 val;
1070         struct phy_device *phydev;
1071
1072         phydev = tp->mdio_bus->phy_map[TG3_PHY_MII_ADDR];
1073         switch (phydev->drv->phy_id & phydev->drv->phy_id_mask) {
1074         case PHY_ID_BCM50610:
1075         case PHY_ID_BCM50610M:
1076                 val = MAC_PHYCFG2_50610_LED_MODES;
1077                 break;
1078         case PHY_ID_BCMAC131:
1079                 val = MAC_PHYCFG2_AC131_LED_MODES;
1080                 break;
1081         case PHY_ID_RTL8211C:
1082                 val = MAC_PHYCFG2_RTL8211C_LED_MODES;
1083                 break;
1084         case PHY_ID_RTL8201E:
1085                 val = MAC_PHYCFG2_RTL8201E_LED_MODES;
1086                 break;
1087         default:
1088                 return;
1089         }
1090
1091         if (phydev->interface != PHY_INTERFACE_MODE_RGMII) {
1092                 tw32(MAC_PHYCFG2, val);
1093
1094                 val = tr32(MAC_PHYCFG1);
1095                 val &= ~(MAC_PHYCFG1_RGMII_INT |
1096                          MAC_PHYCFG1_RXCLK_TO_MASK | MAC_PHYCFG1_TXCLK_TO_MASK);
1097                 val |= MAC_PHYCFG1_RXCLK_TIMEOUT | MAC_PHYCFG1_TXCLK_TIMEOUT;
1098                 tw32(MAC_PHYCFG1, val);
1099
1100                 return;
1101         }
1102
1103         if (!tg3_flag(tp, RGMII_INBAND_DISABLE))
1104                 val |= MAC_PHYCFG2_EMODE_MASK_MASK |
1105                        MAC_PHYCFG2_FMODE_MASK_MASK |
1106                        MAC_PHYCFG2_GMODE_MASK_MASK |
1107                        MAC_PHYCFG2_ACT_MASK_MASK   |
1108                        MAC_PHYCFG2_QUAL_MASK_MASK |
1109                        MAC_PHYCFG2_INBAND_ENABLE;
1110
1111         tw32(MAC_PHYCFG2, val);
1112
1113         val = tr32(MAC_PHYCFG1);
1114         val &= ~(MAC_PHYCFG1_RXCLK_TO_MASK | MAC_PHYCFG1_TXCLK_TO_MASK |
1115                  MAC_PHYCFG1_RGMII_EXT_RX_DEC | MAC_PHYCFG1_RGMII_SND_STAT_EN);
1116         if (!tg3_flag(tp, RGMII_INBAND_DISABLE)) {
1117                 if (tg3_flag(tp, RGMII_EXT_IBND_RX_EN))
1118                         val |= MAC_PHYCFG1_RGMII_EXT_RX_DEC;
1119                 if (tg3_flag(tp, RGMII_EXT_IBND_TX_EN))
1120                         val |= MAC_PHYCFG1_RGMII_SND_STAT_EN;
1121         }
1122         val |= MAC_PHYCFG1_RXCLK_TIMEOUT | MAC_PHYCFG1_TXCLK_TIMEOUT |
1123                MAC_PHYCFG1_RGMII_INT | MAC_PHYCFG1_TXC_DRV;
1124         tw32(MAC_PHYCFG1, val);
1125
1126         val = tr32(MAC_EXT_RGMII_MODE);
1127         val &= ~(MAC_RGMII_MODE_RX_INT_B |
1128                  MAC_RGMII_MODE_RX_QUALITY |
1129                  MAC_RGMII_MODE_RX_ACTIVITY |
1130                  MAC_RGMII_MODE_RX_ENG_DET |
1131                  MAC_RGMII_MODE_TX_ENABLE |
1132                  MAC_RGMII_MODE_TX_LOWPWR |
1133                  MAC_RGMII_MODE_TX_RESET);
1134         if (!tg3_flag(tp, RGMII_INBAND_DISABLE)) {
1135                 if (tg3_flag(tp, RGMII_EXT_IBND_RX_EN))
1136                         val |= MAC_RGMII_MODE_RX_INT_B |
1137                                MAC_RGMII_MODE_RX_QUALITY |
1138                                MAC_RGMII_MODE_RX_ACTIVITY |
1139                                MAC_RGMII_MODE_RX_ENG_DET;
1140                 if (tg3_flag(tp, RGMII_EXT_IBND_TX_EN))
1141                         val |= MAC_RGMII_MODE_TX_ENABLE |
1142                                MAC_RGMII_MODE_TX_LOWPWR |
1143                                MAC_RGMII_MODE_TX_RESET;
1144         }
1145         tw32(MAC_EXT_RGMII_MODE, val);
1146 }
1147
1148 static void tg3_mdio_start(struct tg3 *tp)
1149 {
1150         tp->mi_mode &= ~MAC_MI_MODE_AUTO_POLL;
1151         tw32_f(MAC_MI_MODE, tp->mi_mode);
1152         udelay(80);
1153
1154         if (tg3_flag(tp, MDIOBUS_INITED) &&
1155             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5785)
1156                 tg3_mdio_config_5785(tp);
1157 }
1158
1159 static int tg3_mdio_init(struct tg3 *tp)
1160 {
1161         int i;
1162         u32 reg;
1163         struct phy_device *phydev;
1164
1165         if (tg3_flag(tp, 5717_PLUS)) {
1166                 u32 is_serdes;
1167
1168                 tp->phy_addr = PCI_FUNC(tp->pdev->devfn) + 1;
1169
1170                 if (tp->pci_chip_rev_id != CHIPREV_ID_5717_A0)
1171                         is_serdes = tr32(SG_DIG_STATUS) & SG_DIG_IS_SERDES;
1172                 else
1173                         is_serdes = tr32(TG3_CPMU_PHY_STRAP) &
1174                                     TG3_CPMU_PHY_STRAP_IS_SERDES;
1175                 if (is_serdes)
1176                         tp->phy_addr += 7;
1177         } else
1178                 tp->phy_addr = TG3_PHY_MII_ADDR;
1179
1180         tg3_mdio_start(tp);
1181
1182         if (!tg3_flag(tp, USE_PHYLIB) || tg3_flag(tp, MDIOBUS_INITED))
1183                 return 0;
1184
1185         tp->mdio_bus = mdiobus_alloc();
1186         if (tp->mdio_bus == NULL)
1187                 return -ENOMEM;
1188
1189         tp->mdio_bus->name     = "tg3 mdio bus";
1190         snprintf(tp->mdio_bus->id, MII_BUS_ID_SIZE, "%x",
1191                  (tp->pdev->bus->number << 8) | tp->pdev->devfn);
1192         tp->mdio_bus->priv     = tp;
1193         tp->mdio_bus->parent   = &tp->pdev->dev;
1194         tp->mdio_bus->read     = &tg3_mdio_read;
1195         tp->mdio_bus->write    = &tg3_mdio_write;
1196         tp->mdio_bus->reset    = &tg3_mdio_reset;
1197         tp->mdio_bus->phy_mask = ~(1 << TG3_PHY_MII_ADDR);
1198         tp->mdio_bus->irq      = &tp->mdio_irq[0];
1199
1200         for (i = 0; i < PHY_MAX_ADDR; i++)
1201                 tp->mdio_bus->irq[i] = PHY_POLL;
1202
1203         /* The bus registration will look for all the PHYs on the mdio bus.
1204          * Unfortunately, it does not ensure the PHY is powered up before
1205          * accessing the PHY ID registers.  A chip reset is the
1206          * quickest way to bring the device back to an operational state..
1207          */
1208         if (tg3_readphy(tp, MII_BMCR, &reg) || (reg & BMCR_PDOWN))
1209                 tg3_bmcr_reset(tp);
1210
1211         i = mdiobus_register(tp->mdio_bus);
1212         if (i) {
1213                 dev_warn(&tp->pdev->dev, "mdiobus_reg failed (0x%x)\n", i);
1214                 mdiobus_free(tp->mdio_bus);
1215                 return i;
1216         }
1217
1218         phydev = tp->mdio_bus->phy_map[TG3_PHY_MII_ADDR];
1219
1220         if (!phydev || !phydev->drv) {
1221                 dev_warn(&tp->pdev->dev, "No PHY devices\n");
1222                 mdiobus_unregister(tp->mdio_bus);
1223                 mdiobus_free(tp->mdio_bus);
1224                 return -ENODEV;
1225         }
1226
1227         switch (phydev->drv->phy_id & phydev->drv->phy_id_mask) {
1228         case PHY_ID_BCM57780:
1229                 phydev->interface = PHY_INTERFACE_MODE_GMII;
1230                 phydev->dev_flags |= PHY_BRCM_AUTO_PWRDWN_ENABLE;
1231                 break;
1232         case PHY_ID_BCM50610:
1233         case PHY_ID_BCM50610M:
1234                 phydev->dev_flags |= PHY_BRCM_CLEAR_RGMII_MODE |
1235                                      PHY_BRCM_RX_REFCLK_UNUSED |
1236                                      PHY_BRCM_DIS_TXCRXC_NOENRGY |
1237                                      PHY_BRCM_AUTO_PWRDWN_ENABLE;
1238                 if (tg3_flag(tp, RGMII_INBAND_DISABLE))
1239                         phydev->dev_flags |= PHY_BRCM_STD_IBND_DISABLE;
1240                 if (tg3_flag(tp, RGMII_EXT_IBND_RX_EN))
1241                         phydev->dev_flags |= PHY_BRCM_EXT_IBND_RX_ENABLE;
1242                 if (tg3_flag(tp, RGMII_EXT_IBND_TX_EN))
1243                         phydev->dev_flags |= PHY_BRCM_EXT_IBND_TX_ENABLE;
1244                 /* fallthru */
1245         case PHY_ID_RTL8211C:
1246                 phydev->interface = PHY_INTERFACE_MODE_RGMII;
1247                 break;
1248         case PHY_ID_RTL8201E:
1249         case PHY_ID_BCMAC131:
1250                 phydev->interface = PHY_INTERFACE_MODE_MII;
1251                 phydev->dev_flags |= PHY_BRCM_AUTO_PWRDWN_ENABLE;
1252                 tp->phy_flags |= TG3_PHYFLG_IS_FET;
1253                 break;
1254         }
1255
1256         tg3_flag_set(tp, MDIOBUS_INITED);
1257
1258         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5785)
1259                 tg3_mdio_config_5785(tp);
1260
1261         return 0;
1262 }
1263
1264 static void tg3_mdio_fini(struct tg3 *tp)
1265 {
1266         if (tg3_flag(tp, MDIOBUS_INITED)) {
1267                 tg3_flag_clear(tp, MDIOBUS_INITED);
1268                 mdiobus_unregister(tp->mdio_bus);
1269                 mdiobus_free(tp->mdio_bus);
1270         }
1271 }
1272
1273 /* tp->lock is held. */
1274 static inline void tg3_generate_fw_event(struct tg3 *tp)
1275 {
1276         u32 val;
1277
1278         val = tr32(GRC_RX_CPU_EVENT);
1279         val |= GRC_RX_CPU_DRIVER_EVENT;
1280         tw32_f(GRC_RX_CPU_EVENT, val);
1281
1282         tp->last_event_jiffies = jiffies;
1283 }
1284
1285 #define TG3_FW_EVENT_TIMEOUT_USEC 2500
1286
1287 /* tp->lock is held. */
1288 static void tg3_wait_for_event_ack(struct tg3 *tp)
1289 {
1290         int i;
1291         unsigned int delay_cnt;
1292         long time_remain;
1293
1294         /* If enough time has passed, no wait is necessary. */
1295         time_remain = (long)(tp->last_event_jiffies + 1 +
1296                       usecs_to_jiffies(TG3_FW_EVENT_TIMEOUT_USEC)) -
1297                       (long)jiffies;
1298         if (time_remain < 0)
1299                 return;
1300
1301         /* Check if we can shorten the wait time. */
1302         delay_cnt = jiffies_to_usecs(time_remain);
1303         if (delay_cnt > TG3_FW_EVENT_TIMEOUT_USEC)
1304                 delay_cnt = TG3_FW_EVENT_TIMEOUT_USEC;
1305         delay_cnt = (delay_cnt >> 3) + 1;
1306
1307         for (i = 0; i < delay_cnt; i++) {
1308                 if (!(tr32(GRC_RX_CPU_EVENT) & GRC_RX_CPU_DRIVER_EVENT))
1309                         break;
1310                 udelay(8);
1311         }
1312 }
1313
1314 /* tp->lock is held. */
1315 static void tg3_ump_link_report(struct tg3 *tp)
1316 {
1317         u32 reg;
1318         u32 val;
1319
1320         if (!tg3_flag(tp, 5780_CLASS) || !tg3_flag(tp, ENABLE_ASF))
1321                 return;
1322
1323         tg3_wait_for_event_ack(tp);
1324
1325         tg3_write_mem(tp, NIC_SRAM_FW_CMD_MBOX, FWCMD_NICDRV_LINK_UPDATE);
1326
1327         tg3_write_mem(tp, NIC_SRAM_FW_CMD_LEN_MBOX, 14);
1328
1329         val = 0;
1330         if (!tg3_readphy(tp, MII_BMCR, &reg))
1331                 val = reg << 16;
1332         if (!tg3_readphy(tp, MII_BMSR, &reg))
1333                 val |= (reg & 0xffff);
1334         tg3_write_mem(tp, NIC_SRAM_FW_CMD_DATA_MBOX, val);
1335
1336         val = 0;
1337         if (!tg3_readphy(tp, MII_ADVERTISE, &reg))
1338                 val = reg << 16;
1339         if (!tg3_readphy(tp, MII_LPA, &reg))
1340                 val |= (reg & 0xffff);
1341         tg3_write_mem(tp, NIC_SRAM_FW_CMD_DATA_MBOX + 4, val);
1342
1343         val = 0;
1344         if (!(tp->phy_flags & TG3_PHYFLG_MII_SERDES)) {
1345                 if (!tg3_readphy(tp, MII_CTRL1000, &reg))
1346                         val = reg << 16;
1347                 if (!tg3_readphy(tp, MII_STAT1000, &reg))
1348                         val |= (reg & 0xffff);
1349         }
1350         tg3_write_mem(tp, NIC_SRAM_FW_CMD_DATA_MBOX + 8, val);
1351
1352         if (!tg3_readphy(tp, MII_PHYADDR, &reg))
1353                 val = reg << 16;
1354         else
1355                 val = 0;
1356         tg3_write_mem(tp, NIC_SRAM_FW_CMD_DATA_MBOX + 12, val);
1357
1358         tg3_generate_fw_event(tp);
1359 }
1360
1361 static void tg3_link_report(struct tg3 *tp)
1362 {
1363         if (!netif_carrier_ok(tp->dev)) {
1364                 netif_info(tp, link, tp->dev, "Link is down\n");
1365                 tg3_ump_link_report(tp);
1366         } else if (netif_msg_link(tp)) {
1367                 netdev_info(tp->dev, "Link is up at %d Mbps, %s duplex\n",
1368                             (tp->link_config.active_speed == SPEED_1000 ?
1369                              1000 :
1370                              (tp->link_config.active_speed == SPEED_100 ?
1371                               100 : 10)),
1372                             (tp->link_config.active_duplex == DUPLEX_FULL ?
1373                              "full" : "half"));
1374
1375                 netdev_info(tp->dev, "Flow control is %s for TX and %s for RX\n",
1376                             (tp->link_config.active_flowctrl & FLOW_CTRL_TX) ?
1377                             "on" : "off",
1378                             (tp->link_config.active_flowctrl & FLOW_CTRL_RX) ?
1379                             "on" : "off");
1380
1381                 if (tp->phy_flags & TG3_PHYFLG_EEE_CAP)
1382                         netdev_info(tp->dev, "EEE is %s\n",
1383                                     tp->setlpicnt ? "enabled" : "disabled");
1384
1385                 tg3_ump_link_report(tp);
1386         }
1387 }
1388
1389 static u16 tg3_advert_flowctrl_1000T(u8 flow_ctrl)
1390 {
1391         u16 miireg;
1392
1393         if ((flow_ctrl & FLOW_CTRL_TX) && (flow_ctrl & FLOW_CTRL_RX))
1394                 miireg = ADVERTISE_PAUSE_CAP;
1395         else if (flow_ctrl & FLOW_CTRL_TX)
1396                 miireg = ADVERTISE_PAUSE_ASYM;
1397         else if (flow_ctrl & FLOW_CTRL_RX)
1398                 miireg = ADVERTISE_PAUSE_CAP | ADVERTISE_PAUSE_ASYM;
1399         else
1400                 miireg = 0;
1401
1402         return miireg;
1403 }
1404
1405 static u16 tg3_advert_flowctrl_1000X(u8 flow_ctrl)
1406 {
1407         u16 miireg;
1408
1409         if ((flow_ctrl & FLOW_CTRL_TX) && (flow_ctrl & FLOW_CTRL_RX))
1410                 miireg = ADVERTISE_1000XPAUSE;
1411         else if (flow_ctrl & FLOW_CTRL_TX)
1412                 miireg = ADVERTISE_1000XPSE_ASYM;
1413         else if (flow_ctrl & FLOW_CTRL_RX)
1414                 miireg = ADVERTISE_1000XPAUSE | ADVERTISE_1000XPSE_ASYM;
1415         else
1416                 miireg = 0;
1417
1418         return miireg;
1419 }
1420
1421 static u8 tg3_resolve_flowctrl_1000X(u16 lcladv, u16 rmtadv)
1422 {
1423         u8 cap = 0;
1424
1425         if (lcladv & ADVERTISE_1000XPAUSE) {
1426                 if (lcladv & ADVERTISE_1000XPSE_ASYM) {
1427                         if (rmtadv & LPA_1000XPAUSE)
1428                                 cap = FLOW_CTRL_TX | FLOW_CTRL_RX;
1429                         else if (rmtadv & LPA_1000XPAUSE_ASYM)
1430                                 cap = FLOW_CTRL_RX;
1431                 } else {
1432                         if (rmtadv & LPA_1000XPAUSE)
1433                                 cap = FLOW_CTRL_TX | FLOW_CTRL_RX;
1434                 }
1435         } else if (lcladv & ADVERTISE_1000XPSE_ASYM) {
1436                 if ((rmtadv & LPA_1000XPAUSE) && (rmtadv & LPA_1000XPAUSE_ASYM))
1437                         cap = FLOW_CTRL_TX;
1438         }
1439
1440         return cap;
1441 }
1442
1443 static void tg3_setup_flow_control(struct tg3 *tp, u32 lcladv, u32 rmtadv)
1444 {
1445         u8 autoneg;
1446         u8 flowctrl = 0;
1447         u32 old_rx_mode = tp->rx_mode;
1448         u32 old_tx_mode = tp->tx_mode;
1449
1450         if (tg3_flag(tp, USE_PHYLIB))
1451                 autoneg = tp->mdio_bus->phy_map[TG3_PHY_MII_ADDR]->autoneg;
1452         else
1453                 autoneg = tp->link_config.autoneg;
1454
1455         if (autoneg == AUTONEG_ENABLE && tg3_flag(tp, PAUSE_AUTONEG)) {
1456                 if (tp->phy_flags & TG3_PHYFLG_ANY_SERDES)
1457                         flowctrl = tg3_resolve_flowctrl_1000X(lcladv, rmtadv);
1458                 else
1459                         flowctrl = mii_resolve_flowctrl_fdx(lcladv, rmtadv);
1460         } else
1461                 flowctrl = tp->link_config.flowctrl;
1462
1463         tp->link_config.active_flowctrl = flowctrl;
1464
1465         if (flowctrl & FLOW_CTRL_RX)
1466                 tp->rx_mode |= RX_MODE_FLOW_CTRL_ENABLE;
1467         else
1468                 tp->rx_mode &= ~RX_MODE_FLOW_CTRL_ENABLE;
1469
1470         if (old_rx_mode != tp->rx_mode)
1471                 tw32_f(MAC_RX_MODE, tp->rx_mode);
1472
1473         if (flowctrl & FLOW_CTRL_TX)
1474                 tp->tx_mode |= TX_MODE_FLOW_CTRL_ENABLE;
1475         else
1476                 tp->tx_mode &= ~TX_MODE_FLOW_CTRL_ENABLE;
1477
1478         if (old_tx_mode != tp->tx_mode)
1479                 tw32_f(MAC_TX_MODE, tp->tx_mode);
1480 }
1481
1482 static void tg3_adjust_link(struct net_device *dev)
1483 {
1484         u8 oldflowctrl, linkmesg = 0;
1485         u32 mac_mode, lcl_adv, rmt_adv;
1486         struct tg3 *tp = netdev_priv(dev);
1487         struct phy_device *phydev = tp->mdio_bus->phy_map[TG3_PHY_MII_ADDR];
1488
1489         spin_lock_bh(&tp->lock);
1490
1491         mac_mode = tp->mac_mode & ~(MAC_MODE_PORT_MODE_MASK |
1492                                     MAC_MODE_HALF_DUPLEX);
1493
1494         oldflowctrl = tp->link_config.active_flowctrl;
1495
1496         if (phydev->link) {
1497                 lcl_adv = 0;
1498                 rmt_adv = 0;
1499
1500                 if (phydev->speed == SPEED_100 || phydev->speed == SPEED_10)
1501                         mac_mode |= MAC_MODE_PORT_MODE_MII;
1502                 else if (phydev->speed == SPEED_1000 ||
1503                          GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5785)
1504                         mac_mode |= MAC_MODE_PORT_MODE_GMII;
1505                 else
1506                         mac_mode |= MAC_MODE_PORT_MODE_MII;
1507
1508                 if (phydev->duplex == DUPLEX_HALF)
1509                         mac_mode |= MAC_MODE_HALF_DUPLEX;
1510                 else {
1511                         lcl_adv = tg3_advert_flowctrl_1000T(
1512                                   tp->link_config.flowctrl);
1513
1514                         if (phydev->pause)
1515                                 rmt_adv = LPA_PAUSE_CAP;
1516                         if (phydev->asym_pause)
1517                                 rmt_adv |= LPA_PAUSE_ASYM;
1518                 }
1519
1520                 tg3_setup_flow_control(tp, lcl_adv, rmt_adv);
1521         } else
1522                 mac_mode |= MAC_MODE_PORT_MODE_GMII;
1523
1524         if (mac_mode != tp->mac_mode) {
1525                 tp->mac_mode = mac_mode;
1526                 tw32_f(MAC_MODE, tp->mac_mode);
1527                 udelay(40);
1528         }
1529
1530         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5785) {
1531                 if (phydev->speed == SPEED_10)
1532                         tw32(MAC_MI_STAT,
1533                              MAC_MI_STAT_10MBPS_MODE |
1534                              MAC_MI_STAT_LNKSTAT_ATTN_ENAB);
1535                 else
1536                         tw32(MAC_MI_STAT, MAC_MI_STAT_LNKSTAT_ATTN_ENAB);
1537         }
1538
1539         if (phydev->speed == SPEED_1000 && phydev->duplex == DUPLEX_HALF)
1540                 tw32(MAC_TX_LENGTHS,
1541                      ((2 << TX_LENGTHS_IPG_CRS_SHIFT) |
1542                       (6 << TX_LENGTHS_IPG_SHIFT) |
1543                       (0xff << TX_LENGTHS_SLOT_TIME_SHIFT)));
1544         else
1545                 tw32(MAC_TX_LENGTHS,
1546                      ((2 << TX_LENGTHS_IPG_CRS_SHIFT) |
1547                       (6 << TX_LENGTHS_IPG_SHIFT) |
1548                       (32 << TX_LENGTHS_SLOT_TIME_SHIFT)));
1549
1550         if ((phydev->link && tp->link_config.active_speed == SPEED_INVALID) ||
1551             (!phydev->link && tp->link_config.active_speed != SPEED_INVALID) ||
1552             phydev->speed != tp->link_config.active_speed ||
1553             phydev->duplex != tp->link_config.active_duplex ||
1554             oldflowctrl != tp->link_config.active_flowctrl)
1555                 linkmesg = 1;
1556
1557         tp->link_config.active_speed = phydev->speed;
1558         tp->link_config.active_duplex = phydev->duplex;
1559
1560         spin_unlock_bh(&tp->lock);
1561
1562         if (linkmesg)
1563                 tg3_link_report(tp);
1564 }
1565
1566 static int tg3_phy_init(struct tg3 *tp)
1567 {
1568         struct phy_device *phydev;
1569
1570         if (tp->phy_flags & TG3_PHYFLG_IS_CONNECTED)
1571                 return 0;
1572
1573         /* Bring the PHY back to a known state. */
1574         tg3_bmcr_reset(tp);
1575
1576         phydev = tp->mdio_bus->phy_map[TG3_PHY_MII_ADDR];
1577
1578         /* Attach the MAC to the PHY. */
1579         phydev = phy_connect(tp->dev, dev_name(&phydev->dev), tg3_adjust_link,
1580                              phydev->dev_flags, phydev->interface);
1581         if (IS_ERR(phydev)) {
1582                 dev_err(&tp->pdev->dev, "Could not attach to PHY\n");
1583                 return PTR_ERR(phydev);
1584         }
1585
1586         /* Mask with MAC supported features. */
1587         switch (phydev->interface) {
1588         case PHY_INTERFACE_MODE_GMII:
1589         case PHY_INTERFACE_MODE_RGMII:
1590                 if (!(tp->phy_flags & TG3_PHYFLG_10_100_ONLY)) {
1591                         phydev->supported &= (PHY_GBIT_FEATURES |
1592                                               SUPPORTED_Pause |
1593                                               SUPPORTED_Asym_Pause);
1594                         break;
1595                 }
1596                 /* fallthru */
1597         case PHY_INTERFACE_MODE_MII:
1598                 phydev->supported &= (PHY_BASIC_FEATURES |
1599                                       SUPPORTED_Pause |
1600                                       SUPPORTED_Asym_Pause);
1601                 break;
1602         default:
1603                 phy_disconnect(tp->mdio_bus->phy_map[TG3_PHY_MII_ADDR]);
1604                 return -EINVAL;
1605         }
1606
1607         tp->phy_flags |= TG3_PHYFLG_IS_CONNECTED;
1608
1609         phydev->advertising = phydev->supported;
1610
1611         return 0;
1612 }
1613
1614 static void tg3_phy_start(struct tg3 *tp)
1615 {
1616         struct phy_device *phydev;
1617
1618         if (!(tp->phy_flags & TG3_PHYFLG_IS_CONNECTED))
1619                 return;
1620
1621         phydev = tp->mdio_bus->phy_map[TG3_PHY_MII_ADDR];
1622
1623         if (tp->phy_flags & TG3_PHYFLG_IS_LOW_POWER) {
1624                 tp->phy_flags &= ~TG3_PHYFLG_IS_LOW_POWER;
1625                 phydev->speed = tp->link_config.orig_speed;
1626                 phydev->duplex = tp->link_config.orig_duplex;
1627                 phydev->autoneg = tp->link_config.orig_autoneg;
1628                 phydev->advertising = tp->link_config.orig_advertising;
1629         }
1630
1631         phy_start(phydev);
1632
1633         phy_start_aneg(phydev);
1634 }
1635
1636 static void tg3_phy_stop(struct tg3 *tp)
1637 {
1638         if (!(tp->phy_flags & TG3_PHYFLG_IS_CONNECTED))
1639                 return;
1640
1641         phy_stop(tp->mdio_bus->phy_map[TG3_PHY_MII_ADDR]);
1642 }
1643
1644 static void tg3_phy_fini(struct tg3 *tp)
1645 {
1646         if (tp->phy_flags & TG3_PHYFLG_IS_CONNECTED) {
1647                 phy_disconnect(tp->mdio_bus->phy_map[TG3_PHY_MII_ADDR]);
1648                 tp->phy_flags &= ~TG3_PHYFLG_IS_CONNECTED;
1649         }
1650 }
1651
1652 static void tg3_phy_fet_toggle_apd(struct tg3 *tp, bool enable)
1653 {
1654         u32 phytest;
1655
1656         if (!tg3_readphy(tp, MII_TG3_FET_TEST, &phytest)) {
1657                 u32 phy;
1658
1659                 tg3_writephy(tp, MII_TG3_FET_TEST,
1660                              phytest | MII_TG3_FET_SHADOW_EN);
1661                 if (!tg3_readphy(tp, MII_TG3_FET_SHDW_AUXSTAT2, &phy)) {
1662                         if (enable)
1663                                 phy |= MII_TG3_FET_SHDW_AUXSTAT2_APD;
1664                         else
1665                                 phy &= ~MII_TG3_FET_SHDW_AUXSTAT2_APD;
1666                         tg3_writephy(tp, MII_TG3_FET_SHDW_AUXSTAT2, phy);
1667                 }
1668                 tg3_writephy(tp, MII_TG3_FET_TEST, phytest);
1669         }
1670 }
1671
1672 static void tg3_phy_toggle_apd(struct tg3 *tp, bool enable)
1673 {
1674         u32 reg;
1675
1676         if (!tg3_flag(tp, 5705_PLUS) ||
1677             (tg3_flag(tp, 5717_PLUS) &&
1678              (tp->phy_flags & TG3_PHYFLG_MII_SERDES)))
1679                 return;
1680
1681         if (tp->phy_flags & TG3_PHYFLG_IS_FET) {
1682                 tg3_phy_fet_toggle_apd(tp, enable);
1683                 return;
1684         }
1685
1686         reg = MII_TG3_MISC_SHDW_WREN |
1687               MII_TG3_MISC_SHDW_SCR5_SEL |
1688               MII_TG3_MISC_SHDW_SCR5_LPED |
1689               MII_TG3_MISC_SHDW_SCR5_DLPTLM |
1690               MII_TG3_MISC_SHDW_SCR5_SDTL |
1691               MII_TG3_MISC_SHDW_SCR5_C125OE;
1692         if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5784 || !enable)
1693                 reg |= MII_TG3_MISC_SHDW_SCR5_DLLAPD;
1694
1695         tg3_writephy(tp, MII_TG3_MISC_SHDW, reg);
1696
1697
1698         reg = MII_TG3_MISC_SHDW_WREN |
1699               MII_TG3_MISC_SHDW_APD_SEL |
1700               MII_TG3_MISC_SHDW_APD_WKTM_84MS;
1701         if (enable)
1702                 reg |= MII_TG3_MISC_SHDW_APD_ENABLE;
1703
1704         tg3_writephy(tp, MII_TG3_MISC_SHDW, reg);
1705 }
1706
1707 static void tg3_phy_toggle_automdix(struct tg3 *tp, int enable)
1708 {
1709         u32 phy;
1710
1711         if (!tg3_flag(tp, 5705_PLUS) ||
1712             (tp->phy_flags & TG3_PHYFLG_ANY_SERDES))
1713                 return;
1714
1715         if (tp->phy_flags & TG3_PHYFLG_IS_FET) {
1716                 u32 ephy;
1717
1718                 if (!tg3_readphy(tp, MII_TG3_FET_TEST, &ephy)) {
1719                         u32 reg = MII_TG3_FET_SHDW_MISCCTRL;
1720
1721                         tg3_writephy(tp, MII_TG3_FET_TEST,
1722                                      ephy | MII_TG3_FET_SHADOW_EN);
1723                         if (!tg3_readphy(tp, reg, &phy)) {
1724                                 if (enable)
1725                                         phy |= MII_TG3_FET_SHDW_MISCCTRL_MDIX;
1726                                 else
1727                                         phy &= ~MII_TG3_FET_SHDW_MISCCTRL_MDIX;
1728                                 tg3_writephy(tp, reg, phy);
1729                         }
1730                         tg3_writephy(tp, MII_TG3_FET_TEST, ephy);
1731                 }
1732         } else {
1733                 int ret;
1734
1735                 ret = tg3_phy_auxctl_read(tp,
1736                                           MII_TG3_AUXCTL_SHDWSEL_MISC, &phy);
1737                 if (!ret) {
1738                         if (enable)
1739                                 phy |= MII_TG3_AUXCTL_MISC_FORCE_AMDIX;
1740                         else
1741                                 phy &= ~MII_TG3_AUXCTL_MISC_FORCE_AMDIX;
1742                         tg3_phy_auxctl_write(tp,
1743                                              MII_TG3_AUXCTL_SHDWSEL_MISC, phy);
1744                 }
1745         }
1746 }
1747
1748 static void tg3_phy_set_wirespeed(struct tg3 *tp)
1749 {
1750         int ret;
1751         u32 val;
1752
1753         if (tp->phy_flags & TG3_PHYFLG_NO_ETH_WIRE_SPEED)
1754                 return;
1755
1756         ret = tg3_phy_auxctl_read(tp, MII_TG3_AUXCTL_SHDWSEL_MISC, &val);
1757         if (!ret)
1758                 tg3_phy_auxctl_write(tp, MII_TG3_AUXCTL_SHDWSEL_MISC,
1759                                      val | MII_TG3_AUXCTL_MISC_WIRESPD_EN);
1760 }
1761
1762 static void tg3_phy_apply_otp(struct tg3 *tp)
1763 {
1764         u32 otp, phy;
1765
1766         if (!tp->phy_otp)
1767                 return;
1768
1769         otp = tp->phy_otp;
1770
1771         if (TG3_PHY_AUXCTL_SMDSP_ENABLE(tp))
1772                 return;
1773
1774         phy = ((otp & TG3_OTP_AGCTGT_MASK) >> TG3_OTP_AGCTGT_SHIFT);
1775         phy |= MII_TG3_DSP_TAP1_AGCTGT_DFLT;
1776         tg3_phydsp_write(tp, MII_TG3_DSP_TAP1, phy);
1777
1778         phy = ((otp & TG3_OTP_HPFFLTR_MASK) >> TG3_OTP_HPFFLTR_SHIFT) |
1779               ((otp & TG3_OTP_HPFOVER_MASK) >> TG3_OTP_HPFOVER_SHIFT);
1780         tg3_phydsp_write(tp, MII_TG3_DSP_AADJ1CH0, phy);
1781
1782         phy = ((otp & TG3_OTP_LPFDIS_MASK) >> TG3_OTP_LPFDIS_SHIFT);
1783         phy |= MII_TG3_DSP_AADJ1CH3_ADCCKADJ;
1784         tg3_phydsp_write(tp, MII_TG3_DSP_AADJ1CH3, phy);
1785
1786         phy = ((otp & TG3_OTP_VDAC_MASK) >> TG3_OTP_VDAC_SHIFT);
1787         tg3_phydsp_write(tp, MII_TG3_DSP_EXP75, phy);
1788
1789         phy = ((otp & TG3_OTP_10BTAMP_MASK) >> TG3_OTP_10BTAMP_SHIFT);
1790         tg3_phydsp_write(tp, MII_TG3_DSP_EXP96, phy);
1791
1792         phy = ((otp & TG3_OTP_ROFF_MASK) >> TG3_OTP_ROFF_SHIFT) |
1793               ((otp & TG3_OTP_RCOFF_MASK) >> TG3_OTP_RCOFF_SHIFT);
1794         tg3_phydsp_write(tp, MII_TG3_DSP_EXP97, phy);
1795
1796         TG3_PHY_AUXCTL_SMDSP_DISABLE(tp);
1797 }
1798
1799 static void tg3_phy_eee_adjust(struct tg3 *tp, u32 current_link_up)
1800 {
1801         u32 val;
1802
1803         if (!(tp->phy_flags & TG3_PHYFLG_EEE_CAP))
1804                 return;
1805
1806         tp->setlpicnt = 0;
1807
1808         if (tp->link_config.autoneg == AUTONEG_ENABLE &&
1809             current_link_up == 1 &&
1810             tp->link_config.active_duplex == DUPLEX_FULL &&
1811             (tp->link_config.active_speed == SPEED_100 ||
1812              tp->link_config.active_speed == SPEED_1000)) {
1813                 u32 eeectl;
1814
1815                 if (tp->link_config.active_speed == SPEED_1000)
1816                         eeectl = TG3_CPMU_EEE_CTRL_EXIT_16_5_US;
1817                 else
1818                         eeectl = TG3_CPMU_EEE_CTRL_EXIT_36_US;
1819
1820                 tw32(TG3_CPMU_EEE_CTRL, eeectl);
1821
1822                 tg3_phy_cl45_read(tp, MDIO_MMD_AN,
1823                                   TG3_CL45_D7_EEERES_STAT, &val);
1824
1825                 switch (val) {
1826                 case TG3_CL45_D7_EEERES_STAT_LP_1000T:
1827                         switch (GET_ASIC_REV(tp->pci_chip_rev_id)) {
1828                         case ASIC_REV_5717:
1829                         case ASIC_REV_5719:
1830                         case ASIC_REV_57765:
1831                                 if (!TG3_PHY_AUXCTL_SMDSP_ENABLE(tp)) {
1832                                         tg3_phydsp_write(tp, MII_TG3_DSP_TAP26,
1833                                                          0x0000);
1834                                         TG3_PHY_AUXCTL_SMDSP_DISABLE(tp);
1835                                 }
1836                         }
1837                         /* Fallthrough */
1838                 case TG3_CL45_D7_EEERES_STAT_LP_100TX:
1839                         tp->setlpicnt = 2;
1840                 }
1841         }
1842
1843         if (!tp->setlpicnt) {
1844                 val = tr32(TG3_CPMU_EEE_MODE);
1845                 tw32(TG3_CPMU_EEE_MODE, val & ~TG3_CPMU_EEEMD_LPI_ENABLE);
1846         }
1847 }
1848
1849 static int tg3_wait_macro_done(struct tg3 *tp)
1850 {
1851         int limit = 100;
1852
1853         while (limit--) {
1854                 u32 tmp32;
1855
1856                 if (!tg3_readphy(tp, MII_TG3_DSP_CONTROL, &tmp32)) {
1857                         if ((tmp32 & 0x1000) == 0)
1858                                 break;
1859                 }
1860         }
1861         if (limit < 0)
1862                 return -EBUSY;
1863
1864         return 0;
1865 }
1866
1867 static int tg3_phy_write_and_check_testpat(struct tg3 *tp, int *resetp)
1868 {
1869         static const u32 test_pat[4][6] = {
1870         { 0x00005555, 0x00000005, 0x00002aaa, 0x0000000a, 0x00003456, 0x00000003 },
1871         { 0x00002aaa, 0x0000000a, 0x00003333, 0x00000003, 0x0000789a, 0x00000005 },
1872         { 0x00005a5a, 0x00000005, 0x00002a6a, 0x0000000a, 0x00001bcd, 0x00000003 },
1873         { 0x00002a5a, 0x0000000a, 0x000033c3, 0x00000003, 0x00002ef1, 0x00000005 }
1874         };
1875         int chan;
1876
1877         for (chan = 0; chan < 4; chan++) {
1878                 int i;
1879
1880                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS,
1881                              (chan * 0x2000) | 0x0200);
1882                 tg3_writephy(tp, MII_TG3_DSP_CONTROL, 0x0002);
1883
1884                 for (i = 0; i < 6; i++)
1885                         tg3_writephy(tp, MII_TG3_DSP_RW_PORT,
1886                                      test_pat[chan][i]);
1887
1888                 tg3_writephy(tp, MII_TG3_DSP_CONTROL, 0x0202);
1889                 if (tg3_wait_macro_done(tp)) {
1890                         *resetp = 1;
1891                         return -EBUSY;
1892                 }
1893
1894                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS,
1895                              (chan * 0x2000) | 0x0200);
1896                 tg3_writephy(tp, MII_TG3_DSP_CONTROL, 0x0082);
1897                 if (tg3_wait_macro_done(tp)) {
1898                         *resetp = 1;
1899                         return -EBUSY;
1900                 }
1901
1902                 tg3_writephy(tp, MII_TG3_DSP_CONTROL, 0x0802);
1903                 if (tg3_wait_macro_done(tp)) {
1904                         *resetp = 1;
1905                         return -EBUSY;
1906                 }
1907
1908                 for (i = 0; i < 6; i += 2) {
1909                         u32 low, high;
1910
1911                         if (tg3_readphy(tp, MII_TG3_DSP_RW_PORT, &low) ||
1912                             tg3_readphy(tp, MII_TG3_DSP_RW_PORT, &high) ||
1913                             tg3_wait_macro_done(tp)) {
1914                                 *resetp = 1;
1915                                 return -EBUSY;
1916                         }
1917                         low &= 0x7fff;
1918                         high &= 0x000f;
1919                         if (low != test_pat[chan][i] ||
1920                             high != test_pat[chan][i+1]) {
1921                                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x000b);
1922                                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x4001);
1923                                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x4005);
1924
1925                                 return -EBUSY;
1926                         }
1927                 }
1928         }
1929
1930         return 0;
1931 }
1932
1933 static int tg3_phy_reset_chanpat(struct tg3 *tp)
1934 {
1935         int chan;
1936
1937         for (chan = 0; chan < 4; chan++) {
1938                 int i;
1939
1940                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS,
1941                              (chan * 0x2000) | 0x0200);
1942                 tg3_writephy(tp, MII_TG3_DSP_CONTROL, 0x0002);
1943                 for (i = 0; i < 6; i++)
1944                         tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x000);
1945                 tg3_writephy(tp, MII_TG3_DSP_CONTROL, 0x0202);
1946                 if (tg3_wait_macro_done(tp))
1947                         return -EBUSY;
1948         }
1949
1950         return 0;
1951 }
1952
1953 static int tg3_phy_reset_5703_4_5(struct tg3 *tp)
1954 {
1955         u32 reg32, phy9_orig;
1956         int retries, do_phy_reset, err;
1957
1958         retries = 10;
1959         do_phy_reset = 1;
1960         do {
1961                 if (do_phy_reset) {
1962                         err = tg3_bmcr_reset(tp);
1963                         if (err)
1964                                 return err;
1965                         do_phy_reset = 0;
1966                 }
1967
1968                 /* Disable transmitter and interrupt.  */
1969                 if (tg3_readphy(tp, MII_TG3_EXT_CTRL, &reg32))
1970                         continue;
1971
1972                 reg32 |= 0x3000;
1973                 tg3_writephy(tp, MII_TG3_EXT_CTRL, reg32);
1974
1975                 /* Set full-duplex, 1000 mbps.  */
1976                 tg3_writephy(tp, MII_BMCR,
1977                              BMCR_FULLDPLX | TG3_BMCR_SPEED1000);
1978
1979                 /* Set to master mode.  */
1980                 if (tg3_readphy(tp, MII_TG3_CTRL, &phy9_orig))
1981                         continue;
1982
1983                 tg3_writephy(tp, MII_TG3_CTRL,
1984                              (MII_TG3_CTRL_AS_MASTER |
1985                               MII_TG3_CTRL_ENABLE_AS_MASTER));
1986
1987                 err = TG3_PHY_AUXCTL_SMDSP_ENABLE(tp);
1988                 if (err)
1989                         return err;
1990
1991                 /* Block the PHY control access.  */
1992                 tg3_phydsp_write(tp, 0x8005, 0x0800);
1993
1994                 err = tg3_phy_write_and_check_testpat(tp, &do_phy_reset);
1995                 if (!err)
1996                         break;
1997         } while (--retries);
1998
1999         err = tg3_phy_reset_chanpat(tp);
2000         if (err)
2001                 return err;
2002
2003         tg3_phydsp_write(tp, 0x8005, 0x0000);
2004
2005         tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8200);
2006         tg3_writephy(tp, MII_TG3_DSP_CONTROL, 0x0000);
2007
2008         TG3_PHY_AUXCTL_SMDSP_DISABLE(tp);
2009
2010         tg3_writephy(tp, MII_TG3_CTRL, phy9_orig);
2011
2012         if (!tg3_readphy(tp, MII_TG3_EXT_CTRL, &reg32)) {
2013                 reg32 &= ~0x3000;
2014                 tg3_writephy(tp, MII_TG3_EXT_CTRL, reg32);
2015         } else if (!err)
2016                 err = -EBUSY;
2017
2018         return err;
2019 }
2020
2021 /* This will reset the tigon3 PHY if there is no valid
2022  * link unless the FORCE argument is non-zero.
2023  */
2024 static int tg3_phy_reset(struct tg3 *tp)
2025 {
2026         u32 val, cpmuctrl;
2027         int err;
2028
2029         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
2030                 val = tr32(GRC_MISC_CFG);
2031                 tw32_f(GRC_MISC_CFG, val & ~GRC_MISC_CFG_EPHY_IDDQ);
2032                 udelay(40);
2033         }
2034         err  = tg3_readphy(tp, MII_BMSR, &val);
2035         err |= tg3_readphy(tp, MII_BMSR, &val);
2036         if (err != 0)
2037                 return -EBUSY;
2038
2039         if (netif_running(tp->dev) && netif_carrier_ok(tp->dev)) {
2040                 netif_carrier_off(tp->dev);
2041                 tg3_link_report(tp);
2042         }
2043
2044         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
2045             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704 ||
2046             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) {
2047                 err = tg3_phy_reset_5703_4_5(tp);
2048                 if (err)
2049                         return err;
2050                 goto out;
2051         }
2052
2053         cpmuctrl = 0;
2054         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 &&
2055             GET_CHIP_REV(tp->pci_chip_rev_id) != CHIPREV_5784_AX) {
2056                 cpmuctrl = tr32(TG3_CPMU_CTRL);
2057                 if (cpmuctrl & CPMU_CTRL_GPHY_10MB_RXONLY)
2058                         tw32(TG3_CPMU_CTRL,
2059                              cpmuctrl & ~CPMU_CTRL_GPHY_10MB_RXONLY);
2060         }
2061
2062         err = tg3_bmcr_reset(tp);
2063         if (err)
2064                 return err;
2065
2066         if (cpmuctrl & CPMU_CTRL_GPHY_10MB_RXONLY) {
2067                 val = MII_TG3_DSP_EXP8_AEDW | MII_TG3_DSP_EXP8_REJ2MHz;
2068                 tg3_phydsp_write(tp, MII_TG3_DSP_EXP8, val);
2069
2070                 tw32(TG3_CPMU_CTRL, cpmuctrl);
2071         }
2072
2073         if (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5784_AX ||
2074             GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5761_AX) {
2075                 val = tr32(TG3_CPMU_LSPD_1000MB_CLK);
2076                 if ((val & CPMU_LSPD_1000MB_MACCLK_MASK) ==
2077                     CPMU_LSPD_1000MB_MACCLK_12_5) {
2078                         val &= ~CPMU_LSPD_1000MB_MACCLK_MASK;
2079                         udelay(40);
2080                         tw32_f(TG3_CPMU_LSPD_1000MB_CLK, val);
2081                 }
2082         }
2083
2084         if (tg3_flag(tp, 5717_PLUS) &&
2085             (tp->phy_flags & TG3_PHYFLG_MII_SERDES))
2086                 return 0;
2087
2088         tg3_phy_apply_otp(tp);
2089
2090         if (tp->phy_flags & TG3_PHYFLG_ENABLE_APD)
2091                 tg3_phy_toggle_apd(tp, true);
2092         else
2093                 tg3_phy_toggle_apd(tp, false);
2094
2095 out:
2096         if ((tp->phy_flags & TG3_PHYFLG_ADC_BUG) &&
2097             !TG3_PHY_AUXCTL_SMDSP_ENABLE(tp)) {
2098                 tg3_phydsp_write(tp, 0x201f, 0x2aaa);
2099                 tg3_phydsp_write(tp, 0x000a, 0x0323);
2100                 TG3_PHY_AUXCTL_SMDSP_DISABLE(tp);
2101         }
2102
2103         if (tp->phy_flags & TG3_PHYFLG_5704_A0_BUG) {
2104                 tg3_writephy(tp, MII_TG3_MISC_SHDW, 0x8d68);
2105                 tg3_writephy(tp, MII_TG3_MISC_SHDW, 0x8d68);
2106         }
2107
2108         if (tp->phy_flags & TG3_PHYFLG_BER_BUG) {
2109                 if (!TG3_PHY_AUXCTL_SMDSP_ENABLE(tp)) {
2110                         tg3_phydsp_write(tp, 0x000a, 0x310b);
2111                         tg3_phydsp_write(tp, 0x201f, 0x9506);
2112                         tg3_phydsp_write(tp, 0x401f, 0x14e2);
2113                         TG3_PHY_AUXCTL_SMDSP_DISABLE(tp);
2114                 }
2115         } else if (tp->phy_flags & TG3_PHYFLG_JITTER_BUG) {
2116                 if (!TG3_PHY_AUXCTL_SMDSP_ENABLE(tp)) {
2117                         tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x000a);
2118                         if (tp->phy_flags & TG3_PHYFLG_ADJUST_TRIM) {
2119                                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x110b);
2120                                 tg3_writephy(tp, MII_TG3_TEST1,
2121                                              MII_TG3_TEST1_TRIM_EN | 0x4);
2122                         } else
2123                                 tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x010b);
2124
2125                         TG3_PHY_AUXCTL_SMDSP_DISABLE(tp);
2126                 }
2127         }
2128
2129         /* Set Extended packet length bit (bit 14) on all chips that */
2130         /* support jumbo frames */
2131         if ((tp->phy_id & TG3_PHY_ID_MASK) == TG3_PHY_ID_BCM5401) {
2132                 /* Cannot do read-modify-write on 5401 */
2133                 tg3_phy_auxctl_write(tp, MII_TG3_AUXCTL_SHDWSEL_AUXCTL, 0x4c20);
2134         } else if (tg3_flag(tp, JUMBO_CAPABLE)) {
2135                 /* Set bit 14 with read-modify-write to preserve other bits */
2136                 err = tg3_phy_auxctl_read(tp,
2137                                           MII_TG3_AUXCTL_SHDWSEL_AUXCTL, &val);
2138                 if (!err)
2139                         tg3_phy_auxctl_write(tp, MII_TG3_AUXCTL_SHDWSEL_AUXCTL,
2140                                            val | MII_TG3_AUXCTL_ACTL_EXTPKTLEN);
2141         }
2142
2143         /* Set phy register 0x10 bit 0 to high fifo elasticity to support
2144          * jumbo frames transmission.
2145          */
2146         if (tg3_flag(tp, JUMBO_CAPABLE)) {
2147                 if (!tg3_readphy(tp, MII_TG3_EXT_CTRL, &val))
2148                         tg3_writephy(tp, MII_TG3_EXT_CTRL,
2149                                      val | MII_TG3_EXT_CTRL_FIFO_ELASTIC);
2150         }
2151
2152         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
2153                 /* adjust output voltage */
2154                 tg3_writephy(tp, MII_TG3_FET_PTEST, 0x12);
2155         }
2156
2157         tg3_phy_toggle_automdix(tp, 1);
2158         tg3_phy_set_wirespeed(tp);
2159         return 0;
2160 }
2161
2162 static void tg3_frob_aux_power(struct tg3 *tp)
2163 {
2164         bool need_vaux = false;
2165
2166         /* The GPIOs do something completely different on 57765. */
2167         if (!tg3_flag(tp, IS_NIC) ||
2168             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5719 ||
2169             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_57765)
2170                 return;
2171
2172         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704 ||
2173              GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714 ||
2174              GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5717 ||
2175              GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5720) &&
2176             tp->pdev_peer != tp->pdev) {
2177                 struct net_device *dev_peer;
2178
2179                 dev_peer = pci_get_drvdata(tp->pdev_peer);
2180
2181                 /* remove_one() may have been run on the peer. */
2182                 if (dev_peer) {
2183                         struct tg3 *tp_peer = netdev_priv(dev_peer);
2184
2185                         if (tg3_flag(tp_peer, INIT_COMPLETE))
2186                                 return;
2187
2188                         if (tg3_flag(tp_peer, WOL_ENABLE) ||
2189                             tg3_flag(tp_peer, ENABLE_ASF))
2190                                 need_vaux = true;
2191                 }
2192         }
2193
2194         if (tg3_flag(tp, WOL_ENABLE) || tg3_flag(tp, ENABLE_ASF))
2195                 need_vaux = true;
2196
2197         if (need_vaux) {
2198                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
2199                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) {
2200                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
2201                                     (GRC_LCLCTRL_GPIO_OE0 |
2202                                      GRC_LCLCTRL_GPIO_OE1 |
2203                                      GRC_LCLCTRL_GPIO_OE2 |
2204                                      GRC_LCLCTRL_GPIO_OUTPUT0 |
2205                                      GRC_LCLCTRL_GPIO_OUTPUT1),
2206                                     100);
2207                 } else if (tp->pdev->device == PCI_DEVICE_ID_TIGON3_5761 ||
2208                            tp->pdev->device == TG3PCI_DEVICE_TIGON3_5761S) {
2209                         /* The 5761 non-e device swaps GPIO 0 and GPIO 2. */
2210                         u32 grc_local_ctrl = GRC_LCLCTRL_GPIO_OE0 |
2211                                              GRC_LCLCTRL_GPIO_OE1 |
2212                                              GRC_LCLCTRL_GPIO_OE2 |
2213                                              GRC_LCLCTRL_GPIO_OUTPUT0 |
2214                                              GRC_LCLCTRL_GPIO_OUTPUT1 |
2215                                              tp->grc_local_ctrl;
2216                         tw32_wait_f(GRC_LOCAL_CTRL, grc_local_ctrl, 100);
2217
2218                         grc_local_ctrl |= GRC_LCLCTRL_GPIO_OUTPUT2;
2219                         tw32_wait_f(GRC_LOCAL_CTRL, grc_local_ctrl, 100);
2220
2221                         grc_local_ctrl &= ~GRC_LCLCTRL_GPIO_OUTPUT0;
2222                         tw32_wait_f(GRC_LOCAL_CTRL, grc_local_ctrl, 100);
2223                 } else {
2224                         u32 no_gpio2;
2225                         u32 grc_local_ctrl = 0;
2226
2227                         /* Workaround to prevent overdrawing Amps. */
2228                         if (GET_ASIC_REV(tp->pci_chip_rev_id) ==
2229                             ASIC_REV_5714) {
2230                                 grc_local_ctrl |= GRC_LCLCTRL_GPIO_OE3;
2231                                 tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
2232                                             grc_local_ctrl, 100);
2233                         }
2234
2235                         /* On 5753 and variants, GPIO2 cannot be used. */
2236                         no_gpio2 = tp->nic_sram_data_cfg &
2237                                     NIC_SRAM_DATA_CFG_NO_GPIO2;
2238
2239                         grc_local_ctrl |= GRC_LCLCTRL_GPIO_OE0 |
2240                                          GRC_LCLCTRL_GPIO_OE1 |
2241                                          GRC_LCLCTRL_GPIO_OE2 |
2242                                          GRC_LCLCTRL_GPIO_OUTPUT1 |
2243                                          GRC_LCLCTRL_GPIO_OUTPUT2;
2244                         if (no_gpio2) {
2245                                 grc_local_ctrl &= ~(GRC_LCLCTRL_GPIO_OE2 |
2246                                                     GRC_LCLCTRL_GPIO_OUTPUT2);
2247                         }
2248                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
2249                                                     grc_local_ctrl, 100);
2250
2251                         grc_local_ctrl |= GRC_LCLCTRL_GPIO_OUTPUT0;
2252
2253                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
2254                                                     grc_local_ctrl, 100);
2255
2256                         if (!no_gpio2) {
2257                                 grc_local_ctrl &= ~GRC_LCLCTRL_GPIO_OUTPUT2;
2258                                 tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
2259                                             grc_local_ctrl, 100);
2260                         }
2261                 }
2262         } else {
2263                 if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700 &&
2264                     GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5701) {
2265                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
2266                                     (GRC_LCLCTRL_GPIO_OE1 |
2267                                      GRC_LCLCTRL_GPIO_OUTPUT1), 100);
2268
2269                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
2270                                     GRC_LCLCTRL_GPIO_OE1, 100);
2271
2272                         tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl |
2273                                     (GRC_LCLCTRL_GPIO_OE1 |
2274                                      GRC_LCLCTRL_GPIO_OUTPUT1), 100);
2275                 }
2276         }
2277 }
2278
2279 static int tg3_5700_link_polarity(struct tg3 *tp, u32 speed)
2280 {
2281         if (tp->led_ctrl == LED_CTRL_MODE_PHY_2)
2282                 return 1;
2283         else if ((tp->phy_id & TG3_PHY_ID_MASK) == TG3_PHY_ID_BCM5411) {
2284                 if (speed != SPEED_10)
2285                         return 1;
2286         } else if (speed == SPEED_10)
2287                 return 1;
2288
2289         return 0;
2290 }
2291
2292 static int tg3_setup_phy(struct tg3 *, int);
2293
2294 #define RESET_KIND_SHUTDOWN     0
2295 #define RESET_KIND_INIT         1
2296 #define RESET_KIND_SUSPEND      2
2297
2298 static void tg3_write_sig_post_reset(struct tg3 *, int);
2299 static int tg3_halt_cpu(struct tg3 *, u32);
2300
2301 static void tg3_power_down_phy(struct tg3 *tp, bool do_low_power)
2302 {
2303         u32 val;
2304
2305         if (tp->phy_flags & TG3_PHYFLG_PHY_SERDES) {
2306                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) {
2307                         u32 sg_dig_ctrl = tr32(SG_DIG_CTRL);
2308                         u32 serdes_cfg = tr32(MAC_SERDES_CFG);
2309
2310                         sg_dig_ctrl |=
2311                                 SG_DIG_USING_HW_AUTONEG | SG_DIG_SOFT_RESET;
2312                         tw32(SG_DIG_CTRL, sg_dig_ctrl);
2313                         tw32(MAC_SERDES_CFG, serdes_cfg | (1 << 15));
2314                 }
2315                 return;
2316         }
2317
2318         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
2319                 tg3_bmcr_reset(tp);
2320                 val = tr32(GRC_MISC_CFG);
2321                 tw32_f(GRC_MISC_CFG, val | GRC_MISC_CFG_EPHY_IDDQ);
2322                 udelay(40);
2323                 return;
2324         } else if (tp->phy_flags & TG3_PHYFLG_IS_FET) {
2325                 u32 phytest;
2326                 if (!tg3_readphy(tp, MII_TG3_FET_TEST, &phytest)) {
2327                         u32 phy;
2328
2329                         tg3_writephy(tp, MII_ADVERTISE, 0);
2330                         tg3_writephy(tp, MII_BMCR,
2331                                      BMCR_ANENABLE | BMCR_ANRESTART);
2332
2333                         tg3_writephy(tp, MII_TG3_FET_TEST,
2334                                      phytest | MII_TG3_FET_SHADOW_EN);
2335                         if (!tg3_readphy(tp, MII_TG3_FET_SHDW_AUXMODE4, &phy)) {
2336                                 phy |= MII_TG3_FET_SHDW_AUXMODE4_SBPD;
2337                                 tg3_writephy(tp,
2338                                              MII_TG3_FET_SHDW_AUXMODE4,
2339                                              phy);
2340                         }
2341                         tg3_writephy(tp, MII_TG3_FET_TEST, phytest);
2342                 }
2343                 return;
2344         } else if (do_low_power) {
2345                 tg3_writephy(tp, MII_TG3_EXT_CTRL,
2346                              MII_TG3_EXT_CTRL_FORCE_LED_OFF);
2347
2348                 val = MII_TG3_AUXCTL_PCTL_100TX_LPWR |
2349                       MII_TG3_AUXCTL_PCTL_SPR_ISOLATE |
2350                       MII_TG3_AUXCTL_PCTL_VREG_11V;
2351                 tg3_phy_auxctl_write(tp, MII_TG3_AUXCTL_SHDWSEL_PWRCTL, val);
2352         }
2353
2354         /* The PHY should not be powered down on some chips because
2355          * of bugs.
2356          */
2357         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
2358             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704 ||
2359             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5780 &&
2360              (tp->phy_flags & TG3_PHYFLG_MII_SERDES)))
2361                 return;
2362
2363         if (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5784_AX ||
2364             GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5761_AX) {
2365                 val = tr32(TG3_CPMU_LSPD_1000MB_CLK);
2366                 val &= ~CPMU_LSPD_1000MB_MACCLK_MASK;
2367                 val |= CPMU_LSPD_1000MB_MACCLK_12_5;
2368                 tw32_f(TG3_CPMU_LSPD_1000MB_CLK, val);
2369         }
2370
2371         tg3_writephy(tp, MII_BMCR, BMCR_PDOWN);
2372 }
2373
2374 /* tp->lock is held. */
2375 static int tg3_nvram_lock(struct tg3 *tp)
2376 {
2377         if (tg3_flag(tp, NVRAM)) {
2378                 int i;
2379
2380                 if (tp->nvram_lock_cnt == 0) {
2381                         tw32(NVRAM_SWARB, SWARB_REQ_SET1);
2382                         for (i = 0; i < 8000; i++) {
2383                                 if (tr32(NVRAM_SWARB) & SWARB_GNT1)
2384                                         break;
2385                                 udelay(20);
2386                         }
2387                         if (i == 8000) {
2388                                 tw32(NVRAM_SWARB, SWARB_REQ_CLR1);
2389                                 return -ENODEV;
2390                         }
2391                 }
2392                 tp->nvram_lock_cnt++;
2393         }
2394         return 0;
2395 }
2396
2397 /* tp->lock is held. */
2398 static void tg3_nvram_unlock(struct tg3 *tp)
2399 {
2400         if (tg3_flag(tp, NVRAM)) {
2401                 if (tp->nvram_lock_cnt > 0)
2402                         tp->nvram_lock_cnt--;
2403                 if (tp->nvram_lock_cnt == 0)
2404                         tw32_f(NVRAM_SWARB, SWARB_REQ_CLR1);
2405         }
2406 }
2407
2408 /* tp->lock is held. */
2409 static void tg3_enable_nvram_access(struct tg3 *tp)
2410 {
2411         if (tg3_flag(tp, 5750_PLUS) && !tg3_flag(tp, PROTECTED_NVRAM)) {
2412                 u32 nvaccess = tr32(NVRAM_ACCESS);
2413
2414                 tw32(NVRAM_ACCESS, nvaccess | ACCESS_ENABLE);
2415         }
2416 }
2417
2418 /* tp->lock is held. */
2419 static void tg3_disable_nvram_access(struct tg3 *tp)
2420 {
2421         if (tg3_flag(tp, 5750_PLUS) && !tg3_flag(tp, PROTECTED_NVRAM)) {
2422                 u32 nvaccess = tr32(NVRAM_ACCESS);
2423
2424                 tw32(NVRAM_ACCESS, nvaccess & ~ACCESS_ENABLE);
2425         }
2426 }
2427
2428 static int tg3_nvram_read_using_eeprom(struct tg3 *tp,
2429                                         u32 offset, u32 *val)
2430 {
2431         u32 tmp;
2432         int i;
2433
2434         if (offset > EEPROM_ADDR_ADDR_MASK || (offset % 4) != 0)
2435                 return -EINVAL;
2436
2437         tmp = tr32(GRC_EEPROM_ADDR) & ~(EEPROM_ADDR_ADDR_MASK |
2438                                         EEPROM_ADDR_DEVID_MASK |
2439                                         EEPROM_ADDR_READ);
2440         tw32(GRC_EEPROM_ADDR,
2441              tmp |
2442              (0 << EEPROM_ADDR_DEVID_SHIFT) |
2443              ((offset << EEPROM_ADDR_ADDR_SHIFT) &
2444               EEPROM_ADDR_ADDR_MASK) |
2445              EEPROM_ADDR_READ | EEPROM_ADDR_START);
2446
2447         for (i = 0; i < 1000; i++) {
2448                 tmp = tr32(GRC_EEPROM_ADDR);
2449
2450                 if (tmp & EEPROM_ADDR_COMPLETE)
2451                         break;
2452                 msleep(1);
2453         }
2454         if (!(tmp & EEPROM_ADDR_COMPLETE))
2455                 return -EBUSY;
2456
2457         tmp = tr32(GRC_EEPROM_DATA);
2458
2459         /*
2460          * The data will always be opposite the native endian
2461          * format.  Perform a blind byteswap to compensate.
2462          */
2463         *val = swab32(tmp);
2464
2465         return 0;
2466 }
2467
2468 #define NVRAM_CMD_TIMEOUT 10000
2469
2470 static int tg3_nvram_exec_cmd(struct tg3 *tp, u32 nvram_cmd)
2471 {
2472         int i;
2473
2474         tw32(NVRAM_CMD, nvram_cmd);
2475         for (i = 0; i < NVRAM_CMD_TIMEOUT; i++) {
2476                 udelay(10);
2477                 if (tr32(NVRAM_CMD) & NVRAM_CMD_DONE) {
2478                         udelay(10);
2479                         break;
2480                 }
2481         }
2482
2483         if (i == NVRAM_CMD_TIMEOUT)
2484                 return -EBUSY;
2485
2486         return 0;
2487 }
2488
2489 static u32 tg3_nvram_phys_addr(struct tg3 *tp, u32 addr)
2490 {
2491         if (tg3_flag(tp, NVRAM) &&
2492             tg3_flag(tp, NVRAM_BUFFERED) &&
2493             tg3_flag(tp, FLASH) &&
2494             !tg3_flag(tp, NO_NVRAM_ADDR_TRANS) &&
2495             (tp->nvram_jedecnum == JEDEC_ATMEL))
2496
2497                 addr = ((addr / tp->nvram_pagesize) <<
2498                         ATMEL_AT45DB0X1B_PAGE_POS) +
2499                        (addr % tp->nvram_pagesize);
2500
2501         return addr;
2502 }
2503
2504 static u32 tg3_nvram_logical_addr(struct tg3 *tp, u32 addr)
2505 {
2506         if (tg3_flag(tp, NVRAM) &&
2507             tg3_flag(tp, NVRAM_BUFFERED) &&
2508             tg3_flag(tp, FLASH) &&
2509             !tg3_flag(tp, NO_NVRAM_ADDR_TRANS) &&
2510             (tp->nvram_jedecnum == JEDEC_ATMEL))
2511
2512                 addr = ((addr >> ATMEL_AT45DB0X1B_PAGE_POS) *
2513                         tp->nvram_pagesize) +
2514                        (addr & ((1 << ATMEL_AT45DB0X1B_PAGE_POS) - 1));
2515
2516         return addr;
2517 }
2518
2519 /* NOTE: Data read in from NVRAM is byteswapped according to
2520  * the byteswapping settings for all other register accesses.
2521  * tg3 devices are BE devices, so on a BE machine, the data
2522  * returned will be exactly as it is seen in NVRAM.  On a LE
2523  * machine, the 32-bit value will be byteswapped.
2524  */
2525 static int tg3_nvram_read(struct tg3 *tp, u32 offset, u32 *val)
2526 {
2527         int ret;
2528
2529         if (!tg3_flag(tp, NVRAM))
2530                 return tg3_nvram_read_using_eeprom(tp, offset, val);
2531
2532         offset = tg3_nvram_phys_addr(tp, offset);
2533
2534         if (offset > NVRAM_ADDR_MSK)
2535                 return -EINVAL;
2536
2537         ret = tg3_nvram_lock(tp);
2538         if (ret)
2539                 return ret;
2540
2541         tg3_enable_nvram_access(tp);
2542
2543         tw32(NVRAM_ADDR, offset);
2544         ret = tg3_nvram_exec_cmd(tp, NVRAM_CMD_RD | NVRAM_CMD_GO |
2545                 NVRAM_CMD_FIRST | NVRAM_CMD_LAST | NVRAM_CMD_DONE);
2546
2547         if (ret == 0)
2548                 *val = tr32(NVRAM_RDDATA);
2549
2550         tg3_disable_nvram_access(tp);
2551
2552         tg3_nvram_unlock(tp);
2553
2554         return ret;
2555 }
2556
2557 /* Ensures NVRAM data is in bytestream format. */
2558 static int tg3_nvram_read_be32(struct tg3 *tp, u32 offset, __be32 *val)
2559 {
2560         u32 v;
2561         int res = tg3_nvram_read(tp, offset, &v);
2562         if (!res)
2563                 *val = cpu_to_be32(v);
2564         return res;
2565 }
2566
2567 /* tp->lock is held. */
2568 static void __tg3_set_mac_addr(struct tg3 *tp, int skip_mac_1)
2569 {
2570         u32 addr_high, addr_low;
2571         int i;
2572
2573         addr_high = ((tp->dev->dev_addr[0] << 8) |
2574                      tp->dev->dev_addr[1]);
2575         addr_low = ((tp->dev->dev_addr[2] << 24) |
2576                     (tp->dev->dev_addr[3] << 16) |
2577                     (tp->dev->dev_addr[4] <<  8) |
2578                     (tp->dev->dev_addr[5] <<  0));
2579         for (i = 0; i < 4; i++) {
2580                 if (i == 1 && skip_mac_1)
2581                         continue;
2582                 tw32(MAC_ADDR_0_HIGH + (i * 8), addr_high);
2583                 tw32(MAC_ADDR_0_LOW + (i * 8), addr_low);
2584         }
2585
2586         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
2587             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) {
2588                 for (i = 0; i < 12; i++) {
2589                         tw32(MAC_EXTADDR_0_HIGH + (i * 8), addr_high);
2590                         tw32(MAC_EXTADDR_0_LOW + (i * 8), addr_low);
2591                 }
2592         }
2593
2594         addr_high = (tp->dev->dev_addr[0] +
2595                      tp->dev->dev_addr[1] +
2596                      tp->dev->dev_addr[2] +
2597                      tp->dev->dev_addr[3] +
2598                      tp->dev->dev_addr[4] +
2599                      tp->dev->dev_addr[5]) &
2600                 TX_BACKOFF_SEED_MASK;
2601         tw32(MAC_TX_BACKOFF_SEED, addr_high);
2602 }
2603
2604 static void tg3_enable_register_access(struct tg3 *tp)
2605 {
2606         /*
2607          * Make sure register accesses (indirect or otherwise) will function
2608          * correctly.
2609          */
2610         pci_write_config_dword(tp->pdev,
2611                                TG3PCI_MISC_HOST_CTRL, tp->misc_host_ctrl);
2612 }
2613
2614 static int tg3_power_up(struct tg3 *tp)
2615 {
2616         tg3_enable_register_access(tp);
2617
2618         pci_set_power_state(tp->pdev, PCI_D0);
2619
2620         /* Switch out of Vaux if it is a NIC */
2621         if (tg3_flag(tp, IS_NIC))
2622                 tw32_wait_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl, 100);
2623
2624         return 0;
2625 }
2626
2627 static int tg3_power_down_prepare(struct tg3 *tp)
2628 {
2629         u32 misc_host_ctrl;
2630         bool device_should_wake, do_low_power;
2631
2632         tg3_enable_register_access(tp);
2633
2634         /* Restore the CLKREQ setting. */
2635         if (tg3_flag(tp, CLKREQ_BUG)) {
2636                 u16 lnkctl;
2637
2638                 pci_read_config_word(tp->pdev,
2639                                      tp->pcie_cap + PCI_EXP_LNKCTL,
2640                                      &lnkctl);
2641                 lnkctl |= PCI_EXP_LNKCTL_CLKREQ_EN;
2642                 pci_write_config_word(tp->pdev,
2643                                       tp->pcie_cap + PCI_EXP_LNKCTL,
2644                                       lnkctl);
2645         }
2646
2647         misc_host_ctrl = tr32(TG3PCI_MISC_HOST_CTRL);
2648         tw32(TG3PCI_MISC_HOST_CTRL,
2649              misc_host_ctrl | MISC_HOST_CTRL_MASK_PCI_INT);
2650
2651         device_should_wake = device_may_wakeup(&tp->pdev->dev) &&
2652                              tg3_flag(tp, WOL_ENABLE);
2653
2654         if (tg3_flag(tp, USE_PHYLIB)) {
2655                 do_low_power = false;
2656                 if ((tp->phy_flags & TG3_PHYFLG_IS_CONNECTED) &&
2657                     !(tp->phy_flags & TG3_PHYFLG_IS_LOW_POWER)) {
2658                         struct phy_device *phydev;
2659                         u32 phyid, advertising;
2660
2661                         phydev = tp->mdio_bus->phy_map[TG3_PHY_MII_ADDR];
2662
2663                         tp->phy_flags |= TG3_PHYFLG_IS_LOW_POWER;
2664
2665                         tp->link_config.orig_speed = phydev->speed;
2666                         tp->link_config.orig_duplex = phydev->duplex;
2667                         tp->link_config.orig_autoneg = phydev->autoneg;
2668                         tp->link_config.orig_advertising = phydev->advertising;
2669
2670                         advertising = ADVERTISED_TP |
2671                                       ADVERTISED_Pause |
2672                                       ADVERTISED_Autoneg |
2673                                       ADVERTISED_10baseT_Half;
2674
2675                         if (tg3_flag(tp, ENABLE_ASF) || device_should_wake) {
2676                                 if (tg3_flag(tp, WOL_SPEED_100MB))
2677                                         advertising |=
2678                                                 ADVERTISED_100baseT_Half |
2679                                                 ADVERTISED_100baseT_Full |
2680                                                 ADVERTISED_10baseT_Full;
2681                                 else
2682                                         advertising |= ADVERTISED_10baseT_Full;
2683                         }
2684
2685                         phydev->advertising = advertising;
2686
2687                         phy_start_aneg(phydev);
2688
2689                         phyid = phydev->drv->phy_id & phydev->drv->phy_id_mask;
2690                         if (phyid != PHY_ID_BCMAC131) {
2691                                 phyid &= PHY_BCM_OUI_MASK;
2692                                 if (phyid == PHY_BCM_OUI_1 ||
2693                                     phyid == PHY_BCM_OUI_2 ||
2694                                     phyid == PHY_BCM_OUI_3)
2695                                         do_low_power = true;
2696                         }
2697                 }
2698         } else {
2699                 do_low_power = true;
2700
2701                 if (!(tp->phy_flags & TG3_PHYFLG_IS_LOW_POWER)) {
2702                         tp->phy_flags |= TG3_PHYFLG_IS_LOW_POWER;
2703                         tp->link_config.orig_speed = tp->link_config.speed;
2704                         tp->link_config.orig_duplex = tp->link_config.duplex;
2705                         tp->link_config.orig_autoneg = tp->link_config.autoneg;
2706                 }
2707
2708                 if (!(tp->phy_flags & TG3_PHYFLG_ANY_SERDES)) {
2709                         tp->link_config.speed = SPEED_10;
2710                         tp->link_config.duplex = DUPLEX_HALF;
2711                         tp->link_config.autoneg = AUTONEG_ENABLE;
2712                         tg3_setup_phy(tp, 0);
2713                 }
2714         }
2715
2716         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
2717                 u32 val;
2718
2719                 val = tr32(GRC_VCPU_EXT_CTRL);
2720                 tw32(GRC_VCPU_EXT_CTRL, val | GRC_VCPU_EXT_CTRL_DISABLE_WOL);
2721         } else if (!tg3_flag(tp, ENABLE_ASF)) {
2722                 int i;
2723                 u32 val;
2724
2725                 for (i = 0; i < 200; i++) {
2726                         tg3_read_mem(tp, NIC_SRAM_FW_ASF_STATUS_MBOX, &val);
2727                         if (val == ~NIC_SRAM_FIRMWARE_MBOX_MAGIC1)
2728                                 break;
2729                         msleep(1);
2730                 }
2731         }
2732         if (tg3_flag(tp, WOL_CAP))
2733                 tg3_write_mem(tp, NIC_SRAM_WOL_MBOX, WOL_SIGNATURE |
2734                                                      WOL_DRV_STATE_SHUTDOWN |
2735                                                      WOL_DRV_WOL |
2736                                                      WOL_SET_MAGIC_PKT);
2737
2738         if (device_should_wake) {
2739                 u32 mac_mode;
2740
2741                 if (!(tp->phy_flags & TG3_PHYFLG_PHY_SERDES)) {
2742                         if (do_low_power &&
2743                             !(tp->phy_flags & TG3_PHYFLG_IS_FET)) {
2744                                 tg3_phy_auxctl_write(tp,
2745                                                MII_TG3_AUXCTL_SHDWSEL_PWRCTL,
2746                                                MII_TG3_AUXCTL_PCTL_WOL_EN |
2747                                                MII_TG3_AUXCTL_PCTL_100TX_LPWR |
2748                                                MII_TG3_AUXCTL_PCTL_CL_AB_TXDAC);
2749                                 udelay(40);
2750                         }
2751
2752                         if (tp->phy_flags & TG3_PHYFLG_MII_SERDES)
2753                                 mac_mode = MAC_MODE_PORT_MODE_GMII;
2754                         else
2755                                 mac_mode = MAC_MODE_PORT_MODE_MII;
2756
2757                         mac_mode |= tp->mac_mode & MAC_MODE_LINK_POLARITY;
2758                         if (GET_ASIC_REV(tp->pci_chip_rev_id) ==
2759                             ASIC_REV_5700) {
2760                                 u32 speed = tg3_flag(tp, WOL_SPEED_100MB) ?
2761                                              SPEED_100 : SPEED_10;
2762                                 if (tg3_5700_link_polarity(tp, speed))
2763                                         mac_mode |= MAC_MODE_LINK_POLARITY;
2764                                 else
2765                                         mac_mode &= ~MAC_MODE_LINK_POLARITY;
2766                         }
2767                 } else {
2768                         mac_mode = MAC_MODE_PORT_MODE_TBI;
2769                 }
2770
2771                 if (!tg3_flag(tp, 5750_PLUS))
2772                         tw32(MAC_LED_CTRL, tp->led_ctrl);
2773
2774                 mac_mode |= MAC_MODE_MAGIC_PKT_ENABLE;
2775                 if ((tg3_flag(tp, 5705_PLUS) && !tg3_flag(tp, 5780_CLASS)) &&
2776                     (tg3_flag(tp, ENABLE_ASF) || tg3_flag(tp, ENABLE_APE)))
2777                         mac_mode |= MAC_MODE_KEEP_FRAME_IN_WOL;
2778
2779                 if (tg3_flag(tp, ENABLE_APE))
2780                         mac_mode |= MAC_MODE_APE_TX_EN |
2781                                     MAC_MODE_APE_RX_EN |
2782                                     MAC_MODE_TDE_ENABLE;
2783
2784                 tw32_f(MAC_MODE, mac_mode);
2785                 udelay(100);
2786
2787                 tw32_f(MAC_RX_MODE, RX_MODE_ENABLE);
2788                 udelay(10);
2789         }
2790
2791         if (!tg3_flag(tp, WOL_SPEED_100MB) &&
2792             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
2793              GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701)) {
2794                 u32 base_val;
2795
2796                 base_val = tp->pci_clock_ctrl;
2797                 base_val |= (CLOCK_CTRL_RXCLK_DISABLE |
2798                              CLOCK_CTRL_TXCLK_DISABLE);
2799
2800                 tw32_wait_f(TG3PCI_CLOCK_CTRL, base_val | CLOCK_CTRL_ALTCLK |
2801                             CLOCK_CTRL_PWRDOWN_PLL133, 40);
2802         } else if (tg3_flag(tp, 5780_CLASS) ||
2803                    tg3_flag(tp, CPMU_PRESENT) ||
2804                    (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906)) {
2805                 /* do nothing */
2806         } else if (!(tg3_flag(tp, 5750_PLUS) && tg3_flag(tp, ENABLE_ASF))) {
2807                 u32 newbits1, newbits2;
2808
2809                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
2810                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) {
2811                         newbits1 = (CLOCK_CTRL_RXCLK_DISABLE |
2812                                     CLOCK_CTRL_TXCLK_DISABLE |
2813                                     CLOCK_CTRL_ALTCLK);
2814                         newbits2 = newbits1 | CLOCK_CTRL_44MHZ_CORE;
2815                 } else if (tg3_flag(tp, 5705_PLUS)) {
2816                         newbits1 = CLOCK_CTRL_625_CORE;
2817                         newbits2 = newbits1 | CLOCK_CTRL_ALTCLK;
2818                 } else {
2819                         newbits1 = CLOCK_CTRL_ALTCLK;
2820                         newbits2 = newbits1 | CLOCK_CTRL_44MHZ_CORE;
2821                 }
2822
2823                 tw32_wait_f(TG3PCI_CLOCK_CTRL, tp->pci_clock_ctrl | newbits1,
2824                             40);
2825
2826                 tw32_wait_f(TG3PCI_CLOCK_CTRL, tp->pci_clock_ctrl | newbits2,
2827                             40);
2828
2829                 if (!tg3_flag(tp, 5705_PLUS)) {
2830                         u32 newbits3;
2831
2832                         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
2833                             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) {
2834                                 newbits3 = (CLOCK_CTRL_RXCLK_DISABLE |
2835                                             CLOCK_CTRL_TXCLK_DISABLE |
2836                                             CLOCK_CTRL_44MHZ_CORE);
2837                         } else {
2838                                 newbits3 = CLOCK_CTRL_44MHZ_CORE;
2839                         }
2840
2841                         tw32_wait_f(TG3PCI_CLOCK_CTRL,
2842                                     tp->pci_clock_ctrl | newbits3, 40);
2843                 }
2844         }
2845
2846         if (!(device_should_wake) && !tg3_flag(tp, ENABLE_ASF))
2847                 tg3_power_down_phy(tp, do_low_power);
2848
2849         tg3_frob_aux_power(tp);
2850
2851         /* Workaround for unstable PLL clock */
2852         if ((GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5750_AX) ||
2853             (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5750_BX)) {
2854                 u32 val = tr32(0x7d00);
2855
2856                 val &= ~((1 << 16) | (1 << 4) | (1 << 2) | (1 << 1) | 1);
2857                 tw32(0x7d00, val);
2858                 if (!tg3_flag(tp, ENABLE_ASF)) {
2859                         int err;
2860
2861                         err = tg3_nvram_lock(tp);
2862                         tg3_halt_cpu(tp, RX_CPU_BASE);
2863                         if (!err)
2864                                 tg3_nvram_unlock(tp);
2865                 }
2866         }
2867
2868         tg3_write_sig_post_reset(tp, RESET_KIND_SHUTDOWN);
2869
2870         return 0;
2871 }
2872
2873 static void tg3_power_down(struct tg3 *tp)
2874 {
2875         tg3_power_down_prepare(tp);
2876
2877         pci_wake_from_d3(tp->pdev, tg3_flag(tp, WOL_ENABLE));
2878         pci_set_power_state(tp->pdev, PCI_D3hot);
2879 }
2880
2881 static void tg3_aux_stat_to_speed_duplex(struct tg3 *tp, u32 val, u16 *speed, u8 *duplex)
2882 {
2883         switch (val & MII_TG3_AUX_STAT_SPDMASK) {
2884         case MII_TG3_AUX_STAT_10HALF:
2885                 *speed = SPEED_10;
2886                 *duplex = DUPLEX_HALF;
2887                 break;
2888
2889         case MII_TG3_AUX_STAT_10FULL:
2890                 *speed = SPEED_10;
2891                 *duplex = DUPLEX_FULL;
2892                 break;
2893
2894         case MII_TG3_AUX_STAT_100HALF:
2895                 *speed = SPEED_100;
2896                 *duplex = DUPLEX_HALF;
2897                 break;
2898
2899         case MII_TG3_AUX_STAT_100FULL:
2900                 *speed = SPEED_100;
2901                 *duplex = DUPLEX_FULL;
2902                 break;
2903
2904         case MII_TG3_AUX_STAT_1000HALF:
2905                 *speed = SPEED_1000;
2906                 *duplex = DUPLEX_HALF;
2907                 break;
2908
2909         case MII_TG3_AUX_STAT_1000FULL:
2910                 *speed = SPEED_1000;
2911                 *duplex = DUPLEX_FULL;
2912                 break;
2913
2914         default:
2915                 if (tp->phy_flags & TG3_PHYFLG_IS_FET) {
2916                         *speed = (val & MII_TG3_AUX_STAT_100) ? SPEED_100 :
2917                                  SPEED_10;
2918                         *duplex = (val & MII_TG3_AUX_STAT_FULL) ? DUPLEX_FULL :
2919                                   DUPLEX_HALF;
2920                         break;
2921                 }
2922                 *speed = SPEED_INVALID;
2923                 *duplex = DUPLEX_INVALID;
2924                 break;
2925         }
2926 }
2927
2928 static void tg3_phy_copper_begin(struct tg3 *tp)
2929 {
2930         u32 new_adv;
2931         int i;
2932
2933         if (tp->phy_flags & TG3_PHYFLG_IS_LOW_POWER) {
2934                 /* Entering low power mode.  Disable gigabit and
2935                  * 100baseT advertisements.
2936                  */
2937                 tg3_writephy(tp, MII_TG3_CTRL, 0);
2938
2939                 new_adv = (ADVERTISE_10HALF | ADVERTISE_10FULL |
2940                            ADVERTISE_CSMA | ADVERTISE_PAUSE_CAP);
2941                 if (tg3_flag(tp, WOL_SPEED_100MB))
2942                         new_adv |= (ADVERTISE_100HALF | ADVERTISE_100FULL);
2943
2944                 tg3_writephy(tp, MII_ADVERTISE, new_adv);
2945         } else if (tp->link_config.speed == SPEED_INVALID) {
2946                 if (tp->phy_flags & TG3_PHYFLG_10_100_ONLY)
2947                         tp->link_config.advertising &=
2948                                 ~(ADVERTISED_1000baseT_Half |
2949                                   ADVERTISED_1000baseT_Full);
2950
2951                 new_adv = ADVERTISE_CSMA;
2952                 if (tp->link_config.advertising & ADVERTISED_10baseT_Half)
2953                         new_adv |= ADVERTISE_10HALF;
2954                 if (tp->link_config.advertising & ADVERTISED_10baseT_Full)
2955                         new_adv |= ADVERTISE_10FULL;
2956                 if (tp->link_config.advertising & ADVERTISED_100baseT_Half)
2957                         new_adv |= ADVERTISE_100HALF;
2958                 if (tp->link_config.advertising & ADVERTISED_100baseT_Full)
2959                         new_adv |= ADVERTISE_100FULL;
2960
2961                 new_adv |= tg3_advert_flowctrl_1000T(tp->link_config.flowctrl);
2962
2963                 tg3_writephy(tp, MII_ADVERTISE, new_adv);
2964
2965                 if (tp->link_config.advertising &
2966                     (ADVERTISED_1000baseT_Half | ADVERTISED_1000baseT_Full)) {
2967                         new_adv = 0;
2968                         if (tp->link_config.advertising & ADVERTISED_1000baseT_Half)
2969                                 new_adv |= MII_TG3_CTRL_ADV_1000_HALF;
2970                         if (tp->link_config.advertising & ADVERTISED_1000baseT_Full)
2971                                 new_adv |= MII_TG3_CTRL_ADV_1000_FULL;
2972                         if (!(tp->phy_flags & TG3_PHYFLG_10_100_ONLY) &&
2973                             (tp->pci_chip_rev_id == CHIPREV_ID_5701_A0 ||
2974                              tp->pci_chip_rev_id == CHIPREV_ID_5701_B0))
2975                                 new_adv |= (MII_TG3_CTRL_AS_MASTER |
2976                                             MII_TG3_CTRL_ENABLE_AS_MASTER);
2977                         tg3_writephy(tp, MII_TG3_CTRL, new_adv);
2978                 } else {
2979                         tg3_writephy(tp, MII_TG3_CTRL, 0);
2980                 }
2981         } else {
2982                 new_adv = tg3_advert_flowctrl_1000T(tp->link_config.flowctrl);
2983                 new_adv |= ADVERTISE_CSMA;
2984
2985                 /* Asking for a specific link mode. */
2986                 if (tp->link_config.speed == SPEED_1000) {
2987                         tg3_writephy(tp, MII_ADVERTISE, new_adv);
2988
2989                         if (tp->link_config.duplex == DUPLEX_FULL)
2990                                 new_adv = MII_TG3_CTRL_ADV_1000_FULL;
2991                         else
2992                                 new_adv = MII_TG3_CTRL_ADV_1000_HALF;
2993                         if (tp->pci_chip_rev_id == CHIPREV_ID_5701_A0 ||
2994                             tp->pci_chip_rev_id == CHIPREV_ID_5701_B0)
2995                                 new_adv |= (MII_TG3_CTRL_AS_MASTER |
2996                                             MII_TG3_CTRL_ENABLE_AS_MASTER);
2997                 } else {
2998                         if (tp->link_config.speed == SPEED_100) {
2999                                 if (tp->link_config.duplex == DUPLEX_FULL)
3000                                         new_adv |= ADVERTISE_100FULL;
3001                                 else
3002                                         new_adv |= ADVERTISE_100HALF;
3003                         } else {
3004                                 if (tp->link_config.duplex == DUPLEX_FULL)
3005                                         new_adv |= ADVERTISE_10FULL;
3006                                 else
3007                                         new_adv |= ADVERTISE_10HALF;
3008                         }
3009                         tg3_writephy(tp, MII_ADVERTISE, new_adv);
3010
3011                         new_adv = 0;
3012                 }
3013
3014                 tg3_writephy(tp, MII_TG3_CTRL, new_adv);
3015         }
3016
3017         if (tp->phy_flags & TG3_PHYFLG_EEE_CAP) {
3018                 u32 val;
3019
3020                 tw32(TG3_CPMU_EEE_MODE,
3021                      tr32(TG3_CPMU_EEE_MODE) & ~TG3_CPMU_EEEMD_LPI_ENABLE);
3022
3023                 TG3_PHY_AUXCTL_SMDSP_ENABLE(tp);
3024
3025                 switch (GET_ASIC_REV(tp->pci_chip_rev_id)) {
3026                 case ASIC_REV_5717:
3027                 case ASIC_REV_57765:
3028                         if (!tg3_phydsp_read(tp, MII_TG3_DSP_CH34TP2, &val))
3029                                 tg3_phydsp_write(tp, MII_TG3_DSP_CH34TP2, val |
3030                                                  MII_TG3_DSP_CH34TP2_HIBW01);
3031                         /* Fall through */
3032                 case ASIC_REV_5719:
3033                         val = MII_TG3_DSP_TAP26_ALNOKO |
3034                               MII_TG3_DSP_TAP26_RMRXSTO |
3035                               MII_TG3_DSP_TAP26_OPCSINPT;
3036                         tg3_phydsp_write(tp, MII_TG3_DSP_TAP26, val);
3037                 }
3038
3039                 val = 0;
3040                 if (tp->link_config.autoneg == AUTONEG_ENABLE) {
3041                         /* Advertise 100-BaseTX EEE ability */
3042                         if (tp->link_config.advertising &
3043                             ADVERTISED_100baseT_Full)
3044                                 val |= MDIO_AN_EEE_ADV_100TX;
3045                         /* Advertise 1000-BaseT EEE ability */
3046                         if (tp->link_config.advertising &
3047                             ADVERTISED_1000baseT_Full)
3048                                 val |= MDIO_AN_EEE_ADV_1000T;
3049                 }
3050                 tg3_phy_cl45_write(tp, MDIO_MMD_AN, MDIO_AN_EEE_ADV, val);
3051
3052                 TG3_PHY_AUXCTL_SMDSP_DISABLE(tp);
3053         }
3054
3055         if (tp->link_config.autoneg == AUTONEG_DISABLE &&
3056             tp->link_config.speed != SPEED_INVALID) {
3057                 u32 bmcr, orig_bmcr;
3058
3059                 tp->link_config.active_speed = tp->link_config.speed;
3060                 tp->link_config.active_duplex = tp->link_config.duplex;
3061
3062                 bmcr = 0;
3063                 switch (tp->link_config.speed) {
3064                 default:
3065                 case SPEED_10:
3066                         break;
3067
3068                 case SPEED_100:
3069                         bmcr |= BMCR_SPEED100;
3070                         break;
3071
3072                 case SPEED_1000:
3073                         bmcr |= TG3_BMCR_SPEED1000;
3074                         break;
3075                 }
3076
3077                 if (tp->link_config.duplex == DUPLEX_FULL)
3078                         bmcr |= BMCR_FULLDPLX;
3079
3080                 if (!tg3_readphy(tp, MII_BMCR, &orig_bmcr) &&
3081                     (bmcr != orig_bmcr)) {
3082                         tg3_writephy(tp, MII_BMCR, BMCR_LOOPBACK);
3083                         for (i = 0; i < 1500; i++) {
3084                                 u32 tmp;
3085
3086                                 udelay(10);
3087                                 if (tg3_readphy(tp, MII_BMSR, &tmp) ||
3088                                     tg3_readphy(tp, MII_BMSR, &tmp))
3089                                         continue;
3090                                 if (!(tmp & BMSR_LSTATUS)) {
3091                                         udelay(40);
3092                                         break;
3093                                 }
3094                         }
3095                         tg3_writephy(tp, MII_BMCR, bmcr);
3096                         udelay(40);
3097                 }
3098         } else {
3099                 tg3_writephy(tp, MII_BMCR,
3100                              BMCR_ANENABLE | BMCR_ANRESTART);
3101         }
3102 }
3103
3104 static int tg3_init_5401phy_dsp(struct tg3 *tp)
3105 {
3106         int err;
3107
3108         /* Turn off tap power management. */
3109         /* Set Extended packet length bit */
3110         err = tg3_phy_auxctl_write(tp, MII_TG3_AUXCTL_SHDWSEL_AUXCTL, 0x4c20);
3111
3112         err |= tg3_phydsp_write(tp, 0x0012, 0x1804);
3113         err |= tg3_phydsp_write(tp, 0x0013, 0x1204);
3114         err |= tg3_phydsp_write(tp, 0x8006, 0x0132);
3115         err |= tg3_phydsp_write(tp, 0x8006, 0x0232);
3116         err |= tg3_phydsp_write(tp, 0x201f, 0x0a20);
3117
3118         udelay(40);
3119
3120         return err;
3121 }
3122
3123 static int tg3_copper_is_advertising_all(struct tg3 *tp, u32 mask)
3124 {
3125         u32 adv_reg, all_mask = 0;
3126
3127         if (mask & ADVERTISED_10baseT_Half)
3128                 all_mask |= ADVERTISE_10HALF;
3129         if (mask & ADVERTISED_10baseT_Full)
3130                 all_mask |= ADVERTISE_10FULL;
3131         if (mask & ADVERTISED_100baseT_Half)
3132                 all_mask |= ADVERTISE_100HALF;
3133         if (mask & ADVERTISED_100baseT_Full)
3134                 all_mask |= ADVERTISE_100FULL;
3135
3136         if (tg3_readphy(tp, MII_ADVERTISE, &adv_reg))
3137                 return 0;
3138
3139         if ((adv_reg & all_mask) != all_mask)
3140                 return 0;
3141         if (!(tp->phy_flags & TG3_PHYFLG_10_100_ONLY)) {
3142                 u32 tg3_ctrl;
3143
3144                 all_mask = 0;
3145                 if (mask & ADVERTISED_1000baseT_Half)
3146                         all_mask |= ADVERTISE_1000HALF;
3147                 if (mask & ADVERTISED_1000baseT_Full)
3148                         all_mask |= ADVERTISE_1000FULL;
3149
3150                 if (tg3_readphy(tp, MII_TG3_CTRL, &tg3_ctrl))
3151                         return 0;
3152
3153                 if ((tg3_ctrl & all_mask) != all_mask)
3154                         return 0;
3155         }
3156         return 1;
3157 }
3158
3159 static int tg3_adv_1000T_flowctrl_ok(struct tg3 *tp, u32 *lcladv, u32 *rmtadv)
3160 {
3161         u32 curadv, reqadv;
3162
3163         if (tg3_readphy(tp, MII_ADVERTISE, lcladv))
3164                 return 1;
3165
3166         curadv = *lcladv & (ADVERTISE_PAUSE_CAP | ADVERTISE_PAUSE_ASYM);
3167         reqadv = tg3_advert_flowctrl_1000T(tp->link_config.flowctrl);
3168
3169         if (tp->link_config.active_duplex == DUPLEX_FULL) {
3170                 if (curadv != reqadv)
3171                         return 0;
3172
3173                 if (tg3_flag(tp, PAUSE_AUTONEG))
3174                         tg3_readphy(tp, MII_LPA, rmtadv);
3175         } else {
3176                 /* Reprogram the advertisement register, even if it
3177                  * does not affect the current link.  If the link
3178                  * gets renegotiated in the future, we can save an
3179                  * additional renegotiation cycle by advertising
3180                  * it correctly in the first place.
3181                  */
3182                 if (curadv != reqadv) {
3183                         *lcladv &= ~(ADVERTISE_PAUSE_CAP |
3184                                      ADVERTISE_PAUSE_ASYM);
3185                         tg3_writephy(tp, MII_ADVERTISE, *lcladv | reqadv);
3186                 }
3187         }
3188
3189         return 1;
3190 }
3191
3192 static int tg3_setup_copper_phy(struct tg3 *tp, int force_reset)
3193 {
3194         int current_link_up;
3195         u32 bmsr, val;
3196         u32 lcl_adv, rmt_adv;
3197         u16 current_speed;
3198         u8 current_duplex;
3199         int i, err;
3200
3201         tw32(MAC_EVENT, 0);
3202
3203         tw32_f(MAC_STATUS,
3204              (MAC_STATUS_SYNC_CHANGED |
3205               MAC_STATUS_CFG_CHANGED |
3206               MAC_STATUS_MI_COMPLETION |
3207               MAC_STATUS_LNKSTATE_CHANGED));
3208         udelay(40);
3209
3210         if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
3211                 tw32_f(MAC_MI_MODE,
3212                      (tp->mi_mode & ~MAC_MI_MODE_AUTO_POLL));
3213                 udelay(80);
3214         }
3215
3216         tg3_phy_auxctl_write(tp, MII_TG3_AUXCTL_SHDWSEL_PWRCTL, 0);
3217
3218         /* Some third-party PHYs need to be reset on link going
3219          * down.
3220          */
3221         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
3222              GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704 ||
3223              GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) &&
3224             netif_carrier_ok(tp->dev)) {
3225                 tg3_readphy(tp, MII_BMSR, &bmsr);
3226                 if (!tg3_readphy(tp, MII_BMSR, &bmsr) &&
3227                     !(bmsr & BMSR_LSTATUS))
3228                         force_reset = 1;
3229         }
3230         if (force_reset)
3231                 tg3_phy_reset(tp);
3232
3233         if ((tp->phy_id & TG3_PHY_ID_MASK) == TG3_PHY_ID_BCM5401) {
3234                 tg3_readphy(tp, MII_BMSR, &bmsr);
3235                 if (tg3_readphy(tp, MII_BMSR, &bmsr) ||
3236                     !tg3_flag(tp, INIT_COMPLETE))
3237                         bmsr = 0;
3238
3239                 if (!(bmsr & BMSR_LSTATUS)) {
3240                         err = tg3_init_5401phy_dsp(tp);
3241                         if (err)
3242                                 return err;
3243
3244                         tg3_readphy(tp, MII_BMSR, &bmsr);
3245                         for (i = 0; i < 1000; i++) {
3246                                 udelay(10);
3247                                 if (!tg3_readphy(tp, MII_BMSR, &bmsr) &&
3248                                     (bmsr & BMSR_LSTATUS)) {
3249                                         udelay(40);
3250                                         break;
3251                                 }
3252                         }
3253
3254                         if ((tp->phy_id & TG3_PHY_ID_REV_MASK) ==
3255                             TG3_PHY_REV_BCM5401_B0 &&
3256                             !(bmsr & BMSR_LSTATUS) &&
3257                             tp->link_config.active_speed == SPEED_1000) {
3258                                 err = tg3_phy_reset(tp);
3259                                 if (!err)
3260                                         err = tg3_init_5401phy_dsp(tp);
3261                                 if (err)
3262                                         return err;
3263                         }
3264                 }
3265         } else if (tp->pci_chip_rev_id == CHIPREV_ID_5701_A0 ||
3266                    tp->pci_chip_rev_id == CHIPREV_ID_5701_B0) {
3267                 /* 5701 {A0,B0} CRC bug workaround */
3268                 tg3_writephy(tp, 0x15, 0x0a75);
3269                 tg3_writephy(tp, MII_TG3_MISC_SHDW, 0x8c68);
3270                 tg3_writephy(tp, MII_TG3_MISC_SHDW, 0x8d68);
3271                 tg3_writephy(tp, MII_TG3_MISC_SHDW, 0x8c68);
3272         }
3273
3274         /* Clear pending interrupts... */
3275         tg3_readphy(tp, MII_TG3_ISTAT, &val);
3276         tg3_readphy(tp, MII_TG3_ISTAT, &val);
3277
3278         if (tp->phy_flags & TG3_PHYFLG_USE_MI_INTERRUPT)
3279                 tg3_writephy(tp, MII_TG3_IMASK, ~MII_TG3_INT_LINKCHG);
3280         else if (!(tp->phy_flags & TG3_PHYFLG_IS_FET))
3281                 tg3_writephy(tp, MII_TG3_IMASK, ~0);
3282
3283         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
3284             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) {
3285                 if (tp->led_ctrl == LED_CTRL_MODE_PHY_1)
3286                         tg3_writephy(tp, MII_TG3_EXT_CTRL,
3287                                      MII_TG3_EXT_CTRL_LNK3_LED_MODE);
3288                 else
3289                         tg3_writephy(tp, MII_TG3_EXT_CTRL, 0);
3290         }
3291
3292         current_link_up = 0;
3293         current_speed = SPEED_INVALID;
3294         current_duplex = DUPLEX_INVALID;
3295
3296         if (tp->phy_flags & TG3_PHYFLG_CAPACITIVE_COUPLING) {
3297                 err = tg3_phy_auxctl_read(tp,
3298                                           MII_TG3_AUXCTL_SHDWSEL_MISCTEST,
3299                                           &val);
3300                 if (!err && !(val & (1 << 10))) {
3301                         tg3_phy_auxctl_write(tp,
3302                                              MII_TG3_AUXCTL_SHDWSEL_MISCTEST,
3303                                              val | (1 << 10));
3304                         goto relink;
3305                 }
3306         }
3307
3308         bmsr = 0;
3309         for (i = 0; i < 100; i++) {
3310                 tg3_readphy(tp, MII_BMSR, &bmsr);
3311                 if (!tg3_readphy(tp, MII_BMSR, &bmsr) &&
3312                     (bmsr & BMSR_LSTATUS))
3313                         break;
3314                 udelay(40);
3315         }
3316
3317         if (bmsr & BMSR_LSTATUS) {
3318                 u32 aux_stat, bmcr;
3319
3320                 tg3_readphy(tp, MII_TG3_AUX_STAT, &aux_stat);
3321                 for (i = 0; i < 2000; i++) {
3322                         udelay(10);
3323                         if (!tg3_readphy(tp, MII_TG3_AUX_STAT, &aux_stat) &&
3324                             aux_stat)
3325                                 break;
3326                 }
3327
3328                 tg3_aux_stat_to_speed_duplex(tp, aux_stat,
3329                                              &current_speed,
3330                                              &current_duplex);
3331
3332                 bmcr = 0;
3333                 for (i = 0; i < 200; i++) {
3334                         tg3_readphy(tp, MII_BMCR, &bmcr);
3335                         if (tg3_readphy(tp, MII_BMCR, &bmcr))
3336                                 continue;
3337                         if (bmcr && bmcr != 0x7fff)
3338                                 break;
3339                         udelay(10);
3340                 }
3341
3342                 lcl_adv = 0;
3343                 rmt_adv = 0;
3344
3345                 tp->link_config.active_speed = current_speed;
3346                 tp->link_config.active_duplex = current_duplex;
3347
3348                 if (tp->link_config.autoneg == AUTONEG_ENABLE) {
3349                         if ((bmcr & BMCR_ANENABLE) &&
3350                             tg3_copper_is_advertising_all(tp,
3351                                                 tp->link_config.advertising)) {
3352                                 if (tg3_adv_1000T_flowctrl_ok(tp, &lcl_adv,
3353                                                                   &rmt_adv))
3354                                         current_link_up = 1;
3355                         }
3356                 } else {
3357                         if (!(bmcr & BMCR_ANENABLE) &&
3358                             tp->link_config.speed == current_speed &&
3359                             tp->link_config.duplex == current_duplex &&
3360                             tp->link_config.flowctrl ==
3361                             tp->link_config.active_flowctrl) {
3362                                 current_link_up = 1;
3363                         }
3364                 }
3365
3366                 if (current_link_up == 1 &&
3367                     tp->link_config.active_duplex == DUPLEX_FULL)
3368                         tg3_setup_flow_control(tp, lcl_adv, rmt_adv);
3369         }
3370
3371 relink:
3372         if (current_link_up == 0 || (tp->phy_flags & TG3_PHYFLG_IS_LOW_POWER)) {
3373                 tg3_phy_copper_begin(tp);
3374
3375                 tg3_readphy(tp, MII_BMSR, &bmsr);
3376                 if ((!tg3_readphy(tp, MII_BMSR, &bmsr) && (bmsr & BMSR_LSTATUS)) ||
3377                     (tp->mac_mode & MAC_MODE_PORT_INT_LPBACK))
3378                         current_link_up = 1;
3379         }
3380
3381         tp->mac_mode &= ~MAC_MODE_PORT_MODE_MASK;
3382         if (current_link_up == 1) {
3383                 if (tp->link_config.active_speed == SPEED_100 ||
3384                     tp->link_config.active_speed == SPEED_10)
3385                         tp->mac_mode |= MAC_MODE_PORT_MODE_MII;
3386                 else
3387                         tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;
3388         } else if (tp->phy_flags & TG3_PHYFLG_IS_FET)
3389                 tp->mac_mode |= MAC_MODE_PORT_MODE_MII;
3390         else
3391                 tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;
3392
3393         tp->mac_mode &= ~MAC_MODE_HALF_DUPLEX;
3394         if (tp->link_config.active_duplex == DUPLEX_HALF)
3395                 tp->mac_mode |= MAC_MODE_HALF_DUPLEX;
3396
3397         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700) {
3398                 if (current_link_up == 1 &&
3399                     tg3_5700_link_polarity(tp, tp->link_config.active_speed))
3400                         tp->mac_mode |= MAC_MODE_LINK_POLARITY;
3401                 else
3402                         tp->mac_mode &= ~MAC_MODE_LINK_POLARITY;
3403         }
3404
3405         /* ??? Without this setting Netgear GA302T PHY does not
3406          * ??? send/receive packets...
3407          */
3408         if ((tp->phy_id & TG3_PHY_ID_MASK) == TG3_PHY_ID_BCM5411 &&
3409             tp->pci_chip_rev_id == CHIPREV_ID_5700_ALTIMA) {
3410                 tp->mi_mode |= MAC_MI_MODE_AUTO_POLL;
3411                 tw32_f(MAC_MI_MODE, tp->mi_mode);
3412                 udelay(80);
3413         }
3414
3415         tw32_f(MAC_MODE, tp->mac_mode);
3416         udelay(40);
3417
3418         tg3_phy_eee_adjust(tp, current_link_up);
3419
3420         if (tg3_flag(tp, USE_LINKCHG_REG)) {
3421                 /* Polled via timer. */
3422                 tw32_f(MAC_EVENT, 0);
3423         } else {
3424                 tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
3425         }
3426         udelay(40);
3427
3428         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 &&
3429             current_link_up == 1 &&
3430             tp->link_config.active_speed == SPEED_1000 &&
3431             (tg3_flag(tp, PCIX_MODE) || tg3_flag(tp, PCI_HIGH_SPEED))) {
3432                 udelay(120);
3433                 tw32_f(MAC_STATUS,
3434                      (MAC_STATUS_SYNC_CHANGED |
3435                       MAC_STATUS_CFG_CHANGED));
3436                 udelay(40);
3437                 tg3_write_mem(tp,
3438                               NIC_SRAM_FIRMWARE_MBOX,
3439                               NIC_SRAM_FIRMWARE_MBOX_MAGIC2);
3440         }
3441
3442         /* Prevent send BD corruption. */
3443         if (tg3_flag(tp, CLKREQ_BUG)) {
3444                 u16 oldlnkctl, newlnkctl;
3445
3446                 pci_read_config_word(tp->pdev,
3447                                      tp->pcie_cap + PCI_EXP_LNKCTL,
3448                                      &oldlnkctl);
3449                 if (tp->link_config.active_speed == SPEED_100 ||
3450                     tp->link_config.active_speed == SPEED_10)
3451                         newlnkctl = oldlnkctl & ~PCI_EXP_LNKCTL_CLKREQ_EN;
3452                 else
3453                         newlnkctl = oldlnkctl | PCI_EXP_LNKCTL_CLKREQ_EN;
3454                 if (newlnkctl != oldlnkctl)
3455                         pci_write_config_word(tp->pdev,
3456                                               tp->pcie_cap + PCI_EXP_LNKCTL,
3457                                               newlnkctl);
3458         }
3459
3460         if (current_link_up != netif_carrier_ok(tp->dev)) {
3461                 if (current_link_up)
3462                         netif_carrier_on(tp->dev);
3463                 else
3464                         netif_carrier_off(tp->dev);
3465                 tg3_link_report(tp);
3466         }
3467
3468         return 0;
3469 }
3470
3471 struct tg3_fiber_aneginfo {
3472         int state;
3473 #define ANEG_STATE_UNKNOWN              0
3474 #define ANEG_STATE_AN_ENABLE            1
3475 #define ANEG_STATE_RESTART_INIT         2
3476 #define ANEG_STATE_RESTART              3
3477 #define ANEG_STATE_DISABLE_LINK_OK      4
3478 #define ANEG_STATE_ABILITY_DETECT_INIT  5
3479 #define ANEG_STATE_ABILITY_DETECT       6
3480 #define ANEG_STATE_ACK_DETECT_INIT      7
3481 #define ANEG_STATE_ACK_DETECT           8
3482 #define ANEG_STATE_COMPLETE_ACK_INIT    9
3483 #define ANEG_STATE_COMPLETE_ACK         10
3484 #define ANEG_STATE_IDLE_DETECT_INIT     11
3485 #define ANEG_STATE_IDLE_DETECT          12
3486 #define ANEG_STATE_LINK_OK              13
3487 #define ANEG_STATE_NEXT_PAGE_WAIT_INIT  14
3488 #define ANEG_STATE_NEXT_PAGE_WAIT       15
3489
3490         u32 flags;
3491 #define MR_AN_ENABLE            0x00000001
3492 #define MR_RESTART_AN           0x00000002
3493 #define MR_AN_COMPLETE          0x00000004
3494 #define MR_PAGE_RX              0x00000008
3495 #define MR_NP_LOADED            0x00000010
3496 #define MR_TOGGLE_TX            0x00000020
3497 #define MR_LP_ADV_FULL_DUPLEX   0x00000040
3498 #define MR_LP_ADV_HALF_DUPLEX   0x00000080
3499 #define MR_LP_ADV_SYM_PAUSE     0x00000100
3500 #define MR_LP_ADV_ASYM_PAUSE    0x00000200
3501 #define MR_LP_ADV_REMOTE_FAULT1 0x00000400
3502 #define MR_LP_ADV_REMOTE_FAULT2 0x00000800
3503 #define MR_LP_ADV_NEXT_PAGE     0x00001000
3504 #define MR_TOGGLE_RX            0x00002000
3505 #define MR_NP_RX                0x00004000
3506
3507 #define MR_LINK_OK              0x80000000
3508
3509         unsigned long link_time, cur_time;
3510
3511         u32 ability_match_cfg;
3512         int ability_match_count;
3513
3514         char ability_match, idle_match, ack_match;
3515
3516         u32 txconfig, rxconfig;
3517 #define ANEG_CFG_NP             0x00000080
3518 #define ANEG_CFG_ACK            0x00000040
3519 #define ANEG_CFG_RF2            0x00000020
3520 #define ANEG_CFG_RF1            0x00000010
3521 #define ANEG_CFG_PS2            0x00000001
3522 #define ANEG_CFG_PS1            0x00008000
3523 #define ANEG_CFG_HD             0x00004000
3524 #define ANEG_CFG_FD             0x00002000
3525 #define ANEG_CFG_INVAL          0x00001f06
3526
3527 };
3528 #define ANEG_OK         0
3529 #define ANEG_DONE       1
3530 #define ANEG_TIMER_ENAB 2
3531 #define ANEG_FAILED     -1
3532
3533 #define ANEG_STATE_SETTLE_TIME  10000
3534
3535 static int tg3_fiber_aneg_smachine(struct tg3 *tp,
3536                                    struct tg3_fiber_aneginfo *ap)
3537 {
3538         u16 flowctrl;
3539         unsigned long delta;
3540         u32 rx_cfg_reg;
3541         int ret;
3542
3543         if (ap->state == ANEG_STATE_UNKNOWN) {
3544                 ap->rxconfig = 0;
3545                 ap->link_time = 0;
3546                 ap->cur_time = 0;
3547                 ap->ability_match_cfg = 0;
3548                 ap->ability_match_count = 0;
3549                 ap->ability_match = 0;
3550                 ap->idle_match = 0;
3551                 ap->ack_match = 0;
3552         }
3553         ap->cur_time++;
3554
3555         if (tr32(MAC_STATUS) & MAC_STATUS_RCVD_CFG) {
3556                 rx_cfg_reg = tr32(MAC_RX_AUTO_NEG);
3557
3558                 if (rx_cfg_reg != ap->ability_match_cfg) {
3559                         ap->ability_match_cfg = rx_cfg_reg;
3560                         ap->ability_match = 0;
3561                         ap->ability_match_count = 0;
3562                 } else {
3563                         if (++ap->ability_match_count > 1) {
3564                                 ap->ability_match = 1;
3565                                 ap->ability_match_cfg = rx_cfg_reg;
3566                         }
3567                 }
3568                 if (rx_cfg_reg & ANEG_CFG_ACK)
3569                         ap->ack_match = 1;
3570                 else
3571                         ap->ack_match = 0;
3572
3573                 ap->idle_match = 0;
3574         } else {
3575                 ap->idle_match = 1;
3576                 ap->ability_match_cfg = 0;
3577                 ap->ability_match_count = 0;
3578                 ap->ability_match = 0;
3579                 ap->ack_match = 0;
3580
3581                 rx_cfg_reg = 0;
3582         }
3583
3584         ap->rxconfig = rx_cfg_reg;
3585         ret = ANEG_OK;
3586
3587         switch (ap->state) {
3588         case ANEG_STATE_UNKNOWN:
3589                 if (ap->flags & (MR_AN_ENABLE | MR_RESTART_AN))
3590                         ap->state = ANEG_STATE_AN_ENABLE;
3591
3592                 /* fallthru */
3593         case ANEG_STATE_AN_ENABLE:
3594                 ap->flags &= ~(MR_AN_COMPLETE | MR_PAGE_RX);
3595                 if (ap->flags & MR_AN_ENABLE) {
3596                         ap->link_time = 0;
3597                         ap->cur_time = 0;
3598                         ap->ability_match_cfg = 0;
3599                         ap->ability_match_count = 0;
3600                         ap->ability_match = 0;
3601                         ap->idle_match = 0;
3602                         ap->ack_match = 0;
3603
3604                         ap->state = ANEG_STATE_RESTART_INIT;
3605                 } else {
3606                         ap->state = ANEG_STATE_DISABLE_LINK_OK;
3607                 }
3608                 break;
3609
3610         case ANEG_STATE_RESTART_INIT:
3611                 ap->link_time = ap->cur_time;
3612                 ap->flags &= ~(MR_NP_LOADED);
3613                 ap->txconfig = 0;
3614                 tw32(MAC_TX_AUTO_NEG, 0);
3615                 tp->mac_mode |= MAC_MODE_SEND_CONFIGS;
3616                 tw32_f(MAC_MODE, tp->mac_mode);
3617                 udelay(40);
3618
3619                 ret = ANEG_TIMER_ENAB;
3620                 ap->state = ANEG_STATE_RESTART;
3621
3622                 /* fallthru */
3623         case ANEG_STATE_RESTART:
3624                 delta = ap->cur_time - ap->link_time;
3625                 if (delta > ANEG_STATE_SETTLE_TIME)
3626                         ap->state = ANEG_STATE_ABILITY_DETECT_INIT;
3627                 else
3628                         ret = ANEG_TIMER_ENAB;
3629                 break;
3630
3631         case ANEG_STATE_DISABLE_LINK_OK:
3632                 ret = ANEG_DONE;
3633                 break;
3634
3635         case ANEG_STATE_ABILITY_DETECT_INIT:
3636                 ap->flags &= ~(MR_TOGGLE_TX);
3637                 ap->txconfig = ANEG_CFG_FD;
3638                 flowctrl = tg3_advert_flowctrl_1000X(tp->link_config.flowctrl);
3639                 if (flowctrl & ADVERTISE_1000XPAUSE)
3640                         ap->txconfig |= ANEG_CFG_PS1;
3641                 if (flowctrl & ADVERTISE_1000XPSE_ASYM)
3642                         ap->txconfig |= ANEG_CFG_PS2;
3643                 tw32(MAC_TX_AUTO_NEG, ap->txconfig);
3644                 tp->mac_mode |= MAC_MODE_SEND_CONFIGS;
3645                 tw32_f(MAC_MODE, tp->mac_mode);
3646                 udelay(40);
3647
3648                 ap->state = ANEG_STATE_ABILITY_DETECT;
3649                 break;
3650
3651         case ANEG_STATE_ABILITY_DETECT:
3652                 if (ap->ability_match != 0 && ap->rxconfig != 0)
3653                         ap->state = ANEG_STATE_ACK_DETECT_INIT;
3654                 break;
3655
3656         case ANEG_STATE_ACK_DETECT_INIT:
3657                 ap->txconfig |= ANEG_CFG_ACK;
3658                 tw32(MAC_TX_AUTO_NEG, ap->txconfig);
3659                 tp->mac_mode |= MAC_MODE_SEND_CONFIGS;
3660                 tw32_f(MAC_MODE, tp->mac_mode);
3661                 udelay(40);
3662
3663                 ap->state = ANEG_STATE_ACK_DETECT;
3664
3665                 /* fallthru */
3666         case ANEG_STATE_ACK_DETECT:
3667                 if (ap->ack_match != 0) {
3668                         if ((ap->rxconfig & ~ANEG_CFG_ACK) ==
3669                             (ap->ability_match_cfg & ~ANEG_CFG_ACK)) {
3670                                 ap->state = ANEG_STATE_COMPLETE_ACK_INIT;
3671                         } else {
3672                                 ap->state = ANEG_STATE_AN_ENABLE;
3673                         }
3674                 } else if (ap->ability_match != 0 &&
3675                            ap->rxconfig == 0) {
3676                         ap->state = ANEG_STATE_AN_ENABLE;
3677                 }
3678                 break;
3679
3680         case ANEG_STATE_COMPLETE_ACK_INIT:
3681                 if (ap->rxconfig & ANEG_CFG_INVAL) {
3682                         ret = ANEG_FAILED;
3683                         break;
3684                 }
3685                 ap->flags &= ~(MR_LP_ADV_FULL_DUPLEX |
3686                                MR_LP_ADV_HALF_DUPLEX |
3687                                MR_LP_ADV_SYM_PAUSE |
3688                                MR_LP_ADV_ASYM_PAUSE |
3689                                MR_LP_ADV_REMOTE_FAULT1 |
3690                                MR_LP_ADV_REMOTE_FAULT2 |
3691                                MR_LP_ADV_NEXT_PAGE |
3692                                MR_TOGGLE_RX |
3693                                MR_NP_RX);
3694                 if (ap->rxconfig & ANEG_CFG_FD)
3695                         ap->flags |= MR_LP_ADV_FULL_DUPLEX;
3696                 if (ap->rxconfig & ANEG_CFG_HD)
3697                         ap->flags |= MR_LP_ADV_HALF_DUPLEX;
3698                 if (ap->rxconfig & ANEG_CFG_PS1)
3699                         ap->flags |= MR_LP_ADV_SYM_PAUSE;
3700                 if (ap->rxconfig & ANEG_CFG_PS2)
3701                         ap->flags |= MR_LP_ADV_ASYM_PAUSE;
3702                 if (ap->rxconfig & ANEG_CFG_RF1)
3703                         ap->flags |= MR_LP_ADV_REMOTE_FAULT1;
3704                 if (ap->rxconfig & ANEG_CFG_RF2)
3705                         ap->flags |= MR_LP_ADV_REMOTE_FAULT2;
3706                 if (ap->rxconfig & ANEG_CFG_NP)
3707                         ap->flags |= MR_LP_ADV_NEXT_PAGE;
3708
3709                 ap->link_time = ap->cur_time;
3710
3711                 ap->flags ^= (MR_TOGGLE_TX);
3712                 if (ap->rxconfig & 0x0008)
3713                         ap->flags |= MR_TOGGLE_RX;
3714                 if (ap->rxconfig & ANEG_CFG_NP)
3715                         ap->flags |= MR_NP_RX;
3716                 ap->flags |= MR_PAGE_RX;
3717
3718                 ap->state = ANEG_STATE_COMPLETE_ACK;
3719                 ret = ANEG_TIMER_ENAB;
3720                 break;
3721
3722         case ANEG_STATE_COMPLETE_ACK:
3723                 if (ap->ability_match != 0 &&
3724                     ap->rxconfig == 0) {
3725                         ap->state = ANEG_STATE_AN_ENABLE;
3726                         break;
3727                 }
3728                 delta = ap->cur_time - ap->link_time;
3729                 if (delta > ANEG_STATE_SETTLE_TIME) {
3730                         if (!(ap->flags & (MR_LP_ADV_NEXT_PAGE))) {
3731                                 ap->state = ANEG_STATE_IDLE_DETECT_INIT;
3732                         } else {
3733                                 if ((ap->txconfig & ANEG_CFG_NP) == 0 &&
3734                                     !(ap->flags & MR_NP_RX)) {
3735                                         ap->state = ANEG_STATE_IDLE_DETECT_INIT;
3736                                 } else {
3737                                         ret = ANEG_FAILED;
3738                                 }
3739                         }
3740                 }
3741                 break;
3742
3743         case ANEG_STATE_IDLE_DETECT_INIT:
3744                 ap->link_time = ap->cur_time;
3745                 tp->mac_mode &= ~MAC_MODE_SEND_CONFIGS;
3746                 tw32_f(MAC_MODE, tp->mac_mode);
3747                 udelay(40);
3748
3749                 ap->state = ANEG_STATE_IDLE_DETECT;
3750                 ret = ANEG_TIMER_ENAB;
3751                 break;
3752
3753         case ANEG_STATE_IDLE_DETECT:
3754                 if (ap->ability_match != 0 &&
3755                     ap->rxconfig == 0) {
3756                         ap->state = ANEG_STATE_AN_ENABLE;
3757                         break;
3758                 }
3759                 delta = ap->cur_time - ap->link_time;
3760                 if (delta > ANEG_STATE_SETTLE_TIME) {
3761                         /* XXX another gem from the Broadcom driver :( */
3762                         ap->state = ANEG_STATE_LINK_OK;
3763                 }
3764                 break;
3765
3766         case ANEG_STATE_LINK_OK:
3767                 ap->flags |= (MR_AN_COMPLETE | MR_LINK_OK);
3768                 ret = ANEG_DONE;
3769                 break;
3770
3771         case ANEG_STATE_NEXT_PAGE_WAIT_INIT:
3772                 /* ??? unimplemented */
3773                 break;
3774
3775         case ANEG_STATE_NEXT_PAGE_WAIT:
3776                 /* ??? unimplemented */
3777                 break;
3778
3779         default:
3780                 ret = ANEG_FAILED;
3781                 break;
3782         }
3783
3784         return ret;
3785 }
3786
3787 static int fiber_autoneg(struct tg3 *tp, u32 *txflags, u32 *rxflags)
3788 {
3789         int res = 0;
3790         struct tg3_fiber_aneginfo aninfo;
3791         int status = ANEG_FAILED;
3792         unsigned int tick;
3793         u32 tmp;
3794
3795         tw32_f(MAC_TX_AUTO_NEG, 0);
3796
3797         tmp = tp->mac_mode & ~MAC_MODE_PORT_MODE_MASK;
3798         tw32_f(MAC_MODE, tmp | MAC_MODE_PORT_MODE_GMII);
3799         udelay(40);
3800
3801         tw32_f(MAC_MODE, tp->mac_mode | MAC_MODE_SEND_CONFIGS);
3802         udelay(40);
3803
3804         memset(&aninfo, 0, sizeof(aninfo));
3805         aninfo.flags |= MR_AN_ENABLE;
3806         aninfo.state = ANEG_STATE_UNKNOWN;
3807         aninfo.cur_time = 0;
3808         tick = 0;
3809         while (++tick < 195000) {
3810                 status = tg3_fiber_aneg_smachine(tp, &aninfo);
3811                 if (status == ANEG_DONE || status == ANEG_FAILED)
3812                         break;
3813
3814                 udelay(1);
3815         }
3816
3817         tp->mac_mode &= ~MAC_MODE_SEND_CONFIGS;
3818         tw32_f(MAC_MODE, tp->mac_mode);
3819         udelay(40);
3820
3821         *txflags = aninfo.txconfig;
3822         *rxflags = aninfo.flags;
3823
3824         if (status == ANEG_DONE &&
3825             (aninfo.flags & (MR_AN_COMPLETE | MR_LINK_OK |
3826                              MR_LP_ADV_FULL_DUPLEX)))
3827                 res = 1;
3828
3829         return res;
3830 }
3831
3832 static void tg3_init_bcm8002(struct tg3 *tp)
3833 {
3834         u32 mac_status = tr32(MAC_STATUS);
3835         int i;
3836
3837         /* Reset when initting first time or we have a link. */
3838         if (tg3_flag(tp, INIT_COMPLETE) &&
3839             !(mac_status & MAC_STATUS_PCS_SYNCED))
3840                 return;
3841
3842         /* Set PLL lock range. */
3843         tg3_writephy(tp, 0x16, 0x8007);
3844
3845         /* SW reset */
3846         tg3_writephy(tp, MII_BMCR, BMCR_RESET);
3847
3848         /* Wait for reset to complete. */
3849         /* XXX schedule_timeout() ... */
3850         for (i = 0; i < 500; i++)
3851                 udelay(10);
3852
3853         /* Config mode; select PMA/Ch 1 regs. */
3854         tg3_writephy(tp, 0x10, 0x8411);
3855
3856         /* Enable auto-lock and comdet, select txclk for tx. */
3857         tg3_writephy(tp, 0x11, 0x0a10);
3858
3859         tg3_writephy(tp, 0x18, 0x00a0);
3860         tg3_writephy(tp, 0x16, 0x41ff);
3861
3862         /* Assert and deassert POR. */
3863         tg3_writephy(tp, 0x13, 0x0400);
3864         udelay(40);
3865         tg3_writephy(tp, 0x13, 0x0000);
3866
3867         tg3_writephy(tp, 0x11, 0x0a50);
3868         udelay(40);
3869         tg3_writephy(tp, 0x11, 0x0a10);
3870
3871         /* Wait for signal to stabilize */
3872         /* XXX schedule_timeout() ... */
3873         for (i = 0; i < 15000; i++)
3874                 udelay(10);
3875
3876         /* Deselect the channel register so we can read the PHYID
3877          * later.
3878          */
3879         tg3_writephy(tp, 0x10, 0x8011);
3880 }
3881
3882 static int tg3_setup_fiber_hw_autoneg(struct tg3 *tp, u32 mac_status)
3883 {
3884         u16 flowctrl;
3885         u32 sg_dig_ctrl, sg_dig_status;
3886         u32 serdes_cfg, expected_sg_dig_ctrl;
3887         int workaround, port_a;
3888         int current_link_up;
3889
3890         serdes_cfg = 0;
3891         expected_sg_dig_ctrl = 0;
3892         workaround = 0;
3893         port_a = 1;
3894         current_link_up = 0;
3895
3896         if (tp->pci_chip_rev_id != CHIPREV_ID_5704_A0 &&
3897             tp->pci_chip_rev_id != CHIPREV_ID_5704_A1) {
3898                 workaround = 1;
3899                 if (tr32(TG3PCI_DUAL_MAC_CTRL) & DUAL_MAC_CTRL_ID)
3900                         port_a = 0;
3901
3902                 /* preserve bits 0-11,13,14 for signal pre-emphasis */
3903                 /* preserve bits 20-23 for voltage regulator */
3904                 serdes_cfg = tr32(MAC_SERDES_CFG) & 0x00f06fff;
3905         }
3906
3907         sg_dig_ctrl = tr32(SG_DIG_CTRL);
3908
3909         if (tp->link_config.autoneg != AUTONEG_ENABLE) {
3910                 if (sg_dig_ctrl & SG_DIG_USING_HW_AUTONEG) {
3911                         if (workaround) {
3912                                 u32 val = serdes_cfg;
3913
3914                                 if (port_a)
3915                                         val |= 0xc010000;
3916                                 else
3917                                         val |= 0x4010000;
3918                                 tw32_f(MAC_SERDES_CFG, val);
3919                         }
3920
3921                         tw32_f(SG_DIG_CTRL, SG_DIG_COMMON_SETUP);
3922                 }
3923                 if (mac_status & MAC_STATUS_PCS_SYNCED) {
3924                         tg3_setup_flow_control(tp, 0, 0);
3925                         current_link_up = 1;
3926                 }
3927                 goto out;
3928         }
3929
3930         /* Want auto-negotiation.  */
3931         expected_sg_dig_ctrl = SG_DIG_USING_HW_AUTONEG | SG_DIG_COMMON_SETUP;
3932
3933         flowctrl = tg3_advert_flowctrl_1000X(tp->link_config.flowctrl);
3934         if (flowctrl & ADVERTISE_1000XPAUSE)
3935                 expected_sg_dig_ctrl |= SG_DIG_PAUSE_CAP;
3936         if (flowctrl & ADVERTISE_1000XPSE_ASYM)
3937                 expected_sg_dig_ctrl |= SG_DIG_ASYM_PAUSE;
3938
3939         if (sg_dig_ctrl != expected_sg_dig_ctrl) {
3940                 if ((tp->phy_flags & TG3_PHYFLG_PARALLEL_DETECT) &&
3941                     tp->serdes_counter &&
3942                     ((mac_status & (MAC_STATUS_PCS_SYNCED |
3943                                     MAC_STATUS_RCVD_CFG)) ==
3944                      MAC_STATUS_PCS_SYNCED)) {
3945                         tp->serdes_counter--;
3946                         current_link_up = 1;
3947                         goto out;
3948                 }
3949 restart_autoneg:
3950                 if (workaround)
3951                         tw32_f(MAC_SERDES_CFG, serdes_cfg | 0xc011000);
3952                 tw32_f(SG_DIG_CTRL, expected_sg_dig_ctrl | SG_DIG_SOFT_RESET);
3953                 udelay(5);
3954                 tw32_f(SG_DIG_CTRL, expected_sg_dig_ctrl);
3955
3956                 tp->serdes_counter = SERDES_AN_TIMEOUT_5704S;
3957                 tp->phy_flags &= ~TG3_PHYFLG_PARALLEL_DETECT;
3958         } else if (mac_status & (MAC_STATUS_PCS_SYNCED |
3959                                  MAC_STATUS_SIGNAL_DET)) {
3960                 sg_dig_status = tr32(SG_DIG_STATUS);
3961                 mac_status = tr32(MAC_STATUS);
3962
3963                 if ((sg_dig_status & SG_DIG_AUTONEG_COMPLETE) &&
3964                     (mac_status & MAC_STATUS_PCS_SYNCED)) {
3965                         u32 local_adv = 0, remote_adv = 0;
3966
3967                         if (sg_dig_ctrl & SG_DIG_PAUSE_CAP)
3968                                 local_adv |= ADVERTISE_1000XPAUSE;
3969                         if (sg_dig_ctrl & SG_DIG_ASYM_PAUSE)
3970                                 local_adv |= ADVERTISE_1000XPSE_ASYM;
3971
3972                         if (sg_dig_status & SG_DIG_PARTNER_PAUSE_CAPABLE)
3973                                 remote_adv |= LPA_1000XPAUSE;
3974                         if (sg_dig_status & SG_DIG_PARTNER_ASYM_PAUSE)
3975                                 remote_adv |= LPA_1000XPAUSE_ASYM;
3976
3977                         tg3_setup_flow_control(tp, local_adv, remote_adv);
3978                         current_link_up = 1;
3979                         tp->serdes_counter = 0;
3980                         tp->phy_flags &= ~TG3_PHYFLG_PARALLEL_DETECT;
3981                 } else if (!(sg_dig_status & SG_DIG_AUTONEG_COMPLETE)) {
3982                         if (tp->serdes_counter)
3983                                 tp->serdes_counter--;
3984                         else {
3985                                 if (workaround) {
3986                                         u32 val = serdes_cfg;
3987
3988                                         if (port_a)
3989                                                 val |= 0xc010000;
3990                                         else
3991                                                 val |= 0x4010000;
3992
3993                                         tw32_f(MAC_SERDES_CFG, val);
3994                                 }
3995
3996                                 tw32_f(SG_DIG_CTRL, SG_DIG_COMMON_SETUP);
3997                                 udelay(40);
3998
3999                                 /* Link parallel detection - link is up */
4000                                 /* only if we have PCS_SYNC and not */
4001                                 /* receiving config code words */
4002                                 mac_status = tr32(MAC_STATUS);
4003                                 if ((mac_status & MAC_STATUS_PCS_SYNCED) &&
4004                                     !(mac_status & MAC_STATUS_RCVD_CFG)) {
4005                                         tg3_setup_flow_control(tp, 0, 0);
4006                                         current_link_up = 1;
4007                                         tp->phy_flags |=
4008                                                 TG3_PHYFLG_PARALLEL_DETECT;
4009                                         tp->serdes_counter =
4010                                                 SERDES_PARALLEL_DET_TIMEOUT;
4011                                 } else
4012                                         goto restart_autoneg;
4013                         }
4014                 }
4015         } else {
4016                 tp->serdes_counter = SERDES_AN_TIMEOUT_5704S;
4017                 tp->phy_flags &= ~TG3_PHYFLG_PARALLEL_DETECT;
4018         }
4019
4020 out:
4021         return current_link_up;
4022 }
4023
4024 static int tg3_setup_fiber_by_hand(struct tg3 *tp, u32 mac_status)
4025 {
4026         int current_link_up = 0;
4027
4028         if (!(mac_status & MAC_STATUS_PCS_SYNCED))
4029                 goto out;
4030
4031         if (tp->link_config.autoneg == AUTONEG_ENABLE) {
4032                 u32 txflags, rxflags;
4033                 int i;
4034
4035                 if (fiber_autoneg(tp, &txflags, &rxflags)) {
4036                         u32 local_adv = 0, remote_adv = 0;
4037
4038                         if (txflags & ANEG_CFG_PS1)
4039                                 local_adv |= ADVERTISE_1000XPAUSE;
4040                         if (txflags & ANEG_CFG_PS2)
4041                                 local_adv |= ADVERTISE_1000XPSE_ASYM;
4042
4043                         if (rxflags & MR_LP_ADV_SYM_PAUSE)
4044                                 remote_adv |= LPA_1000XPAUSE;
4045                         if (rxflags & MR_LP_ADV_ASYM_PAUSE)
4046                                 remote_adv |= LPA_1000XPAUSE_ASYM;
4047
4048                         tg3_setup_flow_control(tp, local_adv, remote_adv);
4049
4050                         current_link_up = 1;
4051                 }
4052                 for (i = 0; i < 30; i++) {
4053                         udelay(20);
4054                         tw32_f(MAC_STATUS,
4055                                (MAC_STATUS_SYNC_CHANGED |
4056                                 MAC_STATUS_CFG_CHANGED));
4057                         udelay(40);
4058                         if ((tr32(MAC_STATUS) &
4059                              (MAC_STATUS_SYNC_CHANGED |
4060                               MAC_STATUS_CFG_CHANGED)) == 0)
4061                                 break;
4062                 }
4063
4064                 mac_status = tr32(MAC_STATUS);
4065                 if (current_link_up == 0 &&
4066                     (mac_status & MAC_STATUS_PCS_SYNCED) &&
4067                     !(mac_status & MAC_STATUS_RCVD_CFG))
4068                         current_link_up = 1;
4069         } else {
4070                 tg3_setup_flow_control(tp, 0, 0);
4071
4072                 /* Forcing 1000FD link up. */
4073                 current_link_up = 1;
4074
4075                 tw32_f(MAC_MODE, (tp->mac_mode | MAC_MODE_SEND_CONFIGS));
4076                 udelay(40);
4077
4078                 tw32_f(MAC_MODE, tp->mac_mode);
4079                 udelay(40);
4080         }
4081
4082 out:
4083         return current_link_up;
4084 }
4085
4086 static int tg3_setup_fiber_phy(struct tg3 *tp, int force_reset)
4087 {
4088         u32 orig_pause_cfg;
4089         u16 orig_active_speed;
4090         u8 orig_active_duplex;
4091         u32 mac_status;
4092         int current_link_up;
4093         int i;
4094
4095         orig_pause_cfg = tp->link_config.active_flowctrl;
4096         orig_active_speed = tp->link_config.active_speed;
4097         orig_active_duplex = tp->link_config.active_duplex;
4098
4099         if (!tg3_flag(tp, HW_AUTONEG) &&
4100             netif_carrier_ok(tp->dev) &&
4101             tg3_flag(tp, INIT_COMPLETE)) {
4102                 mac_status = tr32(MAC_STATUS);
4103                 mac_status &= (MAC_STATUS_PCS_SYNCED |
4104                                MAC_STATUS_SIGNAL_DET |
4105                                MAC_STATUS_CFG_CHANGED |
4106                                MAC_STATUS_RCVD_CFG);
4107                 if (mac_status == (MAC_STATUS_PCS_SYNCED |
4108                                    MAC_STATUS_SIGNAL_DET)) {
4109                         tw32_f(MAC_STATUS, (MAC_STATUS_SYNC_CHANGED |
4110                                             MAC_STATUS_CFG_CHANGED));
4111                         return 0;
4112                 }
4113         }
4114
4115         tw32_f(MAC_TX_AUTO_NEG, 0);
4116
4117         tp->mac_mode &= ~(MAC_MODE_PORT_MODE_MASK | MAC_MODE_HALF_DUPLEX);
4118         tp->mac_mode |= MAC_MODE_PORT_MODE_TBI;
4119         tw32_f(MAC_MODE, tp->mac_mode);
4120         udelay(40);
4121
4122         if (tp->phy_id == TG3_PHY_ID_BCM8002)
4123                 tg3_init_bcm8002(tp);
4124
4125         /* Enable link change event even when serdes polling.  */
4126         tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
4127         udelay(40);
4128
4129         current_link_up = 0;
4130         mac_status = tr32(MAC_STATUS);
4131
4132         if (tg3_flag(tp, HW_AUTONEG))
4133                 current_link_up = tg3_setup_fiber_hw_autoneg(tp, mac_status);
4134         else
4135                 current_link_up = tg3_setup_fiber_by_hand(tp, mac_status);
4136
4137         tp->napi[0].hw_status->status =
4138                 (SD_STATUS_UPDATED |
4139                  (tp->napi[0].hw_status->status & ~SD_STATUS_LINK_CHG));
4140
4141         for (i = 0; i < 100; i++) {
4142                 tw32_f(MAC_STATUS, (MAC_STATUS_SYNC_CHANGED |
4143                                     MAC_STATUS_CFG_CHANGED));
4144                 udelay(5);
4145                 if ((tr32(MAC_STATUS) & (MAC_STATUS_SYNC_CHANGED |
4146                                          MAC_STATUS_CFG_CHANGED |
4147                                          MAC_STATUS_LNKSTATE_CHANGED)) == 0)
4148                         break;
4149         }
4150
4151         mac_status = tr32(MAC_STATUS);
4152         if ((mac_status & MAC_STATUS_PCS_SYNCED) == 0) {
4153                 current_link_up = 0;
4154                 if (tp->link_config.autoneg == AUTONEG_ENABLE &&
4155                     tp->serdes_counter == 0) {
4156                         tw32_f(MAC_MODE, (tp->mac_mode |
4157                                           MAC_MODE_SEND_CONFIGS));
4158                         udelay(1);
4159                         tw32_f(MAC_MODE, tp->mac_mode);
4160                 }
4161         }
4162
4163         if (current_link_up == 1) {
4164                 tp->link_config.active_speed = SPEED_1000;
4165                 tp->link_config.active_duplex = DUPLEX_FULL;
4166                 tw32(MAC_LED_CTRL, (tp->led_ctrl |
4167                                     LED_CTRL_LNKLED_OVERRIDE |
4168                                     LED_CTRL_1000MBPS_ON));
4169         } else {
4170                 tp->link_config.active_speed = SPEED_INVALID;
4171                 tp->link_config.active_duplex = DUPLEX_INVALID;
4172                 tw32(MAC_LED_CTRL, (tp->led_ctrl |
4173                                     LED_CTRL_LNKLED_OVERRIDE |
4174                                     LED_CTRL_TRAFFIC_OVERRIDE));
4175         }
4176
4177         if (current_link_up != netif_carrier_ok(tp->dev)) {
4178                 if (current_link_up)
4179                         netif_carrier_on(tp->dev);
4180                 else
4181                         netif_carrier_off(tp->dev);
4182                 tg3_link_report(tp);
4183         } else {
4184                 u32 now_pause_cfg = tp->link_config.active_flowctrl;
4185                 if (orig_pause_cfg != now_pause_cfg ||
4186                     orig_active_speed != tp->link_config.active_speed ||
4187                     orig_active_duplex != tp->link_config.active_duplex)
4188                         tg3_link_report(tp);
4189         }
4190
4191         return 0;
4192 }
4193
4194 static int tg3_setup_fiber_mii_phy(struct tg3 *tp, int force_reset)
4195 {
4196         int current_link_up, err = 0;
4197         u32 bmsr, bmcr;
4198         u16 current_speed;
4199         u8 current_duplex;
4200         u32 local_adv, remote_adv;
4201
4202         tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;
4203         tw32_f(MAC_MODE, tp->mac_mode);
4204         udelay(40);
4205
4206         tw32(MAC_EVENT, 0);
4207
4208         tw32_f(MAC_STATUS,
4209              (MAC_STATUS_SYNC_CHANGED |
4210               MAC_STATUS_CFG_CHANGED |
4211               MAC_STATUS_MI_COMPLETION |
4212               MAC_STATUS_LNKSTATE_CHANGED));
4213         udelay(40);
4214
4215         if (force_reset)
4216                 tg3_phy_reset(tp);
4217
4218         current_link_up = 0;
4219         current_speed = SPEED_INVALID;
4220         current_duplex = DUPLEX_INVALID;
4221
4222         err |= tg3_readphy(tp, MII_BMSR, &bmsr);
4223         err |= tg3_readphy(tp, MII_BMSR, &bmsr);
4224         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714) {
4225                 if (tr32(MAC_TX_STATUS) & TX_STATUS_LINK_UP)
4226                         bmsr |= BMSR_LSTATUS;
4227                 else
4228                         bmsr &= ~BMSR_LSTATUS;
4229         }
4230
4231         err |= tg3_readphy(tp, MII_BMCR, &bmcr);
4232
4233         if ((tp->link_config.autoneg == AUTONEG_ENABLE) && !force_reset &&
4234             (tp->phy_flags & TG3_PHYFLG_PARALLEL_DETECT)) {
4235                 /* do nothing, just check for link up at the end */
4236         } else if (tp->link_config.autoneg == AUTONEG_ENABLE) {
4237                 u32 adv, new_adv;
4238
4239                 err |= tg3_readphy(tp, MII_ADVERTISE, &adv);
4240                 new_adv = adv & ~(ADVERTISE_1000XFULL | ADVERTISE_1000XHALF |
4241                                   ADVERTISE_1000XPAUSE |
4242                                   ADVERTISE_1000XPSE_ASYM |
4243                                   ADVERTISE_SLCT);
4244
4245                 new_adv |= tg3_advert_flowctrl_1000X(tp->link_config.flowctrl);
4246
4247                 if (tp->link_config.advertising & ADVERTISED_1000baseT_Half)
4248                         new_adv |= ADVERTISE_1000XHALF;
4249                 if (tp->link_config.advertising & ADVERTISED_1000baseT_Full)
4250                         new_adv |= ADVERTISE_1000XFULL;
4251
4252                 if ((new_adv != adv) || !(bmcr & BMCR_ANENABLE)) {
4253                         tg3_writephy(tp, MII_ADVERTISE, new_adv);
4254                         bmcr |= BMCR_ANENABLE | BMCR_ANRESTART;
4255                         tg3_writephy(tp, MII_BMCR, bmcr);
4256
4257                         tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
4258                         tp->serdes_counter = SERDES_AN_TIMEOUT_5714S;
4259                         tp->phy_flags &= ~TG3_PHYFLG_PARALLEL_DETECT;
4260
4261                         return err;
4262                 }
4263         } else {
4264                 u32 new_bmcr;
4265
4266                 bmcr &= ~BMCR_SPEED1000;
4267                 new_bmcr = bmcr & ~(BMCR_ANENABLE | BMCR_FULLDPLX);
4268
4269                 if (tp->link_config.duplex == DUPLEX_FULL)
4270                         new_bmcr |= BMCR_FULLDPLX;
4271
4272                 if (new_bmcr != bmcr) {
4273                         /* BMCR_SPEED1000 is a reserved bit that needs
4274                          * to be set on write.
4275                          */
4276                         new_bmcr |= BMCR_SPEED1000;
4277
4278                         /* Force a linkdown */
4279                         if (netif_carrier_ok(tp->dev)) {
4280                                 u32 adv;
4281
4282                                 err |= tg3_readphy(tp, MII_ADVERTISE, &adv);
4283                                 adv &= ~(ADVERTISE_1000XFULL |
4284                                          ADVERTISE_1000XHALF |
4285                                          ADVERTISE_SLCT);
4286                                 tg3_writephy(tp, MII_ADVERTISE, adv);
4287                                 tg3_writephy(tp, MII_BMCR, bmcr |
4288                                                            BMCR_ANRESTART |
4289                                                            BMCR_ANENABLE);
4290                                 udelay(10);
4291                                 netif_carrier_off(tp->dev);
4292                         }
4293                         tg3_writephy(tp, MII_BMCR, new_bmcr);
4294                         bmcr = new_bmcr;
4295                         err |= tg3_readphy(tp, MII_BMSR, &bmsr);
4296                         err |= tg3_readphy(tp, MII_BMSR, &bmsr);
4297                         if (GET_ASIC_REV(tp->pci_chip_rev_id) ==
4298                             ASIC_REV_5714) {
4299                                 if (tr32(MAC_TX_STATUS) & TX_STATUS_LINK_UP)
4300                                         bmsr |= BMSR_LSTATUS;
4301                                 else
4302                                         bmsr &= ~BMSR_LSTATUS;
4303                         }
4304                         tp->phy_flags &= ~TG3_PHYFLG_PARALLEL_DETECT;
4305                 }
4306         }
4307
4308         if (bmsr & BMSR_LSTATUS) {
4309                 current_speed = SPEED_1000;
4310                 current_link_up = 1;
4311                 if (bmcr & BMCR_FULLDPLX)
4312                         current_duplex = DUPLEX_FULL;
4313                 else
4314                         current_duplex = DUPLEX_HALF;
4315
4316                 local_adv = 0;
4317                 remote_adv = 0;
4318
4319                 if (bmcr & BMCR_ANENABLE) {
4320                         u32 common;
4321
4322                         err |= tg3_readphy(tp, MII_ADVERTISE, &local_adv);
4323                         err |= tg3_readphy(tp, MII_LPA, &remote_adv);
4324                         common = local_adv & remote_adv;
4325                         if (common & (ADVERTISE_1000XHALF |
4326                                       ADVERTISE_1000XFULL)) {
4327                                 if (common & ADVERTISE_1000XFULL)
4328                                         current_duplex = DUPLEX_FULL;
4329                                 else
4330                                         current_duplex = DUPLEX_HALF;
4331                         } else if (!tg3_flag(tp, 5780_CLASS)) {
4332                                 /* Link is up via parallel detect */
4333                         } else {
4334                                 current_link_up = 0;
4335                         }
4336                 }
4337         }
4338
4339         if (current_link_up == 1 && current_duplex == DUPLEX_FULL)
4340                 tg3_setup_flow_control(tp, local_adv, remote_adv);
4341
4342         tp->mac_mode &= ~MAC_MODE_HALF_DUPLEX;
4343         if (tp->link_config.active_duplex == DUPLEX_HALF)
4344                 tp->mac_mode |= MAC_MODE_HALF_DUPLEX;
4345
4346         tw32_f(MAC_MODE, tp->mac_mode);
4347         udelay(40);
4348
4349         tw32_f(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
4350
4351         tp->link_config.active_speed = current_speed;
4352         tp->link_config.active_duplex = current_duplex;
4353
4354         if (current_link_up != netif_carrier_ok(tp->dev)) {
4355                 if (current_link_up)
4356                         netif_carrier_on(tp->dev);
4357                 else {
4358                         netif_carrier_off(tp->dev);
4359                         tp->phy_flags &= ~TG3_PHYFLG_PARALLEL_DETECT;
4360                 }
4361                 tg3_link_report(tp);
4362         }
4363         return err;
4364 }
4365
4366 static void tg3_serdes_parallel_detect(struct tg3 *tp)
4367 {
4368         if (tp->serdes_counter) {
4369                 /* Give autoneg time to complete. */
4370                 tp->serdes_counter--;
4371                 return;
4372         }
4373
4374         if (!netif_carrier_ok(tp->dev) &&
4375             (tp->link_config.autoneg == AUTONEG_ENABLE)) {
4376                 u32 bmcr;
4377
4378                 tg3_readphy(tp, MII_BMCR, &bmcr);
4379                 if (bmcr & BMCR_ANENABLE) {
4380                         u32 phy1, phy2;
4381
4382                         /* Select shadow register 0x1f */
4383                         tg3_writephy(tp, MII_TG3_MISC_SHDW, 0x7c00);
4384                         tg3_readphy(tp, MII_TG3_MISC_SHDW, &phy1);
4385
4386                         /* Select expansion interrupt status register */
4387                         tg3_writephy(tp, MII_TG3_DSP_ADDRESS,
4388                                          MII_TG3_DSP_EXP1_INT_STAT);
4389                         tg3_readphy(tp, MII_TG3_DSP_RW_PORT, &phy2);
4390                         tg3_readphy(tp, MII_TG3_DSP_RW_PORT, &phy2);
4391
4392                         if ((phy1 & 0x10) && !(phy2 & 0x20)) {
4393                                 /* We have signal detect and not receiving
4394                                  * config code words, link is up by parallel
4395                                  * detection.
4396                                  */
4397
4398                                 bmcr &= ~BMCR_ANENABLE;
4399                                 bmcr |= BMCR_SPEED1000 | BMCR_FULLDPLX;
4400                                 tg3_writephy(tp, MII_BMCR, bmcr);
4401                                 tp->phy_flags |= TG3_PHYFLG_PARALLEL_DETECT;
4402                         }
4403                 }
4404         } else if (netif_carrier_ok(tp->dev) &&
4405                    (tp->link_config.autoneg == AUTONEG_ENABLE) &&
4406                    (tp->phy_flags & TG3_PHYFLG_PARALLEL_DETECT)) {
4407                 u32 phy2;
4408
4409                 /* Select expansion interrupt status register */
4410                 tg3_writephy(tp, MII_TG3_DSP_ADDRESS,
4411                                  MII_TG3_DSP_EXP1_INT_STAT);
4412                 tg3_readphy(tp, MII_TG3_DSP_RW_PORT, &phy2);
4413                 if (phy2 & 0x20) {
4414                         u32 bmcr;
4415
4416                         /* Config code words received, turn on autoneg. */
4417                         tg3_readphy(tp, MII_BMCR, &bmcr);
4418                         tg3_writephy(tp, MII_BMCR, bmcr | BMCR_ANENABLE);
4419
4420                         tp->phy_flags &= ~TG3_PHYFLG_PARALLEL_DETECT;
4421
4422                 }
4423         }
4424 }
4425
4426 static int tg3_setup_phy(struct tg3 *tp, int force_reset)
4427 {
4428         u32 val;
4429         int err;
4430
4431         if (tp->phy_flags & TG3_PHYFLG_PHY_SERDES)
4432                 err = tg3_setup_fiber_phy(tp, force_reset);
4433         else if (tp->phy_flags & TG3_PHYFLG_MII_SERDES)
4434                 err = tg3_setup_fiber_mii_phy(tp, force_reset);
4435         else
4436                 err = tg3_setup_copper_phy(tp, force_reset);
4437
4438         if (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5784_AX) {
4439                 u32 scale;
4440
4441                 val = tr32(TG3_CPMU_CLCK_STAT) & CPMU_CLCK_STAT_MAC_CLCK_MASK;
4442                 if (val == CPMU_CLCK_STAT_MAC_CLCK_62_5)
4443                         scale = 65;
4444                 else if (val == CPMU_CLCK_STAT_MAC_CLCK_6_25)
4445                         scale = 6;
4446                 else
4447                         scale = 12;
4448
4449                 val = tr32(GRC_MISC_CFG) & ~GRC_MISC_CFG_PRESCALAR_MASK;
4450                 val |= (scale << GRC_MISC_CFG_PRESCALAR_SHIFT);
4451                 tw32(GRC_MISC_CFG, val);
4452         }
4453
4454         val = (2 << TX_LENGTHS_IPG_CRS_SHIFT) |
4455               (6 << TX_LENGTHS_IPG_SHIFT);
4456         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5720)
4457                 val |= tr32(MAC_TX_LENGTHS) &
4458                        (TX_LENGTHS_JMB_FRM_LEN_MSK |
4459                         TX_LENGTHS_CNT_DWN_VAL_MSK);
4460
4461         if (tp->link_config.active_speed == SPEED_1000 &&
4462             tp->link_config.active_duplex == DUPLEX_HALF)
4463                 tw32(MAC_TX_LENGTHS, val |
4464                      (0xff << TX_LENGTHS_SLOT_TIME_SHIFT));
4465         else
4466                 tw32(MAC_TX_LENGTHS, val |
4467                      (32 << TX_LENGTHS_SLOT_TIME_SHIFT));
4468
4469         if (!tg3_flag(tp, 5705_PLUS)) {
4470                 if (netif_carrier_ok(tp->dev)) {
4471                         tw32(HOSTCC_STAT_COAL_TICKS,
4472                              tp->coal.stats_block_coalesce_usecs);
4473                 } else {
4474                         tw32(HOSTCC_STAT_COAL_TICKS, 0);
4475                 }
4476         }
4477
4478         if (tg3_flag(tp, ASPM_WORKAROUND)) {
4479                 val = tr32(PCIE_PWR_MGMT_THRESH);
4480                 if (!netif_carrier_ok(tp->dev))
4481                         val = (val & ~PCIE_PWR_MGMT_L1_THRESH_MSK) |
4482                               tp->pwrmgmt_thresh;
4483                 else
4484                         val |= PCIE_PWR_MGMT_L1_THRESH_MSK;
4485                 tw32(PCIE_PWR_MGMT_THRESH, val);
4486         }
4487
4488         return err;
4489 }
4490
4491 static inline int tg3_irq_sync(struct tg3 *tp)
4492 {
4493         return tp->irq_sync;
4494 }
4495
4496 static inline void tg3_rd32_loop(struct tg3 *tp, u32 *dst, u32 off, u32 len)
4497 {
4498         int i;
4499
4500         dst = (u32 *)((u8 *)dst + off);
4501         for (i = 0; i < len; i += sizeof(u32))
4502                 *dst++ = tr32(off + i);
4503 }
4504
4505 static void tg3_dump_legacy_regs(struct tg3 *tp, u32 *regs)
4506 {
4507         tg3_rd32_loop(tp, regs, TG3PCI_VENDOR, 0xb0);
4508         tg3_rd32_loop(tp, regs, MAILBOX_INTERRUPT_0, 0x200);
4509         tg3_rd32_loop(tp, regs, MAC_MODE, 0x4f0);
4510         tg3_rd32_loop(tp, regs, SNDDATAI_MODE, 0xe0);
4511         tg3_rd32_loop(tp, regs, SNDDATAC_MODE, 0x04);
4512         tg3_rd32_loop(tp, regs, SNDBDS_MODE, 0x80);
4513         tg3_rd32_loop(tp, regs, SNDBDI_MODE, 0x48);
4514         tg3_rd32_loop(tp, regs, SNDBDC_MODE, 0x04);
4515         tg3_rd32_loop(tp, regs, RCVLPC_MODE, 0x20);
4516         tg3_rd32_loop(tp, regs, RCVLPC_SELLST_BASE, 0x15c);
4517         tg3_rd32_loop(tp, regs, RCVDBDI_MODE, 0x0c);
4518         tg3_rd32_loop(tp, regs, RCVDBDI_JUMBO_BD, 0x3c);
4519         tg3_rd32_loop(tp, regs, RCVDBDI_BD_PROD_IDX_0, 0x44);
4520         tg3_rd32_loop(tp, regs, RCVDCC_MODE, 0x04);
4521         tg3_rd32_loop(tp, regs, RCVBDI_MODE, 0x20);
4522         tg3_rd32_loop(tp, regs, RCVCC_MODE, 0x14);
4523         tg3_rd32_loop(tp, regs, RCVLSC_MODE, 0x08);
4524         tg3_rd32_loop(tp, regs, MBFREE_MODE, 0x08);
4525         tg3_rd32_loop(tp, regs, HOSTCC_MODE, 0x100);
4526
4527         if (tg3_flag(tp, SUPPORT_MSIX))
4528                 tg3_rd32_loop(tp, regs, HOSTCC_RXCOL_TICKS_VEC1, 0x180);
4529
4530         tg3_rd32_loop(tp, regs, MEMARB_MODE, 0x10);
4531         tg3_rd32_loop(tp, regs, BUFMGR_MODE, 0x58);
4532         tg3_rd32_loop(tp, regs, RDMAC_MODE, 0x08);
4533         tg3_rd32_loop(tp, regs, WDMAC_MODE, 0x08);
4534         tg3_rd32_loop(tp, regs, RX_CPU_MODE, 0x04);
4535         tg3_rd32_loop(tp, regs, RX_CPU_STATE, 0x04);
4536         tg3_rd32_loop(tp, regs, RX_CPU_PGMCTR, 0x04);
4537         tg3_rd32_loop(tp, regs, RX_CPU_HWBKPT, 0x04);
4538
4539         if (!tg3_flag(tp, 5705_PLUS)) {
4540                 tg3_rd32_loop(tp, regs, TX_CPU_MODE, 0x04);
4541                 tg3_rd32_loop(tp, regs, TX_CPU_STATE, 0x04);
4542                 tg3_rd32_loop(tp, regs, TX_CPU_PGMCTR, 0x04);
4543         }
4544
4545         tg3_rd32_loop(tp, regs, GRCMBOX_INTERRUPT_0, 0x110);
4546         tg3_rd32_loop(tp, regs, FTQ_RESET, 0x120);
4547         tg3_rd32_loop(tp, regs, MSGINT_MODE, 0x0c);
4548         tg3_rd32_loop(tp, regs, DMAC_MODE, 0x04);
4549         tg3_rd32_loop(tp, regs, GRC_MODE, 0x4c);
4550
4551         if (tg3_flag(tp, NVRAM))
4552                 tg3_rd32_loop(tp, regs, NVRAM_CMD, 0x24);
4553 }
4554
4555 static void tg3_dump_state(struct tg3 *tp)
4556 {
4557         int i;
4558         u32 *regs;
4559
4560         regs = kzalloc(TG3_REG_BLK_SIZE, GFP_ATOMIC);
4561         if (!regs) {
4562                 netdev_err(tp->dev, "Failed allocating register dump buffer\n");
4563                 return;
4564         }
4565
4566         if (tg3_flag(tp, PCI_EXPRESS)) {
4567                 /* Read up to but not including private PCI registers */
4568                 for (i = 0; i < TG3_PCIE_TLDLPL_PORT; i += sizeof(u32))
4569                         regs[i / sizeof(u32)] = tr32(i);
4570         } else
4571                 tg3_dump_legacy_regs(tp, regs);
4572
4573         for (i = 0; i < TG3_REG_BLK_SIZE / sizeof(u32); i += 4) {
4574                 if (!regs[i + 0] && !regs[i + 1] &&
4575                     !regs[i + 2] && !regs[i + 3])
4576                         continue;
4577
4578                 netdev_err(tp->dev, "0x%08x: 0x%08x, 0x%08x, 0x%08x, 0x%08x\n",
4579                            i * 4,
4580                            regs[i + 0], regs[i + 1], regs[i + 2], regs[i + 3]);
4581         }
4582
4583         kfree(regs);
4584
4585         for (i = 0; i < tp->irq_cnt; i++) {
4586                 struct tg3_napi *tnapi = &tp->napi[i];
4587
4588                 /* SW status block */
4589                 netdev_err(tp->dev,
4590                          "%d: Host status block [%08x:%08x:(%04x:%04x:%04x):(%04x:%04x)]\n",
4591                            i,
4592                            tnapi->hw_status->status,
4593                            tnapi->hw_status->status_tag,
4594                            tnapi->hw_status->rx_jumbo_consumer,
4595                            tnapi->hw_status->rx_consumer,
4596                            tnapi->hw_status->rx_mini_consumer,
4597                            tnapi->hw_status->idx[0].rx_producer,
4598                            tnapi->hw_status->idx[0].tx_consumer);
4599
4600                 netdev_err(tp->dev,
4601                 "%d: NAPI info [%08x:%08x:(%04x:%04x:%04x):%04x:(%04x:%04x:%04x:%04x)]\n",
4602                            i,
4603                            tnapi->last_tag, tnapi->last_irq_tag,
4604                            tnapi->tx_prod, tnapi->tx_cons, tnapi->tx_pending,
4605                            tnapi->rx_rcb_ptr,
4606                            tnapi->prodring.rx_std_prod_idx,
4607                            tnapi->prodring.rx_std_cons_idx,
4608                            tnapi->prodring.rx_jmb_prod_idx,
4609                            tnapi->prodring.rx_jmb_cons_idx);
4610         }
4611 }
4612
4613 /* This is called whenever we suspect that the system chipset is re-
4614  * ordering the sequence of MMIO to the tx send mailbox. The symptom
4615  * is bogus tx completions. We try to recover by setting the
4616  * TG3_FLAG_MBOX_WRITE_REORDER flag and resetting the chip later
4617  * in the workqueue.
4618  */
4619 static void tg3_tx_recover(struct tg3 *tp)
4620 {
4621         BUG_ON(tg3_flag(tp, MBOX_WRITE_REORDER) ||
4622                tp->write32_tx_mbox == tg3_write_indirect_mbox);
4623
4624         netdev_warn(tp->dev,
4625                     "The system may be re-ordering memory-mapped I/O "
4626                     "cycles to the network device, attempting to recover. "
4627                     "Please report the problem to the driver maintainer "
4628                     "and include system chipset information.\n");
4629
4630         spin_lock(&tp->lock);
4631         tg3_flag_set(tp, TX_RECOVERY_PENDING);
4632         spin_unlock(&tp->lock);
4633 }
4634
4635 static inline u32 tg3_tx_avail(struct tg3_napi *tnapi)
4636 {
4637         /* Tell compiler to fetch tx indices from memory. */
4638         barrier();
4639         return tnapi->tx_pending -
4640                ((tnapi->tx_prod - tnapi->tx_cons) & (TG3_TX_RING_SIZE - 1));
4641 }
4642
4643 /* Tigon3 never reports partial packet sends.  So we do not
4644  * need special logic to handle SKBs that have not had all
4645  * of their frags sent yet, like SunGEM does.
4646  */
4647 static void tg3_tx(struct tg3_napi *tnapi)
4648 {
4649         struct tg3 *tp = tnapi->tp;
4650         u32 hw_idx = tnapi->hw_status->idx[0].tx_consumer;
4651         u32 sw_idx = tnapi->tx_cons;
4652         struct netdev_queue *txq;
4653         int index = tnapi - tp->napi;
4654
4655         if (tg3_flag(tp, ENABLE_TSS))
4656                 index--;
4657
4658         txq = netdev_get_tx_queue(tp->dev, index);
4659
4660         while (sw_idx != hw_idx) {
4661                 struct ring_info *ri = &tnapi->tx_buffers[sw_idx];
4662                 struct sk_buff *skb = ri->skb;
4663                 int i, tx_bug = 0;
4664
4665                 if (unlikely(skb == NULL)) {
4666                         tg3_tx_recover(tp);
4667                         return;
4668                 }
4669
4670                 pci_unmap_single(tp->pdev,
4671                                  dma_unmap_addr(ri, mapping),
4672                                  skb_headlen(skb),
4673                                  PCI_DMA_TODEVICE);
4674
4675                 ri->skb = NULL;
4676
4677                 sw_idx = NEXT_TX(sw_idx);
4678
4679                 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
4680                         ri = &tnapi->tx_buffers[sw_idx];
4681                         if (unlikely(ri->skb != NULL || sw_idx == hw_idx))
4682                                 tx_bug = 1;
4683
4684                         pci_unmap_page(tp->pdev,
4685                                        dma_unmap_addr(ri, mapping),
4686                                        skb_shinfo(skb)->frags[i].size,
4687                                        PCI_DMA_TODEVICE);
4688                         sw_idx = NEXT_TX(sw_idx);
4689                 }
4690
4691                 dev_kfree_skb(skb);
4692
4693                 if (unlikely(tx_bug)) {
4694                         tg3_tx_recover(tp);
4695                         return;
4696                 }
4697         }
4698
4699         tnapi->tx_cons = sw_idx;
4700
4701         /* Need to make the tx_cons update visible to tg3_start_xmit()
4702          * before checking for netif_queue_stopped().  Without the
4703          * memory barrier, there is a small possibility that tg3_start_xmit()
4704          * will miss it and cause the queue to be stopped forever.
4705          */
4706         smp_mb();
4707
4708         if (unlikely(netif_tx_queue_stopped(txq) &&
4709                      (tg3_tx_avail(tnapi) > TG3_TX_WAKEUP_THRESH(tnapi)))) {
4710                 __netif_tx_lock(txq, smp_processor_id());
4711                 if (netif_tx_queue_stopped(txq) &&
4712                     (tg3_tx_avail(tnapi) > TG3_TX_WAKEUP_THRESH(tnapi)))
4713                         netif_tx_wake_queue(txq);
4714                 __netif_tx_unlock(txq);
4715         }
4716 }
4717
4718 static void tg3_rx_skb_free(struct tg3 *tp, struct ring_info *ri, u32 map_sz)
4719 {
4720         if (!ri->skb)
4721                 return;
4722
4723         pci_unmap_single(tp->pdev, dma_unmap_addr(ri, mapping),
4724                          map_sz, PCI_DMA_FROMDEVICE);
4725         dev_kfree_skb_any(ri->skb);
4726         ri->skb = NULL;
4727 }
4728
4729 /* Returns size of skb allocated or < 0 on error.
4730  *
4731  * We only need to fill in the address because the other members
4732  * of the RX descriptor are invariant, see tg3_init_rings.
4733  *
4734  * Note the purposeful assymetry of cpu vs. chip accesses.  For
4735  * posting buffers we only dirty the first cache line of the RX
4736  * descriptor (containing the address).  Whereas for the RX status
4737  * buffers the cpu only reads the last cacheline of the RX descriptor
4738  * (to fetch the error flags, vlan tag, checksum, and opaque cookie).
4739  */
4740 static int tg3_alloc_rx_skb(struct tg3 *tp, struct tg3_rx_prodring_set *tpr,
4741                             u32 opaque_key, u32 dest_idx_unmasked)
4742 {
4743         struct tg3_rx_buffer_desc *desc;
4744         struct ring_info *map;
4745         struct sk_buff *skb;
4746         dma_addr_t mapping;
4747         int skb_size, dest_idx;
4748
4749         switch (opaque_key) {
4750         case RXD_OPAQUE_RING_STD:
4751                 dest_idx = dest_idx_unmasked & tp->rx_std_ring_mask;
4752                 desc = &tpr->rx_std[dest_idx];
4753                 map = &tpr->rx_std_buffers[dest_idx];
4754                 skb_size = tp->rx_pkt_map_sz;
4755                 break;
4756
4757         case RXD_OPAQUE_RING_JUMBO:
4758                 dest_idx = dest_idx_unmasked & tp->rx_jmb_ring_mask;
4759                 desc = &tpr->rx_jmb[dest_idx].std;
4760                 map = &tpr->rx_jmb_buffers[dest_idx];
4761                 skb_size = TG3_RX_JMB_MAP_SZ;
4762                 break;
4763
4764         default:
4765                 return -EINVAL;
4766         }
4767
4768         /* Do not overwrite any of the map or rp information
4769          * until we are sure we can commit to a new buffer.
4770          *
4771          * Callers depend upon this behavior and assume that
4772          * we leave everything unchanged if we fail.
4773          */
4774         skb = netdev_alloc_skb(tp->dev, skb_size + tp->rx_offset);
4775         if (skb == NULL)
4776                 return -ENOMEM;
4777
4778         skb_reserve(skb, tp->rx_offset);
4779
4780         mapping = pci_map_single(tp->pdev, skb->data, skb_size,
4781                                  PCI_DMA_FROMDEVICE);
4782         if (pci_dma_mapping_error(tp->pdev, mapping)) {
4783                 dev_kfree_skb(skb);
4784                 return -EIO;
4785         }
4786
4787         map->skb = skb;
4788         dma_unmap_addr_set(map, mapping, mapping);
4789
4790         desc->addr_hi = ((u64)mapping >> 32);
4791         desc->addr_lo = ((u64)mapping & 0xffffffff);
4792
4793         return skb_size;
4794 }
4795
4796 /* We only need to move over in the address because the other
4797  * members of the RX descriptor are invariant.  See notes above
4798  * tg3_alloc_rx_skb for full details.
4799  */
4800 static void tg3_recycle_rx(struct tg3_napi *tnapi,
4801                            struct tg3_rx_prodring_set *dpr,
4802                            u32 opaque_key, int src_idx,
4803                            u32 dest_idx_unmasked)
4804 {
4805         struct tg3 *tp = tnapi->tp;
4806         struct tg3_rx_buffer_desc *src_desc, *dest_desc;
4807         struct ring_info *src_map, *dest_map;
4808         struct tg3_rx_prodring_set *spr = &tp->napi[0].prodring;
4809         int dest_idx;
4810
4811         switch (opaque_key) {
4812         case RXD_OPAQUE_RING_STD:
4813                 dest_idx = dest_idx_unmasked & tp->rx_std_ring_mask;
4814                 dest_desc = &dpr->rx_std[dest_idx];
4815                 dest_map = &dpr->rx_std_buffers[dest_idx];
4816                 src_desc = &spr->rx_std[src_idx];
4817                 src_map = &spr->rx_std_buffers[src_idx];
4818                 break;
4819
4820         case RXD_OPAQUE_RING_JUMBO:
4821                 dest_idx = dest_idx_unmasked & tp->rx_jmb_ring_mask;
4822                 dest_desc = &dpr->rx_jmb[dest_idx].std;
4823                 dest_map = &dpr->rx_jmb_buffers[dest_idx];
4824                 src_desc = &spr->rx_jmb[src_idx].std;
4825                 src_map = &spr->rx_jmb_buffers[src_idx];
4826                 break;
4827
4828         default:
4829                 return;
4830         }
4831
4832         dest_map->skb = src_map->skb;
4833         dma_unmap_addr_set(dest_map, mapping,
4834                            dma_unmap_addr(src_map, mapping));
4835         dest_desc->addr_hi = src_desc->addr_hi;
4836         dest_desc->addr_lo = src_desc->addr_lo;
4837
4838         /* Ensure that the update to the skb happens after the physical
4839          * addresses have been transferred to the new BD location.
4840          */
4841         smp_wmb();
4842
4843         src_map->skb = NULL;
4844 }
4845
4846 /* The RX ring scheme is composed of multiple rings which post fresh
4847  * buffers to the chip, and one special ring the chip uses to report
4848  * status back to the host.
4849  *
4850  * The special ring reports the status of received packets to the
4851  * host.  The chip does not write into the original descriptor the
4852  * RX buffer was obtained from.  The chip simply takes the original
4853  * descriptor as provided by the host, updates the status and length
4854  * field, then writes this into the next status ring entry.
4855  *
4856  * Each ring the host uses to post buffers to the chip is described
4857  * by a TG3_BDINFO entry in the chips SRAM area.  When a packet arrives,
4858  * it is first placed into the on-chip ram.  When the packet's length
4859  * is known, it walks down the TG3_BDINFO entries to select the ring.
4860  * Each TG3_BDINFO specifies a MAXLEN field and the first TG3_BDINFO
4861  * which is within the range of the new packet's length is chosen.
4862  *
4863  * The "separate ring for rx status" scheme may sound queer, but it makes
4864  * sense from a cache coherency perspective.  If only the host writes
4865  * to the buffer post rings, and only the chip writes to the rx status
4866  * rings, then cache lines never move beyond shared-modified state.
4867  * If both the host and chip were to write into the same ring, cache line
4868  * eviction could occur since both entities want it in an exclusive state.
4869  */
4870 static int tg3_rx(struct tg3_napi *tnapi, int budget)
4871 {
4872         struct tg3 *tp = tnapi->tp;
4873         u32 work_mask, rx_std_posted = 0;
4874         u32 std_prod_idx, jmb_prod_idx;
4875         u32 sw_idx = tnapi->rx_rcb_ptr;
4876         u16 hw_idx;
4877         int received;
4878         struct tg3_rx_prodring_set *tpr = &tnapi->prodring;
4879
4880         hw_idx = *(tnapi->rx_rcb_prod_idx);
4881         /*
4882          * We need to order the read of hw_idx and the read of
4883          * the opaque cookie.
4884          */
4885         rmb();
4886         work_mask = 0;
4887         received = 0;
4888         std_prod_idx = tpr->rx_std_prod_idx;
4889         jmb_prod_idx = tpr->rx_jmb_prod_idx;
4890         while (sw_idx != hw_idx && budget > 0) {
4891                 struct ring_info *ri;
4892                 struct tg3_rx_buffer_desc *desc = &tnapi->rx_rcb[sw_idx];
4893                 unsigned int len;
4894                 struct sk_buff *skb;
4895                 dma_addr_t dma_addr;
4896                 u32 opaque_key, desc_idx, *post_ptr;
4897
4898                 desc_idx = desc->opaque & RXD_OPAQUE_INDEX_MASK;
4899                 opaque_key = desc->opaque & RXD_OPAQUE_RING_MASK;
4900                 if (opaque_key == RXD_OPAQUE_RING_STD) {
4901                         ri = &tp->napi[0].prodring.rx_std_buffers[desc_idx];
4902                         dma_addr = dma_unmap_addr(ri, mapping);
4903                         skb = ri->skb;
4904                         post_ptr = &std_prod_idx;
4905                         rx_std_posted++;
4906                 } else if (opaque_key == RXD_OPAQUE_RING_JUMBO) {
4907                         ri = &tp->napi[0].prodring.rx_jmb_buffers[desc_idx];
4908                         dma_addr = dma_unmap_addr(ri, mapping);
4909                         skb = ri->skb;
4910                         post_ptr = &jmb_prod_idx;
4911                 } else
4912                         goto next_pkt_nopost;
4913
4914                 work_mask |= opaque_key;
4915
4916                 if ((desc->err_vlan & RXD_ERR_MASK) != 0 &&
4917                     (desc->err_vlan != RXD_ERR_ODD_NIBBLE_RCVD_MII)) {
4918                 drop_it:
4919                         tg3_recycle_rx(tnapi, tpr, opaque_key,
4920                                        desc_idx, *post_ptr);
4921                 drop_it_no_recycle:
4922                         /* Other statistics kept track of by card. */
4923                         tp->rx_dropped++;
4924                         goto next_pkt;
4925                 }
4926
4927                 len = ((desc->idx_len & RXD_LEN_MASK) >> RXD_LEN_SHIFT) -
4928                       ETH_FCS_LEN;
4929
4930                 if (len > TG3_RX_COPY_THRESH(tp)) {
4931                         int skb_size;
4932
4933                         skb_size = tg3_alloc_rx_skb(tp, tpr, opaque_key,
4934                                                     *post_ptr);
4935                         if (skb_size < 0)
4936                                 goto drop_it;
4937
4938                         pci_unmap_single(tp->pdev, dma_addr, skb_size,
4939                                          PCI_DMA_FROMDEVICE);
4940
4941                         /* Ensure that the update to the skb happens
4942                          * after the usage of the old DMA mapping.
4943                          */
4944                         smp_wmb();
4945
4946                         ri->skb = NULL;
4947
4948                         skb_put(skb, len);
4949                 } else {
4950                         struct sk_buff *copy_skb;
4951
4952                         tg3_recycle_rx(tnapi, tpr, opaque_key,
4953                                        desc_idx, *post_ptr);
4954
4955                         copy_skb = netdev_alloc_skb(tp->dev, len +
4956                                                     TG3_RAW_IP_ALIGN);
4957                         if (copy_skb == NULL)
4958                                 goto drop_it_no_recycle;
4959
4960                         skb_reserve(copy_skb, TG3_RAW_IP_ALIGN);
4961                         skb_put(copy_skb, len);
4962                         pci_dma_sync_single_for_cpu(tp->pdev, dma_addr, len, PCI_DMA_FROMDEVICE);
4963                         skb_copy_from_linear_data(skb, copy_skb->data, len);
4964                         pci_dma_sync_single_for_device(tp->pdev, dma_addr, len, PCI_DMA_FROMDEVICE);
4965
4966                         /* We'll reuse the original ring buffer. */
4967                         skb = copy_skb;
4968                 }
4969
4970                 if ((tp->dev->features & NETIF_F_RXCSUM) &&
4971                     (desc->type_flags & RXD_FLAG_TCPUDP_CSUM) &&
4972                     (((desc->ip_tcp_csum & RXD_TCPCSUM_MASK)
4973                       >> RXD_TCPCSUM_SHIFT) == 0xffff))
4974                         skb->ip_summed = CHECKSUM_UNNECESSARY;
4975                 else
4976                         skb_checksum_none_assert(skb);
4977
4978                 skb->protocol = eth_type_trans(skb, tp->dev);
4979
4980                 if (len > (tp->dev->mtu + ETH_HLEN) &&
4981                     skb->protocol != htons(ETH_P_8021Q)) {
4982                         dev_kfree_skb(skb);
4983                         goto drop_it_no_recycle;
4984                 }
4985
4986                 if (desc->type_flags & RXD_FLAG_VLAN &&
4987                     !(tp->rx_mode & RX_MODE_KEEP_VLAN_TAG))
4988                         __vlan_hwaccel_put_tag(skb,
4989                                                desc->err_vlan & RXD_VLAN_MASK);
4990
4991                 napi_gro_receive(&tnapi->napi, skb);
4992
4993                 received++;
4994                 budget--;
4995
4996 next_pkt:
4997                 (*post_ptr)++;
4998
4999                 if (unlikely(rx_std_posted >= tp->rx_std_max_post)) {
5000                         tpr->rx_std_prod_idx = std_prod_idx &
5001                                                tp->rx_std_ring_mask;
5002                         tw32_rx_mbox(TG3_RX_STD_PROD_IDX_REG,
5003                                      tpr->rx_std_prod_idx);
5004                         work_mask &= ~RXD_OPAQUE_RING_STD;
5005                         rx_std_posted = 0;
5006                 }
5007 next_pkt_nopost:
5008                 sw_idx++;
5009                 sw_idx &= tp->rx_ret_ring_mask;
5010
5011                 /* Refresh hw_idx to see if there is new work */
5012                 if (sw_idx == hw_idx) {
5013                         hw_idx = *(tnapi->rx_rcb_prod_idx);
5014                         rmb();
5015                 }
5016         }
5017
5018         /* ACK the status ring. */
5019         tnapi->rx_rcb_ptr = sw_idx;
5020         tw32_rx_mbox(tnapi->consmbox, sw_idx);
5021
5022         /* Refill RX ring(s). */
5023         if (!tg3_flag(tp, ENABLE_RSS)) {
5024                 if (work_mask & RXD_OPAQUE_RING_STD) {
5025                         tpr->rx_std_prod_idx = std_prod_idx &
5026                                                tp->rx_std_ring_mask;
5027                         tw32_rx_mbox(TG3_RX_STD_PROD_IDX_REG,
5028                                      tpr->rx_std_prod_idx);
5029                 }
5030                 if (work_mask & RXD_OPAQUE_RING_JUMBO) {
5031                         tpr->rx_jmb_prod_idx = jmb_prod_idx &
5032                                                tp->rx_jmb_ring_mask;
5033                         tw32_rx_mbox(TG3_RX_JMB_PROD_IDX_REG,
5034                                      tpr->rx_jmb_prod_idx);
5035                 }
5036                 mmiowb();
5037         } else if (work_mask) {
5038                 /* rx_std_buffers[] and rx_jmb_buffers[] entries must be
5039                  * updated before the producer indices can be updated.
5040                  */
5041                 smp_wmb();
5042
5043                 tpr->rx_std_prod_idx = std_prod_idx & tp->rx_std_ring_mask;
5044                 tpr->rx_jmb_prod_idx = jmb_prod_idx & tp->rx_jmb_ring_mask;
5045
5046                 if (tnapi != &tp->napi[1])
5047                         napi_schedule(&tp->napi[1].napi);
5048         }
5049
5050         return received;
5051 }
5052
5053 static void tg3_poll_link(struct tg3 *tp)
5054 {
5055         /* handle link change and other phy events */
5056         if (!(tg3_flag(tp, USE_LINKCHG_REG) || tg3_flag(tp, POLL_SERDES))) {
5057                 struct tg3_hw_status *sblk = tp->napi[0].hw_status;
5058
5059                 if (sblk->status & SD_STATUS_LINK_CHG) {
5060                         sblk->status = SD_STATUS_UPDATED |
5061                                        (sblk->status & ~SD_STATUS_LINK_CHG);
5062                         spin_lock(&tp->lock);
5063                         if (tg3_flag(tp, USE_PHYLIB)) {
5064                                 tw32_f(MAC_STATUS,
5065                                      (MAC_STATUS_SYNC_CHANGED |
5066                                       MAC_STATUS_CFG_CHANGED |
5067                                       MAC_STATUS_MI_COMPLETION |
5068                                       MAC_STATUS_LNKSTATE_CHANGED));
5069                                 udelay(40);
5070                         } else
5071                                 tg3_setup_phy(tp, 0);
5072                         spin_unlock(&tp->lock);
5073                 }
5074         }
5075 }
5076
5077 static int tg3_rx_prodring_xfer(struct tg3 *tp,
5078                                 struct tg3_rx_prodring_set *dpr,
5079                                 struct tg3_rx_prodring_set *spr)
5080 {
5081         u32 si, di, cpycnt, src_prod_idx;
5082         int i, err = 0;
5083
5084         while (1) {
5085                 src_prod_idx = spr->rx_std_prod_idx;
5086
5087                 /* Make sure updates to the rx_std_buffers[] entries and the
5088                  * standard producer index are seen in the correct order.
5089                  */
5090                 smp_rmb();
5091
5092                 if (spr->rx_std_cons_idx == src_prod_idx)
5093                         break;
5094
5095                 if (spr->rx_std_cons_idx < src_prod_idx)
5096                         cpycnt = src_prod_idx - spr->rx_std_cons_idx;
5097                 else
5098                         cpycnt = tp->rx_std_ring_mask + 1 -
5099                                  spr->rx_std_cons_idx;
5100
5101                 cpycnt = min(cpycnt,
5102                              tp->rx_std_ring_mask + 1 - dpr->rx_std_prod_idx);
5103
5104                 si = spr->rx_std_cons_idx;
5105                 di = dpr->rx_std_prod_idx;
5106
5107                 for (i = di; i < di + cpycnt; i++) {
5108                         if (dpr->rx_std_buffers[i].skb) {
5109                                 cpycnt = i - di;
5110                                 err = -ENOSPC;
5111                                 break;
5112                         }
5113                 }
5114
5115                 if (!cpycnt)
5116                         break;
5117
5118                 /* Ensure that updates to the rx_std_buffers ring and the
5119                  * shadowed hardware producer ring from tg3_recycle_skb() are
5120                  * ordered correctly WRT the skb check above.
5121                  */
5122                 smp_rmb();
5123
5124                 memcpy(&dpr->rx_std_buffers[di],
5125                        &spr->rx_std_buffers[si],
5126                        cpycnt * sizeof(struct ring_info));
5127
5128                 for (i = 0; i < cpycnt; i++, di++, si++) {
5129                         struct tg3_rx_buffer_desc *sbd, *dbd;
5130                         sbd = &spr->rx_std[si];
5131                         dbd = &dpr->rx_std[di];
5132                         dbd->addr_hi = sbd->addr_hi;
5133                         dbd->addr_lo = sbd->addr_lo;
5134                 }
5135
5136                 spr->rx_std_cons_idx = (spr->rx_std_cons_idx + cpycnt) &
5137                                        tp->rx_std_ring_mask;
5138                 dpr->rx_std_prod_idx = (dpr->rx_std_prod_idx + cpycnt) &
5139                                        tp->rx_std_ring_mask;
5140         }
5141
5142         while (1) {
5143                 src_prod_idx = spr->rx_jmb_prod_idx;
5144
5145                 /* Make sure updates to the rx_jmb_buffers[] entries and
5146                  * the jumbo producer index are seen in the correct order.
5147                  */
5148                 smp_rmb();
5149
5150                 if (spr->rx_jmb_cons_idx == src_prod_idx)
5151                         break;
5152
5153                 if (spr->rx_jmb_cons_idx < src_prod_idx)
5154                         cpycnt = src_prod_idx - spr->rx_jmb_cons_idx;
5155                 else
5156                         cpycnt = tp->rx_jmb_ring_mask + 1 -
5157                                  spr->rx_jmb_cons_idx;
5158
5159                 cpycnt = min(cpycnt,
5160                              tp->rx_jmb_ring_mask + 1 - dpr->rx_jmb_prod_idx);
5161
5162                 si = spr->rx_jmb_cons_idx;
5163                 di = dpr->rx_jmb_prod_idx;
5164
5165                 for (i = di; i < di + cpycnt; i++) {
5166                         if (dpr->rx_jmb_buffers[i].skb) {
5167                                 cpycnt = i - di;
5168                                 err = -ENOSPC;
5169                                 break;
5170                         }
5171                 }
5172
5173                 if (!cpycnt)
5174                         break;
5175
5176                 /* Ensure that updates to the rx_jmb_buffers ring and the
5177                  * shadowed hardware producer ring from tg3_recycle_skb() are
5178                  * ordered correctly WRT the skb check above.
5179                  */
5180                 smp_rmb();
5181
5182                 memcpy(&dpr->rx_jmb_buffers[di],
5183                        &spr->rx_jmb_buffers[si],
5184                        cpycnt * sizeof(struct ring_info));
5185
5186                 for (i = 0; i < cpycnt; i++, di++, si++) {
5187                         struct tg3_rx_buffer_desc *sbd, *dbd;
5188                         sbd = &spr->rx_jmb[si].std;
5189                         dbd = &dpr->rx_jmb[di].std;
5190                         dbd->addr_hi = sbd->addr_hi;
5191                         dbd->addr_lo = sbd->addr_lo;
5192                 }
5193
5194                 spr->rx_jmb_cons_idx = (spr->rx_jmb_cons_idx + cpycnt) &
5195                                        tp->rx_jmb_ring_mask;
5196                 dpr->rx_jmb_prod_idx = (dpr->rx_jmb_prod_idx + cpycnt) &
5197                                        tp->rx_jmb_ring_mask;
5198         }
5199
5200         return err;
5201 }
5202
5203 static int tg3_poll_work(struct tg3_napi *tnapi, int work_done, int budget)
5204 {
5205         struct tg3 *tp = tnapi->tp;
5206
5207         /* run TX completion thread */
5208         if (tnapi->hw_status->idx[0].tx_consumer != tnapi->tx_cons) {
5209                 tg3_tx(tnapi);
5210                 if (unlikely(tg3_flag(tp, TX_RECOVERY_PENDING)))
5211                         return work_done;
5212         }
5213
5214         /* run RX thread, within the bounds set by NAPI.
5215          * All RX "locking" is done by ensuring outside
5216          * code synchronizes with tg3->napi.poll()
5217          */
5218         if (*(tnapi->rx_rcb_prod_idx) != tnapi->rx_rcb_ptr)
5219                 work_done += tg3_rx(tnapi, budget - work_done);
5220
5221         if (tg3_flag(tp, ENABLE_RSS) && tnapi == &tp->napi[1]) {
5222                 struct tg3_rx_prodring_set *dpr = &tp->napi[0].prodring;
5223                 int i, err = 0;
5224                 u32 std_prod_idx = dpr->rx_std_prod_idx;
5225                 u32 jmb_prod_idx = dpr->rx_jmb_prod_idx;
5226
5227                 for (i = 1; i < tp->irq_cnt; i++)
5228                         err |= tg3_rx_prodring_xfer(tp, dpr,
5229                                                     &tp->napi[i].prodring);
5230
5231                 wmb();
5232
5233                 if (std_prod_idx != dpr->rx_std_prod_idx)
5234                         tw32_rx_mbox(TG3_RX_STD_PROD_IDX_REG,
5235                                      dpr->rx_std_prod_idx);
5236
5237                 if (jmb_prod_idx != dpr->rx_jmb_prod_idx)
5238                         tw32_rx_mbox(TG3_RX_JMB_PROD_IDX_REG,
5239                                      dpr->rx_jmb_prod_idx);
5240
5241                 mmiowb();
5242
5243                 if (err)
5244                         tw32_f(HOSTCC_MODE, tp->coal_now);
5245         }
5246
5247         return work_done;
5248 }
5249
5250 static int tg3_poll_msix(struct napi_struct *napi, int budget)
5251 {
5252         struct tg3_napi *tnapi = container_of(napi, struct tg3_napi, napi);
5253         struct tg3 *tp = tnapi->tp;
5254         int work_done = 0;
5255         struct tg3_hw_status *sblk = tnapi->hw_status;
5256
5257         while (1) {
5258                 work_done = tg3_poll_work(tnapi, work_done, budget);
5259
5260                 if (unlikely(tg3_flag(tp, TX_RECOVERY_PENDING)))
5261                         goto tx_recovery;
5262
5263                 if (unlikely(work_done >= budget))
5264                         break;
5265
5266                 /* tp->last_tag is used in tg3_int_reenable() below
5267                  * to tell the hw how much work has been processed,
5268                  * so we must read it before checking for more work.
5269                  */
5270                 tnapi->last_tag = sblk->status_tag;
5271                 tnapi->last_irq_tag = tnapi->last_tag;
5272                 rmb();
5273
5274                 /* check for RX/TX work to do */
5275                 if (likely(sblk->idx[0].tx_consumer == tnapi->tx_cons &&
5276                            *(tnapi->rx_rcb_prod_idx) == tnapi->rx_rcb_ptr)) {
5277                         napi_complete(napi);
5278                         /* Reenable interrupts. */
5279                         tw32_mailbox(tnapi->int_mbox, tnapi->last_tag << 24);
5280                         mmiowb();
5281                         break;
5282                 }
5283         }
5284
5285         return work_done;
5286
5287 tx_recovery:
5288         /* work_done is guaranteed to be less than budget. */
5289         napi_complete(napi);
5290         schedule_work(&tp->reset_task);
5291         return work_done;
5292 }
5293
5294 static void tg3_process_error(struct tg3 *tp)
5295 {
5296         u32 val;
5297         bool real_error = false;
5298
5299         if (tg3_flag(tp, ERROR_PROCESSED))
5300                 return;
5301
5302         /* Check Flow Attention register */
5303         val = tr32(HOSTCC_FLOW_ATTN);
5304         if (val & ~HOSTCC_FLOW_ATTN_MBUF_LWM) {
5305                 netdev_err(tp->dev, "FLOW Attention error.  Resetting chip.\n");
5306                 real_error = true;
5307         }
5308
5309         if (tr32(MSGINT_STATUS) & ~MSGINT_STATUS_MSI_REQ) {
5310                 netdev_err(tp->dev, "MSI Status error.  Resetting chip.\n");
5311                 real_error = true;
5312         }
5313
5314         if (tr32(RDMAC_STATUS) || tr32(WDMAC_STATUS)) {
5315                 netdev_err(tp->dev, "DMA Status error.  Resetting chip.\n");
5316                 real_error = true;
5317         }
5318
5319         if (!real_error)
5320                 return;
5321
5322         tg3_dump_state(tp);
5323
5324         tg3_flag_set(tp, ERROR_PROCESSED);
5325         schedule_work(&tp->reset_task);
5326 }
5327
5328 static int tg3_poll(struct napi_struct *napi, int budget)
5329 {
5330         struct tg3_napi *tnapi = container_of(napi, struct tg3_napi, napi);
5331         struct tg3 *tp = tnapi->tp;
5332         int work_done = 0;
5333         struct tg3_hw_status *sblk = tnapi->hw_status;
5334
5335         while (1) {
5336                 if (sblk->status & SD_STATUS_ERROR)
5337                         tg3_process_error(tp);
5338
5339                 tg3_poll_link(tp);
5340
5341                 work_done = tg3_poll_work(tnapi, work_done, budget);
5342
5343                 if (unlikely(tg3_flag(tp, TX_RECOVERY_PENDING)))
5344                         goto tx_recovery;
5345
5346                 if (unlikely(work_done >= budget))
5347                         break;
5348
5349                 if (tg3_flag(tp, TAGGED_STATUS)) {
5350                         /* tp->last_tag is used in tg3_int_reenable() below
5351                          * to tell the hw how much work has been processed,
5352                          * so we must read it before checking for more work.
5353                          */
5354                         tnapi->last_tag = sblk->status_tag;
5355                         tnapi->last_irq_tag = tnapi->last_tag;
5356                         rmb();
5357                 } else
5358                         sblk->status &= ~SD_STATUS_UPDATED;
5359
5360                 if (likely(!tg3_has_work(tnapi))) {
5361                         napi_complete(napi);
5362                         tg3_int_reenable(tnapi);
5363                         break;
5364                 }
5365         }
5366
5367         return work_done;
5368
5369 tx_recovery:
5370         /* work_done is guaranteed to be less than budget. */
5371         napi_complete(napi);
5372         schedule_work(&tp->reset_task);
5373         return work_done;
5374 }
5375
5376 static void tg3_napi_disable(struct tg3 *tp)
5377 {
5378         int i;
5379
5380         for (i = tp->irq_cnt - 1; i >= 0; i--)
5381                 napi_disable(&tp->napi[i].napi);
5382 }
5383
5384 static void tg3_napi_enable(struct tg3 *tp)
5385 {
5386         int i;
5387
5388         for (i = 0; i < tp->irq_cnt; i++)
5389                 napi_enable(&tp->napi[i].napi);
5390 }
5391
5392 static void tg3_napi_init(struct tg3 *tp)
5393 {
5394         int i;
5395
5396         netif_napi_add(tp->dev, &tp->napi[0].napi, tg3_poll, 64);
5397         for (i = 1; i < tp->irq_cnt; i++)
5398                 netif_napi_add(tp->dev, &tp->napi[i].napi, tg3_poll_msix, 64);
5399 }
5400
5401 static void tg3_napi_fini(struct tg3 *tp)
5402 {
5403         int i;
5404
5405         for (i = 0; i < tp->irq_cnt; i++)
5406                 netif_napi_del(&tp->napi[i].napi);
5407 }
5408
5409 static inline void tg3_netif_stop(struct tg3 *tp)
5410 {
5411         tp->dev->trans_start = jiffies; /* prevent tx timeout */
5412         tg3_napi_disable(tp);
5413         netif_tx_disable(tp->dev);
5414 }
5415
5416 static inline void tg3_netif_start(struct tg3 *tp)
5417 {
5418         /* NOTE: unconditional netif_tx_wake_all_queues is only
5419          * appropriate so long as all callers are assured to
5420          * have free tx slots (such as after tg3_init_hw)
5421          */
5422         netif_tx_wake_all_queues(tp->dev);
5423
5424         tg3_napi_enable(tp);
5425         tp->napi[0].hw_status->status |= SD_STATUS_UPDATED;
5426         tg3_enable_ints(tp);
5427 }
5428
5429 static void tg3_irq_quiesce(struct tg3 *tp)
5430 {
5431         int i;
5432
5433         BUG_ON(tp->irq_sync);
5434
5435         tp->irq_sync = 1;
5436         smp_mb();
5437
5438         for (i = 0; i < tp->irq_cnt; i++)
5439                 synchronize_irq(tp->napi[i].irq_vec);
5440 }
5441
5442 /* Fully shutdown all tg3 driver activity elsewhere in the system.
5443  * If irq_sync is non-zero, then the IRQ handler must be synchronized
5444  * with as well.  Most of the time, this is not necessary except when
5445  * shutting down the device.
5446  */
5447 static inline void tg3_full_lock(struct tg3 *tp, int irq_sync)
5448 {
5449         spin_lock_bh(&tp->lock);
5450         if (irq_sync)
5451                 tg3_irq_quiesce(tp);
5452 }
5453
5454 static inline void tg3_full_unlock(struct tg3 *tp)
5455 {
5456         spin_unlock_bh(&tp->lock);
5457 }
5458
5459 /* One-shot MSI handler - Chip automatically disables interrupt
5460  * after sending MSI so driver doesn't have to do it.
5461  */
5462 static irqreturn_t tg3_msi_1shot(int irq, void *dev_id)
5463 {
5464         struct tg3_napi *tnapi = dev_id;
5465         struct tg3 *tp = tnapi->tp;
5466
5467         prefetch(tnapi->hw_status);
5468         if (tnapi->rx_rcb)
5469                 prefetch(&tnapi->rx_rcb[tnapi->rx_rcb_ptr]);
5470
5471         if (likely(!tg3_irq_sync(tp)))
5472                 napi_schedule(&tnapi->napi);
5473
5474         return IRQ_HANDLED;
5475 }
5476
5477 /* MSI ISR - No need to check for interrupt sharing and no need to
5478  * flush status block and interrupt mailbox. PCI ordering rules
5479  * guarantee that MSI will arrive after the status block.
5480  */
5481 static irqreturn_t tg3_msi(int irq, void *dev_id)
5482 {
5483         struct tg3_napi *tnapi = dev_id;
5484         struct tg3 *tp = tnapi->tp;
5485
5486         prefetch(tnapi->hw_status);
5487         if (tnapi->rx_rcb)
5488                 prefetch(&tnapi->rx_rcb[tnapi->rx_rcb_ptr]);
5489         /*
5490          * Writing any value to intr-mbox-0 clears PCI INTA# and
5491          * chip-internal interrupt pending events.
5492          * Writing non-zero to intr-mbox-0 additional tells the
5493          * NIC to stop sending us irqs, engaging "in-intr-handler"
5494          * event coalescing.
5495          */
5496         tw32_mailbox(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW, 0x00000001);
5497         if (likely(!tg3_irq_sync(tp)))
5498                 napi_schedule(&tnapi->napi);
5499
5500         return IRQ_RETVAL(1);
5501 }
5502
5503 static irqreturn_t tg3_interrupt(int irq, void *dev_id)
5504 {
5505         struct tg3_napi *tnapi = dev_id;
5506         struct tg3 *tp = tnapi->tp;
5507         struct tg3_hw_status *sblk = tnapi->hw_status;
5508         unsigned int handled = 1;
5509
5510         /* In INTx mode, it is possible for the interrupt to arrive at
5511          * the CPU before the status block posted prior to the interrupt.
5512          * Reading the PCI State register will confirm whether the
5513          * interrupt is ours and will flush the status block.
5514          */
5515         if (unlikely(!(sblk->status & SD_STATUS_UPDATED))) {
5516                 if (tg3_flag(tp, CHIP_RESETTING) ||
5517                     (tr32(TG3PCI_PCISTATE) & PCISTATE_INT_NOT_ACTIVE)) {
5518                         handled = 0;
5519                         goto out;
5520                 }
5521         }
5522
5523         /*
5524          * Writing any value to intr-mbox-0 clears PCI INTA# and
5525          * chip-internal interrupt pending events.
5526          * Writing non-zero to intr-mbox-0 additional tells the
5527          * NIC to stop sending us irqs, engaging "in-intr-handler"
5528          * event coalescing.
5529          *
5530          * Flush the mailbox to de-assert the IRQ immediately to prevent
5531          * spurious interrupts.  The flush impacts performance but
5532          * excessive spurious interrupts can be worse in some cases.
5533          */
5534         tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW, 0x00000001);
5535         if (tg3_irq_sync(tp))
5536                 goto out;
5537         sblk->status &= ~SD_STATUS_UPDATED;
5538         if (likely(tg3_has_work(tnapi))) {
5539                 prefetch(&tnapi->rx_rcb[tnapi->rx_rcb_ptr]);
5540                 napi_schedule(&tnapi->napi);
5541         } else {
5542                 /* No work, shared interrupt perhaps?  re-enable
5543                  * interrupts, and flush that PCI write
5544                  */
5545                 tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW,
5546                                0x00000000);
5547         }
5548 out:
5549         return IRQ_RETVAL(handled);
5550 }
5551
5552 static irqreturn_t tg3_interrupt_tagged(int irq, void *dev_id)
5553 {
5554         struct tg3_napi *tnapi = dev_id;
5555         struct tg3 *tp = tnapi->tp;
5556         struct tg3_hw_status *sblk = tnapi->hw_status;
5557         unsigned int handled = 1;
5558
5559         /* In INTx mode, it is possible for the interrupt to arrive at
5560          * the CPU before the status block posted prior to the interrupt.
5561          * Reading the PCI State register will confirm whether the
5562          * interrupt is ours and will flush the status block.
5563          */
5564         if (unlikely(sblk->status_tag == tnapi->last_irq_tag)) {
5565                 if (tg3_flag(tp, CHIP_RESETTING) ||
5566                     (tr32(TG3PCI_PCISTATE) & PCISTATE_INT_NOT_ACTIVE)) {
5567                         handled = 0;
5568                         goto out;
5569                 }
5570         }
5571
5572         /*
5573          * writing any value to intr-mbox-0 clears PCI INTA# and
5574          * chip-internal interrupt pending events.
5575          * writing non-zero to intr-mbox-0 additional tells the
5576          * NIC to stop sending us irqs, engaging "in-intr-handler"
5577          * event coalescing.
5578          *
5579          * Flush the mailbox to de-assert the IRQ immediately to prevent
5580          * spurious interrupts.  The flush impacts performance but
5581          * excessive spurious interrupts can be worse in some cases.
5582          */
5583         tw32_mailbox_f(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW, 0x00000001);
5584
5585         /*
5586          * In a shared interrupt configuration, sometimes other devices'
5587          * interrupts will scream.  We record the current status tag here
5588          * so that the above check can report that the screaming interrupts
5589          * are unhandled.  Eventually they will be silenced.
5590          */
5591         tnapi->last_irq_tag = sblk->status_tag;
5592
5593         if (tg3_irq_sync(tp))
5594                 goto out;
5595
5596         prefetch(&tnapi->rx_rcb[tnapi->rx_rcb_ptr]);
5597
5598         napi_schedule(&tnapi->napi);
5599
5600 out:
5601         return IRQ_RETVAL(handled);
5602 }
5603
5604 /* ISR for interrupt test */
5605 static irqreturn_t tg3_test_isr(int irq, void *dev_id)
5606 {
5607         struct tg3_napi *tnapi = dev_id;
5608         struct tg3 *tp = tnapi->tp;
5609         struct tg3_hw_status *sblk = tnapi->hw_status;
5610
5611         if ((sblk->status & SD_STATUS_UPDATED) ||
5612             !(tr32(TG3PCI_PCISTATE) & PCISTATE_INT_NOT_ACTIVE)) {
5613                 tg3_disable_ints(tp);
5614                 return IRQ_RETVAL(1);
5615         }
5616         return IRQ_RETVAL(0);
5617 }
5618
5619 static int tg3_init_hw(struct tg3 *, int);
5620 static int tg3_halt(struct tg3 *, int, int);
5621
5622 /* Restart hardware after configuration changes, self-test, etc.
5623  * Invoked with tp->lock held.
5624  */
5625 static int tg3_restart_hw(struct tg3 *tp, int reset_phy)
5626         __releases(tp->lock)
5627         __acquires(tp->lock)
5628 {
5629         int err;
5630
5631         err = tg3_init_hw(tp, reset_phy);
5632         if (err) {
5633                 netdev_err(tp->dev,
5634                            "Failed to re-initialize device, aborting\n");
5635                 tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
5636                 tg3_full_unlock(tp);
5637                 del_timer_sync(&tp->timer);
5638                 tp->irq_sync = 0;
5639                 tg3_napi_enable(tp);
5640                 dev_close(tp->dev);
5641                 tg3_full_lock(tp, 0);
5642         }
5643         return err;
5644 }
5645
5646 #ifdef CONFIG_NET_POLL_CONTROLLER
5647 static void tg3_poll_controller(struct net_device *dev)
5648 {
5649         int i;
5650         struct tg3 *tp = netdev_priv(dev);
5651
5652         for (i = 0; i < tp->irq_cnt; i++)
5653                 tg3_interrupt(tp->napi[i].irq_vec, &tp->napi[i]);
5654 }
5655 #endif
5656
5657 static void tg3_reset_task(struct work_struct *work)
5658 {
5659         struct tg3 *tp = container_of(work, struct tg3, reset_task);
5660         int err;
5661         unsigned int restart_timer;
5662
5663         tg3_full_lock(tp, 0);
5664
5665         if (!netif_running(tp->dev)) {
5666                 tg3_full_unlock(tp);
5667                 return;
5668         }
5669
5670         tg3_full_unlock(tp);
5671
5672         tg3_phy_stop(tp);
5673
5674         tg3_netif_stop(tp);
5675
5676         tg3_full_lock(tp, 1);
5677
5678         restart_timer = tg3_flag(tp, RESTART_TIMER);
5679         tg3_flag_clear(tp, RESTART_TIMER);
5680
5681         if (tg3_flag(tp, TX_RECOVERY_PENDING)) {
5682                 tp->write32_tx_mbox = tg3_write32_tx_mbox;
5683                 tp->write32_rx_mbox = tg3_write_flush_reg32;
5684                 tg3_flag_set(tp, MBOX_WRITE_REORDER);
5685                 tg3_flag_clear(tp, TX_RECOVERY_PENDING);
5686         }
5687
5688         tg3_halt(tp, RESET_KIND_SHUTDOWN, 0);
5689         err = tg3_init_hw(tp, 1);
5690         if (err)
5691                 goto out;
5692
5693         tg3_netif_start(tp);
5694
5695         if (restart_timer)
5696                 mod_timer(&tp->timer, jiffies + 1);
5697
5698 out:
5699         tg3_full_unlock(tp);
5700
5701         if (!err)
5702                 tg3_phy_start(tp);
5703 }
5704
5705 static void tg3_tx_timeout(struct net_device *dev)
5706 {
5707         struct tg3 *tp = netdev_priv(dev);
5708
5709         if (netif_msg_tx_err(tp)) {
5710                 netdev_err(dev, "transmit timed out, resetting\n");
5711                 tg3_dump_state(tp);
5712         }
5713
5714         schedule_work(&tp->reset_task);
5715 }
5716
5717 /* Test for DMA buffers crossing any 4GB boundaries: 4G, 8G, etc */
5718 static inline int tg3_4g_overflow_test(dma_addr_t mapping, int len)
5719 {
5720         u32 base = (u32) mapping & 0xffffffff;
5721
5722         return (base > 0xffffdcc0) && (base + len + 8 < base);
5723 }
5724
5725 /* Test for DMA addresses > 40-bit */
5726 static inline int tg3_40bit_overflow_test(struct tg3 *tp, dma_addr_t mapping,
5727                                           int len)
5728 {
5729 #if defined(CONFIG_HIGHMEM) && (BITS_PER_LONG == 64)
5730         if (tg3_flag(tp, 40BIT_DMA_BUG))
5731                 return ((u64) mapping + len) > DMA_BIT_MASK(40);
5732         return 0;
5733 #else
5734         return 0;
5735 #endif
5736 }
5737
5738 static void tg3_set_txd(struct tg3_napi *tnapi, int entry,
5739                         dma_addr_t mapping, int len, u32 flags,
5740                         u32 mss_and_is_end)
5741 {
5742         struct tg3_tx_buffer_desc *txd = &tnapi->tx_ring[entry];
5743         int is_end = (mss_and_is_end & 0x1);
5744         u32 mss = (mss_and_is_end >> 1);
5745         u32 vlan_tag = 0;
5746
5747         if (is_end)
5748                 flags |= TXD_FLAG_END;
5749         if (flags & TXD_FLAG_VLAN) {
5750                 vlan_tag = flags >> 16;
5751                 flags &= 0xffff;
5752         }
5753         vlan_tag |= (mss << TXD_MSS_SHIFT);
5754
5755         txd->addr_hi = ((u64) mapping >> 32);
5756         txd->addr_lo = ((u64) mapping & 0xffffffff);
5757         txd->len_flags = (len << TXD_LEN_SHIFT) | flags;
5758         txd->vlan_tag = vlan_tag << TXD_VLAN_TAG_SHIFT;
5759 }
5760
5761 static void tg3_skb_error_unmap(struct tg3_napi *tnapi,
5762                                 struct sk_buff *skb, int last)
5763 {
5764         int i;
5765         u32 entry = tnapi->tx_prod;
5766         struct ring_info *txb = &tnapi->tx_buffers[entry];
5767
5768         pci_unmap_single(tnapi->tp->pdev,
5769                          dma_unmap_addr(txb, mapping),
5770                          skb_headlen(skb),
5771                          PCI_DMA_TODEVICE);
5772         for (i = 0; i <= last; i++) {
5773                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
5774
5775                 entry = NEXT_TX(entry);
5776                 txb = &tnapi->tx_buffers[entry];
5777
5778                 pci_unmap_page(tnapi->tp->pdev,
5779                                dma_unmap_addr(txb, mapping),
5780                                frag->size, PCI_DMA_TODEVICE);
5781         }
5782 }
5783
5784 /* Workaround 4GB and 40-bit hardware DMA bugs. */
5785 static int tigon3_dma_hwbug_workaround(struct tg3_napi *tnapi,
5786                                        struct sk_buff *skb,
5787                                        u32 base_flags, u32 mss)
5788 {
5789         struct tg3 *tp = tnapi->tp;
5790         struct sk_buff *new_skb;
5791         dma_addr_t new_addr = 0;
5792         u32 entry = tnapi->tx_prod;
5793         int ret = 0;
5794
5795         if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5701)
5796                 new_skb = skb_copy(skb, GFP_ATOMIC);
5797         else {
5798                 int more_headroom = 4 - ((unsigned long)skb->data & 3);
5799
5800                 new_skb = skb_copy_expand(skb,
5801                                           skb_headroom(skb) + more_headroom,
5802                                           skb_tailroom(skb), GFP_ATOMIC);
5803         }
5804
5805         if (!new_skb) {
5806                 ret = -1;
5807         } else {
5808                 /* New SKB is guaranteed to be linear. */
5809                 new_addr = pci_map_single(tp->pdev, new_skb->data, new_skb->len,
5810                                           PCI_DMA_TODEVICE);
5811                 /* Make sure the mapping succeeded */
5812                 if (pci_dma_mapping_error(tp->pdev, new_addr)) {
5813                         ret = -1;
5814                         dev_kfree_skb(new_skb);
5815
5816                 /* Make sure new skb does not cross any 4G boundaries.
5817                  * Drop the packet if it does.
5818                  */
5819                 } else if (tg3_flag(tp, 4G_DMA_BNDRY_BUG) &&
5820                            tg3_4g_overflow_test(new_addr, new_skb->len)) {
5821                         pci_unmap_single(tp->pdev, new_addr, new_skb->len,
5822                                          PCI_DMA_TODEVICE);
5823                         ret = -1;
5824                         dev_kfree_skb(new_skb);
5825                 } else {
5826                         tnapi->tx_buffers[entry].skb = new_skb;
5827                         dma_unmap_addr_set(&tnapi->tx_buffers[entry],
5828                                            mapping, new_addr);
5829
5830                         tg3_set_txd(tnapi, entry, new_addr, new_skb->len,
5831                                     base_flags, 1 | (mss << 1));
5832                 }
5833         }
5834
5835         dev_kfree_skb(skb);
5836
5837         return ret;
5838 }
5839
5840 static netdev_tx_t tg3_start_xmit(struct sk_buff *, struct net_device *);
5841
5842 /* Use GSO to workaround a rare TSO bug that may be triggered when the
5843  * TSO header is greater than 80 bytes.
5844  */
5845 static int tg3_tso_bug(struct tg3 *tp, struct sk_buff *skb)
5846 {
5847         struct sk_buff *segs, *nskb;
5848         u32 frag_cnt_est = skb_shinfo(skb)->gso_segs * 3;
5849
5850         /* Estimate the number of fragments in the worst case */
5851         if (unlikely(tg3_tx_avail(&tp->napi[0]) <= frag_cnt_est)) {
5852                 netif_stop_queue(tp->dev);
5853
5854                 /* netif_tx_stop_queue() must be done before checking
5855                  * checking tx index in tg3_tx_avail() below, because in
5856                  * tg3_tx(), we update tx index before checking for
5857                  * netif_tx_queue_stopped().
5858                  */
5859                 smp_mb();
5860                 if (tg3_tx_avail(&tp->napi[0]) <= frag_cnt_est)
5861                         return NETDEV_TX_BUSY;
5862
5863                 netif_wake_queue(tp->dev);
5864         }
5865
5866         segs = skb_gso_segment(skb, tp->dev->features & ~NETIF_F_TSO);
5867         if (IS_ERR(segs))
5868                 goto tg3_tso_bug_end;
5869
5870         do {
5871                 nskb = segs;
5872                 segs = segs->next;
5873                 nskb->next = NULL;
5874                 tg3_start_xmit(nskb, tp->dev);
5875         } while (segs);
5876
5877 tg3_tso_bug_end:
5878         dev_kfree_skb(skb);
5879
5880         return NETDEV_TX_OK;
5881 }
5882
5883 /* hard_start_xmit for devices that have the 4G bug and/or 40-bit bug and
5884  * support TG3_FLAG_HW_TSO_1 or firmware TSO only.
5885  */
5886 static netdev_tx_t tg3_start_xmit(struct sk_buff *skb, struct net_device *dev)
5887 {
5888         struct tg3 *tp = netdev_priv(dev);
5889         u32 len, entry, base_flags, mss;
5890         int i = -1, would_hit_hwbug;
5891         dma_addr_t mapping;
5892         struct tg3_napi *tnapi;
5893         struct netdev_queue *txq;
5894         unsigned int last;
5895
5896         txq = netdev_get_tx_queue(dev, skb_get_queue_mapping(skb));
5897         tnapi = &tp->napi[skb_get_queue_mapping(skb)];
5898         if (tg3_flag(tp, ENABLE_TSS))
5899                 tnapi++;
5900
5901         /* We are running in BH disabled context with netif_tx_lock
5902          * and TX reclaim runs via tp->napi.poll inside of a software
5903          * interrupt.  Furthermore, IRQ processing runs lockless so we have
5904          * no IRQ context deadlocks to worry about either.  Rejoice!
5905          */
5906         if (unlikely(tg3_tx_avail(tnapi) <= (skb_shinfo(skb)->nr_frags + 1))) {
5907                 if (!netif_tx_queue_stopped(txq)) {
5908                         netif_tx_stop_queue(txq);
5909
5910                         /* This is a hard error, log it. */
5911                         netdev_err(dev,
5912                                    "BUG! Tx Ring full when queue awake!\n");
5913                 }
5914                 return NETDEV_TX_BUSY;
5915         }
5916
5917         entry = tnapi->tx_prod;
5918         base_flags = 0;
5919         if (skb->ip_summed == CHECKSUM_PARTIAL)
5920                 base_flags |= TXD_FLAG_TCPUDP_CSUM;
5921
5922         mss = skb_shinfo(skb)->gso_size;
5923         if (mss) {
5924                 struct iphdr *iph;
5925                 u32 tcp_opt_len, hdr_len;
5926
5927                 if (skb_header_cloned(skb) &&
5928                     pskb_expand_head(skb, 0, 0, GFP_ATOMIC)) {
5929                         dev_kfree_skb(skb);
5930                         goto out_unlock;
5931                 }
5932
5933                 iph = ip_hdr(skb);
5934                 tcp_opt_len = tcp_optlen(skb);
5935
5936                 if (skb_is_gso_v6(skb)) {
5937                         hdr_len = skb_headlen(skb) - ETH_HLEN;
5938                 } else {
5939                         u32 ip_tcp_len;
5940
5941                         ip_tcp_len = ip_hdrlen(skb) + sizeof(struct tcphdr);
5942                         hdr_len = ip_tcp_len + tcp_opt_len;
5943
5944                         iph->check = 0;
5945                         iph->tot_len = htons(mss + hdr_len);
5946                 }
5947
5948                 if (unlikely((ETH_HLEN + hdr_len) > 80) &&
5949                     tg3_flag(tp, TSO_BUG))
5950                         return tg3_tso_bug(tp, skb);
5951
5952                 base_flags |= (TXD_FLAG_CPU_PRE_DMA |
5953                                TXD_FLAG_CPU_POST_DMA);
5954
5955                 if (tg3_flag(tp, HW_TSO_1) ||
5956                     tg3_flag(tp, HW_TSO_2) ||
5957                     tg3_flag(tp, HW_TSO_3)) {
5958                         tcp_hdr(skb)->check = 0;
5959                         base_flags &= ~TXD_FLAG_TCPUDP_CSUM;
5960                 } else
5961                         tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
5962                                                                  iph->daddr, 0,
5963                                                                  IPPROTO_TCP,
5964                                                                  0);
5965
5966                 if (tg3_flag(tp, HW_TSO_3)) {
5967                         mss |= (hdr_len & 0xc) << 12;
5968                         if (hdr_len & 0x10)
5969                                 base_flags |= 0x00000010;
5970                         base_flags |= (hdr_len & 0x3e0) << 5;
5971                 } else if (tg3_flag(tp, HW_TSO_2))
5972                         mss |= hdr_len << 9;
5973                 else if (tg3_flag(tp, HW_TSO_1) ||
5974                          GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) {
5975                         if (tcp_opt_len || iph->ihl > 5) {
5976                                 int tsflags;
5977
5978                                 tsflags = (iph->ihl - 5) + (tcp_opt_len >> 2);
5979                                 mss |= (tsflags << 11);
5980                         }
5981                 } else {
5982                         if (tcp_opt_len || iph->ihl > 5) {
5983                                 int tsflags;
5984
5985                                 tsflags = (iph->ihl - 5) + (tcp_opt_len >> 2);
5986                                 base_flags |= tsflags << 12;
5987                         }
5988                 }
5989         }
5990
5991         if (vlan_tx_tag_present(skb))
5992                 base_flags |= (TXD_FLAG_VLAN |
5993                                (vlan_tx_tag_get(skb) << 16));
5994
5995         if (tg3_flag(tp, USE_JUMBO_BDFLAG) &&
5996             !mss && skb->len > VLAN_ETH_FRAME_LEN)
5997                 base_flags |= TXD_FLAG_JMB_PKT;
5998
5999         len = skb_headlen(skb);
6000
6001         mapping = pci_map_single(tp->pdev, skb->data, len, PCI_DMA_TODEVICE);
6002         if (pci_dma_mapping_error(tp->pdev, mapping)) {
6003                 dev_kfree_skb(skb);
6004                 goto out_unlock;
6005         }
6006
6007         tnapi->tx_buffers[entry].skb = skb;
6008         dma_unmap_addr_set(&tnapi->tx_buffers[entry], mapping, mapping);
6009
6010         would_hit_hwbug = 0;
6011
6012         if (tg3_flag(tp, SHORT_DMA_BUG) && len <= 8)
6013                 would_hit_hwbug = 1;
6014
6015         if (tg3_flag(tp, 4G_DMA_BNDRY_BUG) &&
6016             tg3_4g_overflow_test(mapping, len))
6017                 would_hit_hwbug = 1;
6018
6019         if (tg3_flag(tp, 40BIT_DMA_LIMIT_BUG) &&
6020             tg3_40bit_overflow_test(tp, mapping, len))
6021                 would_hit_hwbug = 1;
6022
6023         if (tg3_flag(tp, 5701_DMA_BUG))
6024                 would_hit_hwbug = 1;
6025
6026         tg3_set_txd(tnapi, entry, mapping, len, base_flags,
6027                     (skb_shinfo(skb)->nr_frags == 0) | (mss << 1));
6028
6029         entry = NEXT_TX(entry);
6030
6031         /* Now loop through additional data fragments, and queue them. */
6032         if (skb_shinfo(skb)->nr_frags > 0) {
6033                 last = skb_shinfo(skb)->nr_frags - 1;
6034                 for (i = 0; i <= last; i++) {
6035                         skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
6036
6037                         len = frag->size;
6038                         mapping = pci_map_page(tp->pdev,
6039                                                frag->page,
6040                                                frag->page_offset,
6041                                                len, PCI_DMA_TODEVICE);
6042
6043                         tnapi->tx_buffers[entry].skb = NULL;
6044                         dma_unmap_addr_set(&tnapi->tx_buffers[entry], mapping,
6045                                            mapping);
6046                         if (pci_dma_mapping_error(tp->pdev, mapping))
6047                                 goto dma_error;
6048
6049                         if (tg3_flag(tp, SHORT_DMA_BUG) &&
6050                             len <= 8)
6051                                 would_hit_hwbug = 1;
6052
6053                         if (tg3_flag(tp, 4G_DMA_BNDRY_BUG) &&
6054                             tg3_4g_overflow_test(mapping, len))
6055                                 would_hit_hwbug = 1;
6056
6057                         if (tg3_flag(tp, 40BIT_DMA_LIMIT_BUG) &&
6058                             tg3_40bit_overflow_test(tp, mapping, len))
6059                                 would_hit_hwbug = 1;
6060
6061                         if (tg3_flag(tp, HW_TSO_1) ||
6062                             tg3_flag(tp, HW_TSO_2) ||
6063                             tg3_flag(tp, HW_TSO_3))
6064                                 tg3_set_txd(tnapi, entry, mapping, len,
6065                                             base_flags, (i == last)|(mss << 1));
6066                         else
6067                                 tg3_set_txd(tnapi, entry, mapping, len,
6068                                             base_flags, (i == last));
6069
6070                         entry = NEXT_TX(entry);
6071                 }
6072         }
6073
6074         if (would_hit_hwbug) {
6075                 tg3_skb_error_unmap(tnapi, skb, i);
6076
6077                 /* If the workaround fails due to memory/mapping
6078                  * failure, silently drop this packet.
6079                  */
6080                 if (tigon3_dma_hwbug_workaround(tnapi, skb, base_flags, mss))
6081                         goto out_unlock;
6082
6083                 entry = NEXT_TX(tnapi->tx_prod);
6084         }
6085
6086         /* Packets are ready, update Tx producer idx local and on card. */
6087         tw32_tx_mbox(tnapi->prodmbox, entry);
6088
6089         tnapi->tx_prod = entry;
6090         if (unlikely(tg3_tx_avail(tnapi) <= (MAX_SKB_FRAGS + 1))) {
6091                 netif_tx_stop_queue(txq);
6092
6093                 /* netif_tx_stop_queue() must be done before checking
6094                  * checking tx index in tg3_tx_avail() below, because in
6095                  * tg3_tx(), we update tx index before checking for
6096                  * netif_tx_queue_stopped().
6097                  */
6098                 smp_mb();
6099                 if (tg3_tx_avail(tnapi) > TG3_TX_WAKEUP_THRESH(tnapi))
6100                         netif_tx_wake_queue(txq);
6101         }
6102
6103 out_unlock:
6104         mmiowb();
6105
6106         return NETDEV_TX_OK;
6107
6108 dma_error:
6109         tg3_skb_error_unmap(tnapi, skb, i);
6110         dev_kfree_skb(skb);
6111         tnapi->tx_buffers[tnapi->tx_prod].skb = NULL;
6112         return NETDEV_TX_OK;
6113 }
6114
6115 static void tg3_set_loopback(struct net_device *dev, u32 features)
6116 {
6117         struct tg3 *tp = netdev_priv(dev);
6118
6119         if (features & NETIF_F_LOOPBACK) {
6120                 if (tp->mac_mode & MAC_MODE_PORT_INT_LPBACK)
6121                         return;
6122
6123                 /*
6124                  * Clear MAC_MODE_HALF_DUPLEX or you won't get packets back in
6125                  * loopback mode if Half-Duplex mode was negotiated earlier.
6126                  */
6127                 tp->mac_mode &= ~MAC_MODE_HALF_DUPLEX;
6128
6129                 /* Enable internal MAC loopback mode */
6130                 tp->mac_mode |= MAC_MODE_PORT_INT_LPBACK;
6131                 spin_lock_bh(&tp->lock);
6132                 tw32(MAC_MODE, tp->mac_mode);
6133                 netif_carrier_on(tp->dev);
6134                 spin_unlock_bh(&tp->lock);
6135                 netdev_info(dev, "Internal MAC loopback mode enabled.\n");
6136         } else {
6137                 if (!(tp->mac_mode & MAC_MODE_PORT_INT_LPBACK))
6138                         return;
6139
6140                 /* Disable internal MAC loopback mode */
6141                 tp->mac_mode &= ~MAC_MODE_PORT_INT_LPBACK;
6142                 spin_lock_bh(&tp->lock);
6143                 tw32(MAC_MODE, tp->mac_mode);
6144                 /* Force link status check */
6145                 tg3_setup_phy(tp, 1);
6146                 spin_unlock_bh(&tp->lock);
6147                 netdev_info(dev, "Internal MAC loopback mode disabled.\n");
6148         }
6149 }
6150
6151 static u32 tg3_fix_features(struct net_device *dev, u32 features)
6152 {
6153         struct tg3 *tp = netdev_priv(dev);
6154
6155         if (dev->mtu > ETH_DATA_LEN && tg3_flag(tp, 5780_CLASS))
6156                 features &= ~NETIF_F_ALL_TSO;
6157
6158         return features;
6159 }
6160
6161 static int tg3_set_features(struct net_device *dev, u32 features)
6162 {
6163         u32 changed = dev->features ^ features;
6164
6165         if ((changed & NETIF_F_LOOPBACK) && netif_running(dev))
6166                 tg3_set_loopback(dev, features);
6167
6168         return 0;
6169 }
6170
6171 static inline void tg3_set_mtu(struct net_device *dev, struct tg3 *tp,
6172                                int new_mtu)
6173 {
6174         dev->mtu = new_mtu;
6175
6176         if (new_mtu > ETH_DATA_LEN) {
6177                 if (tg3_flag(tp, 5780_CLASS)) {
6178                         netdev_update_features(dev);
6179                         tg3_flag_clear(tp, TSO_CAPABLE);
6180                 } else {
6181                         tg3_flag_set(tp, JUMBO_RING_ENABLE);
6182                 }
6183         } else {
6184                 if (tg3_flag(tp, 5780_CLASS)) {
6185                         tg3_flag_set(tp, TSO_CAPABLE);
6186                         netdev_update_features(dev);
6187                 }
6188                 tg3_flag_clear(tp, JUMBO_RING_ENABLE);
6189         }
6190 }
6191
6192 static int tg3_change_mtu(struct net_device *dev, int new_mtu)
6193 {
6194         struct tg3 *tp = netdev_priv(dev);
6195         int err;
6196
6197         if (new_mtu < TG3_MIN_MTU || new_mtu > TG3_MAX_MTU(tp))
6198                 return -EINVAL;
6199
6200         if (!netif_running(dev)) {
6201                 /* We'll just catch it later when the
6202                  * device is up'd.
6203                  */
6204                 tg3_set_mtu(dev, tp, new_mtu);
6205                 return 0;
6206         }
6207
6208         tg3_phy_stop(tp);
6209
6210         tg3_netif_stop(tp);
6211
6212         tg3_full_lock(tp, 1);
6213
6214         tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
6215
6216         tg3_set_mtu(dev, tp, new_mtu);
6217
6218         err = tg3_restart_hw(tp, 0);
6219
6220         if (!err)
6221                 tg3_netif_start(tp);
6222
6223         tg3_full_unlock(tp);
6224
6225         if (!err)
6226                 tg3_phy_start(tp);
6227
6228         return err;
6229 }
6230
6231 static void tg3_rx_prodring_free(struct tg3 *tp,
6232                                  struct tg3_rx_prodring_set *tpr)
6233 {
6234         int i;
6235
6236         if (tpr != &tp->napi[0].prodring) {
6237                 for (i = tpr->rx_std_cons_idx; i != tpr->rx_std_prod_idx;
6238                      i = (i + 1) & tp->rx_std_ring_mask)
6239                         tg3_rx_skb_free(tp, &tpr->rx_std_buffers[i],
6240                                         tp->rx_pkt_map_sz);
6241
6242                 if (tg3_flag(tp, JUMBO_CAPABLE)) {
6243                         for (i = tpr->rx_jmb_cons_idx;
6244                              i != tpr->rx_jmb_prod_idx;
6245                              i = (i + 1) & tp->rx_jmb_ring_mask) {
6246                                 tg3_rx_skb_free(tp, &tpr->rx_jmb_buffers[i],
6247                                                 TG3_RX_JMB_MAP_SZ);
6248                         }
6249                 }
6250
6251                 return;
6252         }
6253
6254         for (i = 0; i <= tp->rx_std_ring_mask; i++)
6255                 tg3_rx_skb_free(tp, &tpr->rx_std_buffers[i],
6256                                 tp->rx_pkt_map_sz);
6257
6258         if (tg3_flag(tp, JUMBO_CAPABLE) && !tg3_flag(tp, 5780_CLASS)) {
6259                 for (i = 0; i <= tp->rx_jmb_ring_mask; i++)
6260                         tg3_rx_skb_free(tp, &tpr->rx_jmb_buffers[i],
6261                                         TG3_RX_JMB_MAP_SZ);
6262         }
6263 }
6264
6265 /* Initialize rx rings for packet processing.
6266  *
6267  * The chip has been shut down and the driver detached from
6268  * the networking, so no interrupts or new tx packets will
6269  * end up in the driver.  tp->{tx,}lock are held and thus
6270  * we may not sleep.
6271  */
6272 static int tg3_rx_prodring_alloc(struct tg3 *tp,
6273                                  struct tg3_rx_prodring_set *tpr)
6274 {
6275         u32 i, rx_pkt_dma_sz;
6276
6277         tpr->rx_std_cons_idx = 0;
6278         tpr->rx_std_prod_idx = 0;
6279         tpr->rx_jmb_cons_idx = 0;
6280         tpr->rx_jmb_prod_idx = 0;
6281
6282         if (tpr != &tp->napi[0].prodring) {
6283                 memset(&tpr->rx_std_buffers[0], 0,
6284                        TG3_RX_STD_BUFF_RING_SIZE(tp));
6285                 if (tpr->rx_jmb_buffers)
6286                         memset(&tpr->rx_jmb_buffers[0], 0,
6287                                TG3_RX_JMB_BUFF_RING_SIZE(tp));
6288                 goto done;
6289         }
6290
6291         /* Zero out all descriptors. */
6292         memset(tpr->rx_std, 0, TG3_RX_STD_RING_BYTES(tp));
6293
6294         rx_pkt_dma_sz = TG3_RX_STD_DMA_SZ;
6295         if (tg3_flag(tp, 5780_CLASS) &&
6296             tp->dev->mtu > ETH_DATA_LEN)
6297                 rx_pkt_dma_sz = TG3_RX_JMB_DMA_SZ;
6298         tp->rx_pkt_map_sz = TG3_RX_DMA_TO_MAP_SZ(rx_pkt_dma_sz);
6299
6300         /* Initialize invariants of the rings, we only set this
6301          * stuff once.  This works because the card does not
6302          * write into the rx buffer posting rings.
6303          */
6304         for (i = 0; i <= tp->rx_std_ring_mask; i++) {
6305                 struct tg3_rx_buffer_desc *rxd;
6306
6307                 rxd = &tpr->rx_std[i];
6308                 rxd->idx_len = rx_pkt_dma_sz << RXD_LEN_SHIFT;
6309                 rxd->type_flags = (RXD_FLAG_END << RXD_FLAGS_SHIFT);
6310                 rxd->opaque = (RXD_OPAQUE_RING_STD |
6311                                (i << RXD_OPAQUE_INDEX_SHIFT));
6312         }
6313
6314         /* Now allocate fresh SKBs for each rx ring. */
6315         for (i = 0; i < tp->rx_pending; i++) {
6316                 if (tg3_alloc_rx_skb(tp, tpr, RXD_OPAQUE_RING_STD, i) < 0) {
6317                         netdev_warn(tp->dev,
6318                                     "Using a smaller RX standard ring. Only "
6319                                     "%d out of %d buffers were allocated "
6320                                     "successfully\n", i, tp->rx_pending);
6321                         if (i == 0)
6322                                 goto initfail;
6323                         tp->rx_pending = i;
6324                         break;
6325                 }
6326         }
6327
6328         if (!tg3_flag(tp, JUMBO_CAPABLE) || tg3_flag(tp, 5780_CLASS))
6329                 goto done;
6330
6331         memset(tpr->rx_jmb, 0, TG3_RX_JMB_RING_BYTES(tp));
6332
6333         if (!tg3_flag(tp, JUMBO_RING_ENABLE))
6334                 goto done;
6335
6336         for (i = 0; i <= tp->rx_jmb_ring_mask; i++) {
6337                 struct tg3_rx_buffer_desc *rxd;
6338
6339                 rxd = &tpr->rx_jmb[i].std;
6340                 rxd->idx_len = TG3_RX_JMB_DMA_SZ << RXD_LEN_SHIFT;
6341                 rxd->type_flags = (RXD_FLAG_END << RXD_FLAGS_SHIFT) |
6342                                   RXD_FLAG_JUMBO;
6343                 rxd->opaque = (RXD_OPAQUE_RING_JUMBO |
6344                        (i << RXD_OPAQUE_INDEX_SHIFT));
6345         }
6346
6347         for (i = 0; i < tp->rx_jumbo_pending; i++) {
6348                 if (tg3_alloc_rx_skb(tp, tpr, RXD_OPAQUE_RING_JUMBO, i) < 0) {
6349                         netdev_warn(tp->dev,
6350                                     "Using a smaller RX jumbo ring. Only %d "
6351                                     "out of %d buffers were allocated "
6352                                     "successfully\n", i, tp->rx_jumbo_pending);
6353                         if (i == 0)
6354                                 goto initfail;
6355                         tp->rx_jumbo_pending = i;
6356                         break;
6357                 }
6358         }
6359
6360 done:
6361         return 0;
6362
6363 initfail:
6364         tg3_rx_prodring_free(tp, tpr);
6365         return -ENOMEM;
6366 }
6367
6368 static void tg3_rx_prodring_fini(struct tg3 *tp,
6369                                  struct tg3_rx_prodring_set *tpr)
6370 {
6371         kfree(tpr->rx_std_buffers);
6372         tpr->rx_std_buffers = NULL;
6373         kfree(tpr->rx_jmb_buffers);
6374         tpr->rx_jmb_buffers = NULL;
6375         if (tpr->rx_std) {
6376                 dma_free_coherent(&tp->pdev->dev, TG3_RX_STD_RING_BYTES(tp),
6377                                   tpr->rx_std, tpr->rx_std_mapping);
6378                 tpr->rx_std = NULL;
6379         }
6380         if (tpr->rx_jmb) {
6381                 dma_free_coherent(&tp->pdev->dev, TG3_RX_JMB_RING_BYTES(tp),
6382                                   tpr->rx_jmb, tpr->rx_jmb_mapping);
6383                 tpr->rx_jmb = NULL;
6384         }
6385 }
6386
6387 static int tg3_rx_prodring_init(struct tg3 *tp,
6388                                 struct tg3_rx_prodring_set *tpr)
6389 {
6390         tpr->rx_std_buffers = kzalloc(TG3_RX_STD_BUFF_RING_SIZE(tp),
6391                                       GFP_KERNEL);
6392         if (!tpr->rx_std_buffers)
6393                 return -ENOMEM;
6394
6395         tpr->rx_std = dma_alloc_coherent(&tp->pdev->dev,
6396                                          TG3_RX_STD_RING_BYTES(tp),
6397                                          &tpr->rx_std_mapping,
6398                                          GFP_KERNEL);
6399         if (!tpr->rx_std)
6400                 goto err_out;
6401
6402         if (tg3_flag(tp, JUMBO_CAPABLE) && !tg3_flag(tp, 5780_CLASS)) {
6403                 tpr->rx_jmb_buffers = kzalloc(TG3_RX_JMB_BUFF_RING_SIZE(tp),
6404                                               GFP_KERNEL);
6405                 if (!tpr->rx_jmb_buffers)
6406                         goto err_out;
6407
6408                 tpr->rx_jmb = dma_alloc_coherent(&tp->pdev->dev,
6409                                                  TG3_RX_JMB_RING_BYTES(tp),
6410                                                  &tpr->rx_jmb_mapping,
6411                                                  GFP_KERNEL);
6412                 if (!tpr->rx_jmb)
6413                         goto err_out;
6414         }
6415
6416         return 0;
6417
6418 err_out:
6419         tg3_rx_prodring_fini(tp, tpr);
6420         return -ENOMEM;
6421 }
6422
6423 /* Free up pending packets in all rx/tx rings.
6424  *
6425  * The chip has been shut down and the driver detached from
6426  * the networking, so no interrupts or new tx packets will
6427  * end up in the driver.  tp->{tx,}lock is not held and we are not
6428  * in an interrupt context and thus may sleep.
6429  */
6430 static void tg3_free_rings(struct tg3 *tp)
6431 {
6432         int i, j;
6433
6434         for (j = 0; j < tp->irq_cnt; j++) {
6435                 struct tg3_napi *tnapi = &tp->napi[j];
6436
6437                 tg3_rx_prodring_free(tp, &tnapi->prodring);
6438
6439                 if (!tnapi->tx_buffers)
6440                         continue;
6441
6442                 for (i = 0; i < TG3_TX_RING_SIZE; ) {
6443                         struct ring_info *txp;
6444                         struct sk_buff *skb;
6445                         unsigned int k;
6446
6447                         txp = &tnapi->tx_buffers[i];
6448                         skb = txp->skb;
6449
6450                         if (skb == NULL) {
6451                                 i++;
6452                                 continue;
6453                         }
6454
6455                         pci_unmap_single(tp->pdev,
6456                                          dma_unmap_addr(txp, mapping),
6457                                          skb_headlen(skb),
6458                                          PCI_DMA_TODEVICE);
6459                         txp->skb = NULL;
6460
6461                         i++;
6462
6463                         for (k = 0; k < skb_shinfo(skb)->nr_frags; k++) {
6464                                 txp = &tnapi->tx_buffers[i & (TG3_TX_RING_SIZE - 1)];
6465                                 pci_unmap_page(tp->pdev,
6466                                                dma_unmap_addr(txp, mapping),
6467                                                skb_shinfo(skb)->frags[k].size,
6468                                                PCI_DMA_TODEVICE);
6469                                 i++;
6470                         }
6471
6472                         dev_kfree_skb_any(skb);
6473                 }
6474         }
6475 }
6476
6477 /* Initialize tx/rx rings for packet processing.
6478  *
6479  * The chip has been shut down and the driver detached from
6480  * the networking, so no interrupts or new tx packets will
6481  * end up in the driver.  tp->{tx,}lock are held and thus
6482  * we may not sleep.
6483  */
6484 static int tg3_init_rings(struct tg3 *tp)
6485 {
6486         int i;
6487
6488         /* Free up all the SKBs. */
6489         tg3_free_rings(tp);
6490
6491         for (i = 0; i < tp->irq_cnt; i++) {
6492                 struct tg3_napi *tnapi = &tp->napi[i];
6493
6494                 tnapi->last_tag = 0;
6495                 tnapi->last_irq_tag = 0;
6496                 tnapi->hw_status->status = 0;
6497                 tnapi->hw_status->status_tag = 0;
6498                 memset(tnapi->hw_status, 0, TG3_HW_STATUS_SIZE);
6499
6500                 tnapi->tx_prod = 0;
6501                 tnapi->tx_cons = 0;
6502                 if (tnapi->tx_ring)
6503                         memset(tnapi->tx_ring, 0, TG3_TX_RING_BYTES);
6504
6505                 tnapi->rx_rcb_ptr = 0;
6506                 if (tnapi->rx_rcb)
6507                         memset(tnapi->rx_rcb, 0, TG3_RX_RCB_RING_BYTES(tp));
6508
6509                 if (tg3_rx_prodring_alloc(tp, &tnapi->prodring)) {
6510                         tg3_free_rings(tp);
6511                         return -ENOMEM;
6512                 }
6513         }
6514
6515         return 0;
6516 }
6517
6518 /*
6519  * Must not be invoked with interrupt sources disabled and
6520  * the hardware shutdown down.
6521  */
6522 static void tg3_free_consistent(struct tg3 *tp)
6523 {
6524         int i;
6525
6526         for (i = 0; i < tp->irq_cnt; i++) {
6527                 struct tg3_napi *tnapi = &tp->napi[i];
6528
6529                 if (tnapi->tx_ring) {
6530                         dma_free_coherent(&tp->pdev->dev, TG3_TX_RING_BYTES,
6531                                 tnapi->tx_ring, tnapi->tx_desc_mapping);
6532                         tnapi->tx_ring = NULL;
6533                 }
6534
6535                 kfree(tnapi->tx_buffers);
6536                 tnapi->tx_buffers = NULL;
6537
6538                 if (tnapi->rx_rcb) {
6539                         dma_free_coherent(&tp->pdev->dev,
6540                                           TG3_RX_RCB_RING_BYTES(tp),
6541                                           tnapi->rx_rcb,
6542                                           tnapi->rx_rcb_mapping);
6543                         tnapi->rx_rcb = NULL;
6544                 }
6545
6546                 tg3_rx_prodring_fini(tp, &tnapi->prodring);
6547
6548                 if (tnapi->hw_status) {
6549                         dma_free_coherent(&tp->pdev->dev, TG3_HW_STATUS_SIZE,
6550                                           tnapi->hw_status,
6551                                           tnapi->status_mapping);
6552                         tnapi->hw_status = NULL;
6553                 }
6554         }
6555
6556         if (tp->hw_stats) {
6557                 dma_free_coherent(&tp->pdev->dev, sizeof(struct tg3_hw_stats),
6558                                   tp->hw_stats, tp->stats_mapping);
6559                 tp->hw_stats = NULL;
6560         }
6561 }
6562
6563 /*
6564  * Must not be invoked with interrupt sources disabled and
6565  * the hardware shutdown down.  Can sleep.
6566  */
6567 static int tg3_alloc_consistent(struct tg3 *tp)
6568 {
6569         int i;
6570
6571         tp->hw_stats = dma_alloc_coherent(&tp->pdev->dev,
6572                                           sizeof(struct tg3_hw_stats),
6573                                           &tp->stats_mapping,
6574                                           GFP_KERNEL);
6575         if (!tp->hw_stats)
6576                 goto err_out;
6577
6578         memset(tp->hw_stats, 0, sizeof(struct tg3_hw_stats));
6579
6580         for (i = 0; i < tp->irq_cnt; i++) {
6581                 struct tg3_napi *tnapi = &tp->napi[i];
6582                 struct tg3_hw_status *sblk;
6583
6584                 tnapi->hw_status = dma_alloc_coherent(&tp->pdev->dev,
6585                                                       TG3_HW_STATUS_SIZE,
6586                                                       &tnapi->status_mapping,
6587                                                       GFP_KERNEL);
6588                 if (!tnapi->hw_status)
6589                         goto err_out;
6590
6591                 memset(tnapi->hw_status, 0, TG3_HW_STATUS_SIZE);
6592                 sblk = tnapi->hw_status;
6593
6594                 if (tg3_rx_prodring_init(tp, &tnapi->prodring))
6595                         goto err_out;
6596
6597                 /* If multivector TSS is enabled, vector 0 does not handle
6598                  * tx interrupts.  Don't allocate any resources for it.
6599                  */
6600                 if ((!i && !tg3_flag(tp, ENABLE_TSS)) ||
6601                     (i && tg3_flag(tp, ENABLE_TSS))) {
6602                         tnapi->tx_buffers = kzalloc(sizeof(struct ring_info) *
6603                                                     TG3_TX_RING_SIZE,
6604                                                     GFP_KERNEL);
6605                         if (!tnapi->tx_buffers)
6606                                 goto err_out;
6607
6608                         tnapi->tx_ring = dma_alloc_coherent(&tp->pdev->dev,
6609                                                             TG3_TX_RING_BYTES,
6610                                                         &tnapi->tx_desc_mapping,
6611                                                             GFP_KERNEL);
6612                         if (!tnapi->tx_ring)
6613                                 goto err_out;
6614                 }
6615
6616                 /*
6617                  * When RSS is enabled, the status block format changes
6618                  * slightly.  The "rx_jumbo_consumer", "reserved",
6619                  * and "rx_mini_consumer" members get mapped to the
6620                  * other three rx return ring producer indexes.
6621                  */
6622                 switch (i) {
6623                 default:
6624                         tnapi->rx_rcb_prod_idx = &sblk->idx[0].rx_producer;
6625                         break;
6626                 case 2:
6627                         tnapi->rx_rcb_prod_idx = &sblk->rx_jumbo_consumer;
6628                         break;
6629                 case 3:
6630                         tnapi->rx_rcb_prod_idx = &sblk->reserved;
6631                         break;
6632                 case 4:
6633                         tnapi->rx_rcb_prod_idx = &sblk->rx_mini_consumer;
6634                         break;
6635                 }
6636
6637                 /*
6638                  * If multivector RSS is enabled, vector 0 does not handle
6639                  * rx or tx interrupts.  Don't allocate any resources for it.
6640                  */
6641                 if (!i && tg3_flag(tp, ENABLE_RSS))
6642                         continue;
6643
6644                 tnapi->rx_rcb = dma_alloc_coherent(&tp->pdev->dev,
6645                                                    TG3_RX_RCB_RING_BYTES(tp),
6646                                                    &tnapi->rx_rcb_mapping,
6647                                                    GFP_KERNEL);
6648                 if (!tnapi->rx_rcb)
6649                         goto err_out;
6650
6651                 memset(tnapi->rx_rcb, 0, TG3_RX_RCB_RING_BYTES(tp));
6652         }
6653
6654         return 0;
6655
6656 err_out:
6657         tg3_free_consistent(tp);
6658         return -ENOMEM;
6659 }
6660
6661 #define MAX_WAIT_CNT 1000
6662
6663 /* To stop a block, clear the enable bit and poll till it
6664  * clears.  tp->lock is held.
6665  */
6666 static int tg3_stop_block(struct tg3 *tp, unsigned long ofs, u32 enable_bit, int silent)
6667 {
6668         unsigned int i;
6669         u32 val;
6670
6671         if (tg3_flag(tp, 5705_PLUS)) {
6672                 switch (ofs) {
6673                 case RCVLSC_MODE:
6674                 case DMAC_MODE:
6675                 case MBFREE_MODE:
6676                 case BUFMGR_MODE:
6677                 case MEMARB_MODE:
6678                         /* We can't enable/disable these bits of the
6679                          * 5705/5750, just say success.
6680                          */
6681                         return 0;
6682
6683                 default:
6684                         break;
6685                 }
6686         }
6687
6688         val = tr32(ofs);
6689         val &= ~enable_bit;
6690         tw32_f(ofs, val);
6691
6692         for (i = 0; i < MAX_WAIT_CNT; i++) {
6693                 udelay(100);
6694                 val = tr32(ofs);
6695                 if ((val & enable_bit) == 0)
6696                         break;
6697         }
6698
6699         if (i == MAX_WAIT_CNT && !silent) {
6700                 dev_err(&tp->pdev->dev,
6701                         "tg3_stop_block timed out, ofs=%lx enable_bit=%x\n",
6702                         ofs, enable_bit);
6703                 return -ENODEV;
6704         }
6705
6706         return 0;
6707 }
6708
6709 /* tp->lock is held. */
6710 static int tg3_abort_hw(struct tg3 *tp, int silent)
6711 {
6712         int i, err;
6713
6714         tg3_disable_ints(tp);
6715
6716         tp->rx_mode &= ~RX_MODE_ENABLE;
6717         tw32_f(MAC_RX_MODE, tp->rx_mode);
6718         udelay(10);
6719
6720         err  = tg3_stop_block(tp, RCVBDI_MODE, RCVBDI_MODE_ENABLE, silent);
6721         err |= tg3_stop_block(tp, RCVLPC_MODE, RCVLPC_MODE_ENABLE, silent);
6722         err |= tg3_stop_block(tp, RCVLSC_MODE, RCVLSC_MODE_ENABLE, silent);
6723         err |= tg3_stop_block(tp, RCVDBDI_MODE, RCVDBDI_MODE_ENABLE, silent);
6724         err |= tg3_stop_block(tp, RCVDCC_MODE, RCVDCC_MODE_ENABLE, silent);
6725         err |= tg3_stop_block(tp, RCVCC_MODE, RCVCC_MODE_ENABLE, silent);
6726
6727         err |= tg3_stop_block(tp, SNDBDS_MODE, SNDBDS_MODE_ENABLE, silent);
6728         err |= tg3_stop_block(tp, SNDBDI_MODE, SNDBDI_MODE_ENABLE, silent);
6729         err |= tg3_stop_block(tp, SNDDATAI_MODE, SNDDATAI_MODE_ENABLE, silent);
6730         err |= tg3_stop_block(tp, RDMAC_MODE, RDMAC_MODE_ENABLE, silent);
6731         err |= tg3_stop_block(tp, SNDDATAC_MODE, SNDDATAC_MODE_ENABLE, silent);
6732         err |= tg3_stop_block(tp, DMAC_MODE, DMAC_MODE_ENABLE, silent);
6733         err |= tg3_stop_block(tp, SNDBDC_MODE, SNDBDC_MODE_ENABLE, silent);
6734
6735         tp->mac_mode &= ~MAC_MODE_TDE_ENABLE;
6736         tw32_f(MAC_MODE, tp->mac_mode);
6737         udelay(40);
6738
6739         tp->tx_mode &= ~TX_MODE_ENABLE;
6740         tw32_f(MAC_TX_MODE, tp->tx_mode);
6741
6742         for (i = 0; i < MAX_WAIT_CNT; i++) {
6743                 udelay(100);
6744                 if (!(tr32(MAC_TX_MODE) & TX_MODE_ENABLE))
6745                         break;
6746         }
6747         if (i >= MAX_WAIT_CNT) {
6748                 dev_err(&tp->pdev->dev,
6749                         "%s timed out, TX_MODE_ENABLE will not clear "
6750                         "MAC_TX_MODE=%08x\n", __func__, tr32(MAC_TX_MODE));
6751                 err |= -ENODEV;
6752         }
6753
6754         err |= tg3_stop_block(tp, HOSTCC_MODE, HOSTCC_MODE_ENABLE, silent);
6755         err |= tg3_stop_block(tp, WDMAC_MODE, WDMAC_MODE_ENABLE, silent);
6756         err |= tg3_stop_block(tp, MBFREE_MODE, MBFREE_MODE_ENABLE, silent);
6757
6758         tw32(FTQ_RESET, 0xffffffff);
6759         tw32(FTQ_RESET, 0x00000000);
6760
6761         err |= tg3_stop_block(tp, BUFMGR_MODE, BUFMGR_MODE_ENABLE, silent);
6762         err |= tg3_stop_block(tp, MEMARB_MODE, MEMARB_MODE_ENABLE, silent);
6763
6764         for (i = 0; i < tp->irq_cnt; i++) {
6765                 struct tg3_napi *tnapi = &tp->napi[i];
6766                 if (tnapi->hw_status)
6767                         memset(tnapi->hw_status, 0, TG3_HW_STATUS_SIZE);
6768         }
6769         if (tp->hw_stats)
6770                 memset(tp->hw_stats, 0, sizeof(struct tg3_hw_stats));
6771
6772         return err;
6773 }
6774
6775 static void tg3_ape_send_event(struct tg3 *tp, u32 event)
6776 {
6777         int i;
6778         u32 apedata;
6779
6780         /* NCSI does not support APE events */
6781         if (tg3_flag(tp, APE_HAS_NCSI))
6782                 return;
6783
6784         apedata = tg3_ape_read32(tp, TG3_APE_SEG_SIG);
6785         if (apedata != APE_SEG_SIG_MAGIC)
6786                 return;
6787
6788         apedata = tg3_ape_read32(tp, TG3_APE_FW_STATUS);
6789         if (!(apedata & APE_FW_STATUS_READY))
6790                 return;
6791
6792         /* Wait for up to 1 millisecond for APE to service previous event. */
6793         for (i = 0; i < 10; i++) {
6794                 if (tg3_ape_lock(tp, TG3_APE_LOCK_MEM))
6795                         return;
6796
6797                 apedata = tg3_ape_read32(tp, TG3_APE_EVENT_STATUS);
6798
6799                 if (!(apedata & APE_EVENT_STATUS_EVENT_PENDING))
6800                         tg3_ape_write32(tp, TG3_APE_EVENT_STATUS,
6801                                         event | APE_EVENT_STATUS_EVENT_PENDING);
6802
6803                 tg3_ape_unlock(tp, TG3_APE_LOCK_MEM);
6804
6805                 if (!(apedata & APE_EVENT_STATUS_EVENT_PENDING))
6806                         break;
6807
6808                 udelay(100);
6809         }
6810
6811         if (!(apedata & APE_EVENT_STATUS_EVENT_PENDING))
6812                 tg3_ape_write32(tp, TG3_APE_EVENT, APE_EVENT_1);
6813 }
6814
6815 static void tg3_ape_driver_state_change(struct tg3 *tp, int kind)
6816 {
6817         u32 event;
6818         u32 apedata;
6819
6820         if (!tg3_flag(tp, ENABLE_APE))
6821                 return;
6822
6823         switch (kind) {
6824         case RESET_KIND_INIT:
6825                 tg3_ape_write32(tp, TG3_APE_HOST_SEG_SIG,
6826                                 APE_HOST_SEG_SIG_MAGIC);
6827                 tg3_ape_write32(tp, TG3_APE_HOST_SEG_LEN,
6828                                 APE_HOST_SEG_LEN_MAGIC);
6829                 apedata = tg3_ape_read32(tp, TG3_APE_HOST_INIT_COUNT);
6830                 tg3_ape_write32(tp, TG3_APE_HOST_INIT_COUNT, ++apedata);
6831                 tg3_ape_write32(tp, TG3_APE_HOST_DRIVER_ID,
6832                         APE_HOST_DRIVER_ID_MAGIC(TG3_MAJ_NUM, TG3_MIN_NUM));
6833                 tg3_ape_write32(tp, TG3_APE_HOST_BEHAVIOR,
6834                                 APE_HOST_BEHAV_NO_PHYLOCK);
6835                 tg3_ape_write32(tp, TG3_APE_HOST_DRVR_STATE,
6836                                     TG3_APE_HOST_DRVR_STATE_START);
6837
6838                 event = APE_EVENT_STATUS_STATE_START;
6839                 break;
6840         case RESET_KIND_SHUTDOWN:
6841                 /* With the interface we are currently using,
6842                  * APE does not track driver state.  Wiping
6843                  * out the HOST SEGMENT SIGNATURE forces
6844                  * the APE to assume OS absent status.
6845                  */
6846                 tg3_ape_write32(tp, TG3_APE_HOST_SEG_SIG, 0x0);
6847
6848                 if (device_may_wakeup(&tp->pdev->dev) &&
6849                     tg3_flag(tp, WOL_ENABLE)) {
6850                         tg3_ape_write32(tp, TG3_APE_HOST_WOL_SPEED,
6851                                             TG3_APE_HOST_WOL_SPEED_AUTO);
6852                         apedata = TG3_APE_HOST_DRVR_STATE_WOL;
6853                 } else
6854                         apedata = TG3_APE_HOST_DRVR_STATE_UNLOAD;
6855
6856                 tg3_ape_write32(tp, TG3_APE_HOST_DRVR_STATE, apedata);
6857
6858                 event = APE_EVENT_STATUS_STATE_UNLOAD;
6859                 break;
6860         case RESET_KIND_SUSPEND:
6861                 event = APE_EVENT_STATUS_STATE_SUSPEND;
6862                 break;
6863         default:
6864                 return;
6865         }
6866
6867         event |= APE_EVENT_STATUS_DRIVER_EVNT | APE_EVENT_STATUS_STATE_CHNGE;
6868
6869         tg3_ape_send_event(tp, event);
6870 }
6871
6872 /* tp->lock is held. */
6873 static void tg3_write_sig_pre_reset(struct tg3 *tp, int kind)
6874 {
6875         tg3_write_mem(tp, NIC_SRAM_FIRMWARE_MBOX,
6876                       NIC_SRAM_FIRMWARE_MBOX_MAGIC1);
6877
6878         if (tg3_flag(tp, ASF_NEW_HANDSHAKE)) {
6879                 switch (kind) {
6880                 case RESET_KIND_INIT:
6881                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
6882                                       DRV_STATE_START);
6883                         break;
6884
6885                 case RESET_KIND_SHUTDOWN:
6886                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
6887                                       DRV_STATE_UNLOAD);
6888                         break;
6889
6890                 case RESET_KIND_SUSPEND:
6891                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
6892                                       DRV_STATE_SUSPEND);
6893                         break;
6894
6895                 default:
6896                         break;
6897                 }
6898         }
6899
6900         if (kind == RESET_KIND_INIT ||
6901             kind == RESET_KIND_SUSPEND)
6902                 tg3_ape_driver_state_change(tp, kind);
6903 }
6904
6905 /* tp->lock is held. */
6906 static void tg3_write_sig_post_reset(struct tg3 *tp, int kind)
6907 {
6908         if (tg3_flag(tp, ASF_NEW_HANDSHAKE)) {
6909                 switch (kind) {
6910                 case RESET_KIND_INIT:
6911                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
6912                                       DRV_STATE_START_DONE);
6913                         break;
6914
6915                 case RESET_KIND_SHUTDOWN:
6916                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
6917                                       DRV_STATE_UNLOAD_DONE);
6918                         break;
6919
6920                 default:
6921                         break;
6922                 }
6923         }
6924
6925         if (kind == RESET_KIND_SHUTDOWN)
6926                 tg3_ape_driver_state_change(tp, kind);
6927 }
6928
6929 /* tp->lock is held. */
6930 static void tg3_write_sig_legacy(struct tg3 *tp, int kind)
6931 {
6932         if (tg3_flag(tp, ENABLE_ASF)) {
6933                 switch (kind) {
6934                 case RESET_KIND_INIT:
6935                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
6936                                       DRV_STATE_START);
6937                         break;
6938
6939                 case RESET_KIND_SHUTDOWN:
6940                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
6941                                       DRV_STATE_UNLOAD);
6942                         break;
6943
6944                 case RESET_KIND_SUSPEND:
6945                         tg3_write_mem(tp, NIC_SRAM_FW_DRV_STATE_MBOX,
6946                                       DRV_STATE_SUSPEND);
6947                         break;
6948
6949                 default:
6950                         break;
6951                 }
6952         }
6953 }
6954
6955 static int tg3_poll_fw(struct tg3 *tp)
6956 {
6957         int i;
6958         u32 val;
6959
6960         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
6961                 /* Wait up to 20ms for init done. */
6962                 for (i = 0; i < 200; i++) {
6963                         if (tr32(VCPU_STATUS) & VCPU_STATUS_INIT_DONE)
6964                                 return 0;
6965                         udelay(100);
6966                 }
6967                 return -ENODEV;
6968         }
6969
6970         /* Wait for firmware initialization to complete. */
6971         for (i = 0; i < 100000; i++) {
6972                 tg3_read_mem(tp, NIC_SRAM_FIRMWARE_MBOX, &val);
6973                 if (val == ~NIC_SRAM_FIRMWARE_MBOX_MAGIC1)
6974                         break;
6975                 udelay(10);
6976         }
6977
6978         /* Chip might not be fitted with firmware.  Some Sun onboard
6979          * parts are configured like that.  So don't signal the timeout
6980          * of the above loop as an error, but do report the lack of
6981          * running firmware once.
6982          */
6983         if (i >= 100000 && !tg3_flag(tp, NO_FWARE_REPORTED)) {
6984                 tg3_flag_set(tp, NO_FWARE_REPORTED);
6985
6986                 netdev_info(tp->dev, "No firmware running\n");
6987         }
6988
6989         if (tp->pci_chip_rev_id == CHIPREV_ID_57765_A0) {
6990                 /* The 57765 A0 needs a little more
6991                  * time to do some important work.
6992                  */
6993                 mdelay(10);
6994         }
6995
6996         return 0;
6997 }
6998
6999 /* Save PCI command register before chip reset */
7000 static void tg3_save_pci_state(struct tg3 *tp)
7001 {
7002         pci_read_config_word(tp->pdev, PCI_COMMAND, &tp->pci_cmd);
7003 }
7004
7005 /* Restore PCI state after chip reset */
7006 static void tg3_restore_pci_state(struct tg3 *tp)
7007 {
7008         u32 val;
7009
7010         /* Re-enable indirect register accesses. */
7011         pci_write_config_dword(tp->pdev, TG3PCI_MISC_HOST_CTRL,
7012                                tp->misc_host_ctrl);
7013
7014         /* Set MAX PCI retry to zero. */
7015         val = (PCISTATE_ROM_ENABLE | PCISTATE_ROM_RETRY_ENABLE);
7016         if (tp->pci_chip_rev_id == CHIPREV_ID_5704_A0 &&
7017             tg3_flag(tp, PCIX_MODE))
7018                 val |= PCISTATE_RETRY_SAME_DMA;
7019         /* Allow reads and writes to the APE register and memory space. */
7020         if (tg3_flag(tp, ENABLE_APE))
7021                 val |= PCISTATE_ALLOW_APE_CTLSPC_WR |
7022                        PCISTATE_ALLOW_APE_SHMEM_WR |
7023                        PCISTATE_ALLOW_APE_PSPACE_WR;
7024         pci_write_config_dword(tp->pdev, TG3PCI_PCISTATE, val);
7025
7026         pci_write_config_word(tp->pdev, PCI_COMMAND, tp->pci_cmd);
7027
7028         if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5785) {
7029                 if (tg3_flag(tp, PCI_EXPRESS))
7030                         pcie_set_readrq(tp->pdev, tp->pcie_readrq);
7031                 else {
7032                         pci_write_config_byte(tp->pdev, PCI_CACHE_LINE_SIZE,
7033                                               tp->pci_cacheline_sz);
7034                         pci_write_config_byte(tp->pdev, PCI_LATENCY_TIMER,
7035                                               tp->pci_lat_timer);
7036                 }
7037         }
7038
7039         /* Make sure PCI-X relaxed ordering bit is clear. */
7040         if (tg3_flag(tp, PCIX_MODE)) {
7041                 u16 pcix_cmd;
7042
7043                 pci_read_config_word(tp->pdev, tp->pcix_cap + PCI_X_CMD,
7044                                      &pcix_cmd);
7045                 pcix_cmd &= ~PCI_X_CMD_ERO;
7046                 pci_write_config_word(tp->pdev, tp->pcix_cap + PCI_X_CMD,
7047                                       pcix_cmd);
7048         }
7049
7050         if (tg3_flag(tp, 5780_CLASS)) {
7051
7052                 /* Chip reset on 5780 will reset MSI enable bit,
7053                  * so need to restore it.
7054                  */
7055                 if (tg3_flag(tp, USING_MSI)) {
7056                         u16 ctrl;
7057
7058                         pci_read_config_word(tp->pdev,
7059                                              tp->msi_cap + PCI_MSI_FLAGS,
7060                                              &ctrl);
7061                         pci_write_config_word(tp->pdev,
7062                                               tp->msi_cap + PCI_MSI_FLAGS,
7063                                               ctrl | PCI_MSI_FLAGS_ENABLE);
7064                         val = tr32(MSGINT_MODE);
7065                         tw32(MSGINT_MODE, val | MSGINT_MODE_ENABLE);
7066                 }
7067         }
7068 }
7069
7070 static void tg3_stop_fw(struct tg3 *);
7071
7072 /* tp->lock is held. */
7073 static int tg3_chip_reset(struct tg3 *tp)
7074 {
7075         u32 val;
7076         void (*write_op)(struct tg3 *, u32, u32);
7077         int i, err;
7078
7079         tg3_nvram_lock(tp);
7080
7081         tg3_ape_lock(tp, TG3_APE_LOCK_GRC);
7082
7083         /* No matching tg3_nvram_unlock() after this because
7084          * chip reset below will undo the nvram lock.
7085          */
7086         tp->nvram_lock_cnt = 0;
7087
7088         /* GRC_MISC_CFG core clock reset will clear the memory
7089          * enable bit in PCI register 4 and the MSI enable bit
7090          * on some chips, so we save relevant registers here.
7091          */
7092         tg3_save_pci_state(tp);
7093
7094         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5752 ||
7095             tg3_flag(tp, 5755_PLUS))
7096                 tw32(GRC_FASTBOOT_PC, 0);
7097
7098         /*
7099          * We must avoid the readl() that normally takes place.
7100          * It locks machines, causes machine checks, and other
7101          * fun things.  So, temporarily disable the 5701
7102          * hardware workaround, while we do the reset.
7103          */
7104         write_op = tp->write32;
7105         if (write_op == tg3_write_flush_reg32)
7106                 tp->write32 = tg3_write32;
7107
7108         /* Prevent the irq handler from reading or writing PCI registers
7109          * during chip reset when the memory enable bit in the PCI command
7110          * register may be cleared.  The chip does not generate interrupt
7111          * at this time, but the irq handler may still be called due to irq
7112          * sharing or irqpoll.
7113          */
7114         tg3_flag_set(tp, CHIP_RESETTING);
7115         for (i = 0; i < tp->irq_cnt; i++) {
7116                 struct tg3_napi *tnapi = &tp->napi[i];
7117                 if (tnapi->hw_status) {
7118                         tnapi->hw_status->status = 0;
7119                         tnapi->hw_status->status_tag = 0;
7120                 }
7121                 tnapi->last_tag = 0;
7122                 tnapi->last_irq_tag = 0;
7123         }
7124         smp_mb();
7125
7126         for (i = 0; i < tp->irq_cnt; i++)
7127                 synchronize_irq(tp->napi[i].irq_vec);
7128
7129         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_57780) {
7130                 val = tr32(TG3_PCIE_LNKCTL) & ~TG3_PCIE_LNKCTL_L1_PLL_PD_EN;
7131                 tw32(TG3_PCIE_LNKCTL, val | TG3_PCIE_LNKCTL_L1_PLL_PD_DIS);
7132         }
7133
7134         /* do the reset */
7135         val = GRC_MISC_CFG_CORECLK_RESET;
7136
7137         if (tg3_flag(tp, PCI_EXPRESS)) {
7138                 /* Force PCIe 1.0a mode */
7139                 if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5785 &&
7140                     !tg3_flag(tp, 57765_PLUS) &&
7141                     tr32(TG3_PCIE_PHY_TSTCTL) ==
7142                     (TG3_PCIE_PHY_TSTCTL_PCIE10 | TG3_PCIE_PHY_TSTCTL_PSCRAM))
7143                         tw32(TG3_PCIE_PHY_TSTCTL, TG3_PCIE_PHY_TSTCTL_PSCRAM);
7144
7145                 if (tp->pci_chip_rev_id != CHIPREV_ID_5750_A0) {
7146                         tw32(GRC_MISC_CFG, (1 << 29));
7147                         val |= (1 << 29);
7148                 }
7149         }
7150
7151         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
7152                 tw32(VCPU_STATUS, tr32(VCPU_STATUS) | VCPU_STATUS_DRV_RESET);
7153                 tw32(GRC_VCPU_EXT_CTRL,
7154                      tr32(GRC_VCPU_EXT_CTRL) & ~GRC_VCPU_EXT_CTRL_HALT_CPU);
7155         }
7156
7157         /* Manage gphy power for all CPMU absent PCIe devices. */
7158         if (tg3_flag(tp, 5705_PLUS) && !tg3_flag(tp, CPMU_PRESENT))
7159                 val |= GRC_MISC_CFG_KEEP_GPHY_POWER;
7160
7161         tw32(GRC_MISC_CFG, val);
7162
7163         /* restore 5701 hardware bug workaround write method */
7164         tp->write32 = write_op;
7165
7166         /* Unfortunately, we have to delay before the PCI read back.
7167          * Some 575X chips even will not respond to a PCI cfg access
7168          * when the reset command is given to the chip.
7169          *
7170          * How do these hardware designers expect things to work
7171          * properly if the PCI write is posted for a long period
7172          * of time?  It is always necessary to have some method by
7173          * which a register read back can occur to push the write
7174          * out which does the reset.
7175          *
7176          * For most tg3 variants the trick below was working.
7177          * Ho hum...
7178          */
7179         udelay(120);
7180
7181         /* Flush PCI posted writes.  The normal MMIO registers
7182          * are inaccessible at this time so this is the only
7183          * way to make this reliably (actually, this is no longer
7184          * the case, see above).  I tried to use indirect
7185          * register read/write but this upset some 5701 variants.
7186          */
7187         pci_read_config_dword(tp->pdev, PCI_COMMAND, &val);
7188
7189         udelay(120);
7190
7191         if (tg3_flag(tp, PCI_EXPRESS) && tp->pcie_cap) {
7192                 u16 val16;
7193
7194                 if (tp->pci_chip_rev_id == CHIPREV_ID_5750_A0) {
7195                         int i;
7196                         u32 cfg_val;
7197
7198                         /* Wait for link training to complete.  */
7199                         for (i = 0; i < 5000; i++)
7200                                 udelay(100);
7201
7202                         pci_read_config_dword(tp->pdev, 0xc4, &cfg_val);
7203                         pci_write_config_dword(tp->pdev, 0xc4,
7204                                                cfg_val | (1 << 15));
7205                 }
7206
7207                 /* Clear the "no snoop" and "relaxed ordering" bits. */
7208                 pci_read_config_word(tp->pdev,
7209                                      tp->pcie_cap + PCI_EXP_DEVCTL,
7210                                      &val16);
7211                 val16 &= ~(PCI_EXP_DEVCTL_RELAX_EN |
7212                            PCI_EXP_DEVCTL_NOSNOOP_EN);
7213                 /*
7214                  * Older PCIe devices only support the 128 byte
7215                  * MPS setting.  Enforce the restriction.
7216                  */
7217                 if (!tg3_flag(tp, CPMU_PRESENT))
7218                         val16 &= ~PCI_EXP_DEVCTL_PAYLOAD;
7219                 pci_write_config_word(tp->pdev,
7220                                       tp->pcie_cap + PCI_EXP_DEVCTL,
7221                                       val16);
7222
7223                 pcie_set_readrq(tp->pdev, tp->pcie_readrq);
7224
7225                 /* Clear error status */
7226                 pci_write_config_word(tp->pdev,
7227                                       tp->pcie_cap + PCI_EXP_DEVSTA,
7228                                       PCI_EXP_DEVSTA_CED |
7229                                       PCI_EXP_DEVSTA_NFED |
7230                                       PCI_EXP_DEVSTA_FED |
7231                                       PCI_EXP_DEVSTA_URD);
7232         }
7233
7234         tg3_restore_pci_state(tp);
7235
7236         tg3_flag_clear(tp, CHIP_RESETTING);
7237         tg3_flag_clear(tp, ERROR_PROCESSED);
7238
7239         val = 0;
7240         if (tg3_flag(tp, 5780_CLASS))
7241                 val = tr32(MEMARB_MODE);
7242         tw32(MEMARB_MODE, val | MEMARB_MODE_ENABLE);
7243
7244         if (tp->pci_chip_rev_id == CHIPREV_ID_5750_A3) {
7245                 tg3_stop_fw(tp);
7246                 tw32(0x5000, 0x400);
7247         }
7248
7249         tw32(GRC_MODE, tp->grc_mode);
7250
7251         if (tp->pci_chip_rev_id == CHIPREV_ID_5705_A0) {
7252                 val = tr32(0xc4);
7253
7254                 tw32(0xc4, val | (1 << 15));
7255         }
7256
7257         if ((tp->nic_sram_data_cfg & NIC_SRAM_DATA_CFG_MINI_PCI) != 0 &&
7258             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) {
7259                 tp->pci_clock_ctrl |= CLOCK_CTRL_CLKRUN_OENABLE;
7260                 if (tp->pci_chip_rev_id == CHIPREV_ID_5705_A0)
7261                         tp->pci_clock_ctrl |= CLOCK_CTRL_FORCE_CLKRUN;
7262                 tw32(TG3PCI_CLOCK_CTRL, tp->pci_clock_ctrl);
7263         }
7264
7265         if (tg3_flag(tp, ENABLE_APE))
7266                 tp->mac_mode = MAC_MODE_APE_TX_EN |
7267                                MAC_MODE_APE_RX_EN |
7268                                MAC_MODE_TDE_ENABLE;
7269
7270         if (tp->phy_flags & TG3_PHYFLG_PHY_SERDES) {
7271                 tp->mac_mode |= MAC_MODE_PORT_MODE_TBI;
7272                 val = tp->mac_mode;
7273         } else if (tp->phy_flags & TG3_PHYFLG_MII_SERDES) {
7274                 tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;
7275                 val = tp->mac_mode;
7276         } else
7277                 val = 0;
7278
7279         tw32_f(MAC_MODE, val);
7280         udelay(40);
7281
7282         tg3_ape_unlock(tp, TG3_APE_LOCK_GRC);
7283
7284         err = tg3_poll_fw(tp);
7285         if (err)
7286                 return err;
7287
7288         tg3_mdio_start(tp);
7289
7290         if (tg3_flag(tp, PCI_EXPRESS) &&
7291             tp->pci_chip_rev_id != CHIPREV_ID_5750_A0 &&
7292             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5785 &&
7293             !tg3_flag(tp, 57765_PLUS)) {
7294                 val = tr32(0x7c00);
7295
7296                 tw32(0x7c00, val | (1 << 25));
7297         }
7298
7299         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5720) {
7300                 val = tr32(TG3_CPMU_CLCK_ORIDE);
7301                 tw32(TG3_CPMU_CLCK_ORIDE, val & ~CPMU_CLCK_ORIDE_MAC_ORIDE_EN);
7302         }
7303
7304         /* Reprobe ASF enable state.  */
7305         tg3_flag_clear(tp, ENABLE_ASF);
7306         tg3_flag_clear(tp, ASF_NEW_HANDSHAKE);
7307         tg3_read_mem(tp, NIC_SRAM_DATA_SIG, &val);
7308         if (val == NIC_SRAM_DATA_SIG_MAGIC) {
7309                 u32 nic_cfg;
7310
7311                 tg3_read_mem(tp, NIC_SRAM_DATA_CFG, &nic_cfg);
7312                 if (nic_cfg & NIC_SRAM_DATA_CFG_ASF_ENABLE) {
7313                         tg3_flag_set(tp, ENABLE_ASF);
7314                         tp->last_event_jiffies = jiffies;
7315                         if (tg3_flag(tp, 5750_PLUS))
7316                                 tg3_flag_set(tp, ASF_NEW_HANDSHAKE);
7317                 }
7318         }
7319
7320         return 0;
7321 }
7322
7323 /* tp->lock is held. */
7324 static void tg3_stop_fw(struct tg3 *tp)
7325 {
7326         if (tg3_flag(tp, ENABLE_ASF) && !tg3_flag(tp, ENABLE_APE)) {
7327                 /* Wait for RX cpu to ACK the previous event. */
7328                 tg3_wait_for_event_ack(tp);
7329
7330                 tg3_write_mem(tp, NIC_SRAM_FW_CMD_MBOX, FWCMD_NICDRV_PAUSE_FW);
7331
7332                 tg3_generate_fw_event(tp);
7333
7334                 /* Wait for RX cpu to ACK this event. */
7335                 tg3_wait_for_event_ack(tp);
7336         }
7337 }
7338
7339 /* tp->lock is held. */
7340 static int tg3_halt(struct tg3 *tp, int kind, int silent)
7341 {
7342         int err;
7343
7344         tg3_stop_fw(tp);
7345
7346         tg3_write_sig_pre_reset(tp, kind);
7347
7348         tg3_abort_hw(tp, silent);
7349         err = tg3_chip_reset(tp);
7350
7351         __tg3_set_mac_addr(tp, 0);
7352
7353         tg3_write_sig_legacy(tp, kind);
7354         tg3_write_sig_post_reset(tp, kind);
7355
7356         if (err)
7357                 return err;
7358
7359         return 0;
7360 }
7361
7362 #define RX_CPU_SCRATCH_BASE     0x30000
7363 #define RX_CPU_SCRATCH_SIZE     0x04000
7364 #define TX_CPU_SCRATCH_BASE     0x34000
7365 #define TX_CPU_SCRATCH_SIZE     0x04000
7366
7367 /* tp->lock is held. */
7368 static int tg3_halt_cpu(struct tg3 *tp, u32 offset)
7369 {
7370         int i;
7371
7372         BUG_ON(offset == TX_CPU_BASE && tg3_flag(tp, 5705_PLUS));
7373
7374         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
7375                 u32 val = tr32(GRC_VCPU_EXT_CTRL);
7376
7377                 tw32(GRC_VCPU_EXT_CTRL, val | GRC_VCPU_EXT_CTRL_HALT_CPU);
7378                 return 0;
7379         }
7380         if (offset == RX_CPU_BASE) {
7381                 for (i = 0; i < 10000; i++) {
7382                         tw32(offset + CPU_STATE, 0xffffffff);
7383                         tw32(offset + CPU_MODE,  CPU_MODE_HALT);
7384                         if (tr32(offset + CPU_MODE) & CPU_MODE_HALT)
7385                                 break;
7386                 }
7387
7388                 tw32(offset + CPU_STATE, 0xffffffff);
7389                 tw32_f(offset + CPU_MODE,  CPU_MODE_HALT);
7390                 udelay(10);
7391         } else {
7392                 for (i = 0; i < 10000; i++) {
7393                         tw32(offset + CPU_STATE, 0xffffffff);
7394                         tw32(offset + CPU_MODE,  CPU_MODE_HALT);
7395                         if (tr32(offset + CPU_MODE) & CPU_MODE_HALT)
7396                                 break;
7397                 }
7398         }
7399
7400         if (i >= 10000) {
7401                 netdev_err(tp->dev, "%s timed out, %s CPU\n",
7402                            __func__, offset == RX_CPU_BASE ? "RX" : "TX");
7403                 return -ENODEV;
7404         }
7405
7406         /* Clear firmware's nvram arbitration. */
7407         if (tg3_flag(tp, NVRAM))
7408                 tw32(NVRAM_SWARB, SWARB_REQ_CLR0);
7409         return 0;
7410 }
7411
7412 struct fw_info {
7413         unsigned int fw_base;
7414         unsigned int fw_len;
7415         const __be32 *fw_data;
7416 };
7417
7418 /* tp->lock is held. */
7419 static int tg3_load_firmware_cpu(struct tg3 *tp, u32 cpu_base, u32 cpu_scratch_base,
7420                                  int cpu_scratch_size, struct fw_info *info)
7421 {
7422         int err, lock_err, i;
7423         void (*write_op)(struct tg3 *, u32, u32);
7424
7425         if (cpu_base == TX_CPU_BASE && tg3_flag(tp, 5705_PLUS)) {
7426                 netdev_err(tp->dev,
7427                            "%s: Trying to load TX cpu firmware which is 5705\n",
7428                            __func__);
7429                 return -EINVAL;
7430         }
7431
7432         if (tg3_flag(tp, 5705_PLUS))
7433                 write_op = tg3_write_mem;
7434         else
7435                 write_op = tg3_write_indirect_reg32;
7436
7437         /* It is possible that bootcode is still loading at this point.
7438          * Get the nvram lock first before halting the cpu.
7439          */
7440         lock_err = tg3_nvram_lock(tp);
7441         err = tg3_halt_cpu(tp, cpu_base);
7442         if (!lock_err)
7443                 tg3_nvram_unlock(tp);
7444         if (err)
7445                 goto out;
7446
7447         for (i = 0; i < cpu_scratch_size; i += sizeof(u32))
7448                 write_op(tp, cpu_scratch_base + i, 0);
7449         tw32(cpu_base + CPU_STATE, 0xffffffff);
7450         tw32(cpu_base + CPU_MODE, tr32(cpu_base+CPU_MODE)|CPU_MODE_HALT);
7451         for (i = 0; i < (info->fw_len / sizeof(u32)); i++)
7452                 write_op(tp, (cpu_scratch_base +
7453                               (info->fw_base & 0xffff) +
7454                               (i * sizeof(u32))),
7455                               be32_to_cpu(info->fw_data[i]));
7456
7457         err = 0;
7458
7459 out:
7460         return err;
7461 }
7462
7463 /* tp->lock is held. */
7464 static int tg3_load_5701_a0_firmware_fix(struct tg3 *tp)
7465 {
7466         struct fw_info info;
7467         const __be32 *fw_data;
7468         int err, i;
7469
7470         fw_data = (void *)tp->fw->data;
7471
7472         /* Firmware blob starts with version numbers, followed by
7473            start address and length. We are setting complete length.
7474            length = end_address_of_bss - start_address_of_text.
7475            Remainder is the blob to be loaded contiguously
7476            from start address. */
7477
7478         info.fw_base = be32_to_cpu(fw_data[1]);
7479         info.fw_len = tp->fw->size - 12;
7480         info.fw_data = &fw_data[3];
7481
7482         err = tg3_load_firmware_cpu(tp, RX_CPU_BASE,
7483                                     RX_CPU_SCRATCH_BASE, RX_CPU_SCRATCH_SIZE,
7484                                     &info);
7485         if (err)
7486                 return err;
7487
7488         err = tg3_load_firmware_cpu(tp, TX_CPU_BASE,
7489                                     TX_CPU_SCRATCH_BASE, TX_CPU_SCRATCH_SIZE,
7490                                     &info);
7491         if (err)
7492                 return err;
7493
7494         /* Now startup only the RX cpu. */
7495         tw32(RX_CPU_BASE + CPU_STATE, 0xffffffff);
7496         tw32_f(RX_CPU_BASE + CPU_PC, info.fw_base);
7497
7498         for (i = 0; i < 5; i++) {
7499                 if (tr32(RX_CPU_BASE + CPU_PC) == info.fw_base)
7500                         break;
7501                 tw32(RX_CPU_BASE + CPU_STATE, 0xffffffff);
7502                 tw32(RX_CPU_BASE + CPU_MODE,  CPU_MODE_HALT);
7503                 tw32_f(RX_CPU_BASE + CPU_PC, info.fw_base);
7504                 udelay(1000);
7505         }
7506         if (i >= 5) {
7507                 netdev_err(tp->dev, "%s fails to set RX CPU PC, is %08x "
7508                            "should be %08x\n", __func__,
7509                            tr32(RX_CPU_BASE + CPU_PC), info.fw_base);
7510                 return -ENODEV;
7511         }
7512         tw32(RX_CPU_BASE + CPU_STATE, 0xffffffff);
7513         tw32_f(RX_CPU_BASE + CPU_MODE,  0x00000000);
7514
7515         return 0;
7516 }
7517
7518 /* tp->lock is held. */
7519 static int tg3_load_tso_firmware(struct tg3 *tp)
7520 {
7521         struct fw_info info;
7522         const __be32 *fw_data;
7523         unsigned long cpu_base, cpu_scratch_base, cpu_scratch_size;
7524         int err, i;
7525
7526         if (tg3_flag(tp, HW_TSO_1) ||
7527             tg3_flag(tp, HW_TSO_2) ||
7528             tg3_flag(tp, HW_TSO_3))
7529                 return 0;
7530
7531         fw_data = (void *)tp->fw->data;
7532
7533         /* Firmware blob starts with version numbers, followed by
7534            start address and length. We are setting complete length.
7535            length = end_address_of_bss - start_address_of_text.
7536            Remainder is the blob to be loaded contiguously
7537            from start address. */
7538
7539         info.fw_base = be32_to_cpu(fw_data[1]);
7540         cpu_scratch_size = tp->fw_len;
7541         info.fw_len = tp->fw->size - 12;
7542         info.fw_data = &fw_data[3];
7543
7544         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) {
7545                 cpu_base = RX_CPU_BASE;
7546                 cpu_scratch_base = NIC_SRAM_MBUF_POOL_BASE5705;
7547         } else {
7548                 cpu_base = TX_CPU_BASE;
7549                 cpu_scratch_base = TX_CPU_SCRATCH_BASE;
7550                 cpu_scratch_size = TX_CPU_SCRATCH_SIZE;
7551         }
7552
7553         err = tg3_load_firmware_cpu(tp, cpu_base,
7554                                     cpu_scratch_base, cpu_scratch_size,
7555                                     &info);
7556         if (err)
7557                 return err;
7558
7559         /* Now startup the cpu. */
7560         tw32(cpu_base + CPU_STATE, 0xffffffff);
7561         tw32_f(cpu_base + CPU_PC, info.fw_base);
7562
7563         for (i = 0; i < 5; i++) {
7564                 if (tr32(cpu_base + CPU_PC) == info.fw_base)
7565                         break;
7566                 tw32(cpu_base + CPU_STATE, 0xffffffff);
7567                 tw32(cpu_base + CPU_MODE,  CPU_MODE_HALT);
7568                 tw32_f(cpu_base + CPU_PC, info.fw_base);
7569                 udelay(1000);
7570         }
7571         if (i >= 5) {
7572                 netdev_err(tp->dev,
7573                            "%s fails to set CPU PC, is %08x should be %08x\n",
7574                            __func__, tr32(cpu_base + CPU_PC), info.fw_base);
7575                 return -ENODEV;
7576         }
7577         tw32(cpu_base + CPU_STATE, 0xffffffff);
7578         tw32_f(cpu_base + CPU_MODE,  0x00000000);
7579         return 0;
7580 }
7581
7582
7583 static int tg3_set_mac_addr(struct net_device *dev, void *p)
7584 {
7585         struct tg3 *tp = netdev_priv(dev);
7586         struct sockaddr *addr = p;
7587         int err = 0, skip_mac_1 = 0;
7588
7589         if (!is_valid_ether_addr(addr->sa_data))
7590                 return -EINVAL;
7591
7592         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
7593
7594         if (!netif_running(dev))
7595                 return 0;
7596
7597         if (tg3_flag(tp, ENABLE_ASF)) {
7598                 u32 addr0_high, addr0_low, addr1_high, addr1_low;
7599
7600                 addr0_high = tr32(MAC_ADDR_0_HIGH);
7601                 addr0_low = tr32(MAC_ADDR_0_LOW);
7602                 addr1_high = tr32(MAC_ADDR_1_HIGH);
7603                 addr1_low = tr32(MAC_ADDR_1_LOW);
7604
7605                 /* Skip MAC addr 1 if ASF is using it. */
7606                 if ((addr0_high != addr1_high || addr0_low != addr1_low) &&
7607                     !(addr1_high == 0 && addr1_low == 0))
7608                         skip_mac_1 = 1;
7609         }
7610         spin_lock_bh(&tp->lock);
7611         __tg3_set_mac_addr(tp, skip_mac_1);
7612         spin_unlock_bh(&tp->lock);
7613
7614         return err;
7615 }
7616
7617 /* tp->lock is held. */
7618 static void tg3_set_bdinfo(struct tg3 *tp, u32 bdinfo_addr,
7619                            dma_addr_t mapping, u32 maxlen_flags,
7620                            u32 nic_addr)
7621 {
7622         tg3_write_mem(tp,
7623                       (bdinfo_addr + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_HIGH),
7624                       ((u64) mapping >> 32));
7625         tg3_write_mem(tp,
7626                       (bdinfo_addr + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_LOW),
7627                       ((u64) mapping & 0xffffffff));
7628         tg3_write_mem(tp,
7629                       (bdinfo_addr + TG3_BDINFO_MAXLEN_FLAGS),
7630                        maxlen_flags);
7631
7632         if (!tg3_flag(tp, 5705_PLUS))
7633                 tg3_write_mem(tp,
7634                               (bdinfo_addr + TG3_BDINFO_NIC_ADDR),
7635                               nic_addr);
7636 }
7637
7638 static void __tg3_set_rx_mode(struct net_device *);
7639 static void __tg3_set_coalesce(struct tg3 *tp, struct ethtool_coalesce *ec)
7640 {
7641         int i;
7642
7643         if (!tg3_flag(tp, ENABLE_TSS)) {
7644                 tw32(HOSTCC_TXCOL_TICKS, ec->tx_coalesce_usecs);
7645                 tw32(HOSTCC_TXMAX_FRAMES, ec->tx_max_coalesced_frames);
7646                 tw32(HOSTCC_TXCOAL_MAXF_INT, ec->tx_max_coalesced_frames_irq);
7647         } else {
7648                 tw32(HOSTCC_TXCOL_TICKS, 0);
7649                 tw32(HOSTCC_TXMAX_FRAMES, 0);
7650                 tw32(HOSTCC_TXCOAL_MAXF_INT, 0);
7651         }
7652
7653         if (!tg3_flag(tp, ENABLE_RSS)) {
7654                 tw32(HOSTCC_RXCOL_TICKS, ec->rx_coalesce_usecs);
7655                 tw32(HOSTCC_RXMAX_FRAMES, ec->rx_max_coalesced_frames);
7656                 tw32(HOSTCC_RXCOAL_MAXF_INT, ec->rx_max_coalesced_frames_irq);
7657         } else {
7658                 tw32(HOSTCC_RXCOL_TICKS, 0);
7659                 tw32(HOSTCC_RXMAX_FRAMES, 0);
7660                 tw32(HOSTCC_RXCOAL_MAXF_INT, 0);
7661         }
7662
7663         if (!tg3_flag(tp, 5705_PLUS)) {
7664                 u32 val = ec->stats_block_coalesce_usecs;
7665
7666                 tw32(HOSTCC_RXCOAL_TICK_INT, ec->rx_coalesce_usecs_irq);
7667                 tw32(HOSTCC_TXCOAL_TICK_INT, ec->tx_coalesce_usecs_irq);
7668
7669                 if (!netif_carrier_ok(tp->dev))
7670                         val = 0;
7671
7672                 tw32(HOSTCC_STAT_COAL_TICKS, val);
7673         }
7674
7675         for (i = 0; i < tp->irq_cnt - 1; i++) {
7676                 u32 reg;
7677
7678                 reg = HOSTCC_RXCOL_TICKS_VEC1 + i * 0x18;
7679                 tw32(reg, ec->rx_coalesce_usecs);
7680                 reg = HOSTCC_RXMAX_FRAMES_VEC1 + i * 0x18;
7681                 tw32(reg, ec->rx_max_coalesced_frames);
7682                 reg = HOSTCC_RXCOAL_MAXF_INT_VEC1 + i * 0x18;
7683                 tw32(reg, ec->rx_max_coalesced_frames_irq);
7684
7685                 if (tg3_flag(tp, ENABLE_TSS)) {
7686                         reg = HOSTCC_TXCOL_TICKS_VEC1 + i * 0x18;
7687                         tw32(reg, ec->tx_coalesce_usecs);
7688                         reg = HOSTCC_TXMAX_FRAMES_VEC1 + i * 0x18;
7689                         tw32(reg, ec->tx_max_coalesced_frames);
7690                         reg = HOSTCC_TXCOAL_MAXF_INT_VEC1 + i * 0x18;
7691                         tw32(reg, ec->tx_max_coalesced_frames_irq);
7692                 }
7693         }
7694
7695         for (; i < tp->irq_max - 1; i++) {
7696                 tw32(HOSTCC_RXCOL_TICKS_VEC1 + i * 0x18, 0);
7697                 tw32(HOSTCC_RXMAX_FRAMES_VEC1 + i * 0x18, 0);
7698                 tw32(HOSTCC_RXCOAL_MAXF_INT_VEC1 + i * 0x18, 0);
7699
7700                 if (tg3_flag(tp, ENABLE_TSS)) {
7701                         tw32(HOSTCC_TXCOL_TICKS_VEC1 + i * 0x18, 0);
7702                         tw32(HOSTCC_TXMAX_FRAMES_VEC1 + i * 0x18, 0);
7703                         tw32(HOSTCC_TXCOAL_MAXF_INT_VEC1 + i * 0x18, 0);
7704                 }
7705         }
7706 }
7707
7708 /* tp->lock is held. */
7709 static void tg3_rings_reset(struct tg3 *tp)
7710 {
7711         int i;
7712         u32 stblk, txrcb, rxrcb, limit;
7713         struct tg3_napi *tnapi = &tp->napi[0];
7714
7715         /* Disable all transmit rings but the first. */
7716         if (!tg3_flag(tp, 5705_PLUS))
7717                 limit = NIC_SRAM_SEND_RCB + TG3_BDINFO_SIZE * 16;
7718         else if (tg3_flag(tp, 5717_PLUS))
7719                 limit = NIC_SRAM_SEND_RCB + TG3_BDINFO_SIZE * 4;
7720         else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_57765)
7721                 limit = NIC_SRAM_SEND_RCB + TG3_BDINFO_SIZE * 2;
7722         else
7723                 limit = NIC_SRAM_SEND_RCB + TG3_BDINFO_SIZE;
7724
7725         for (txrcb = NIC_SRAM_SEND_RCB + TG3_BDINFO_SIZE;
7726              txrcb < limit; txrcb += TG3_BDINFO_SIZE)
7727                 tg3_write_mem(tp, txrcb + TG3_BDINFO_MAXLEN_FLAGS,
7728                               BDINFO_FLAGS_DISABLED);
7729
7730
7731         /* Disable all receive return rings but the first. */
7732         if (tg3_flag(tp, 5717_PLUS))
7733                 limit = NIC_SRAM_RCV_RET_RCB + TG3_BDINFO_SIZE * 17;
7734         else if (!tg3_flag(tp, 5705_PLUS))
7735                 limit = NIC_SRAM_RCV_RET_RCB + TG3_BDINFO_SIZE * 16;
7736         else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755 ||
7737                  GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_57765)
7738                 limit = NIC_SRAM_RCV_RET_RCB + TG3_BDINFO_SIZE * 4;
7739         else
7740                 limit = NIC_SRAM_RCV_RET_RCB + TG3_BDINFO_SIZE;
7741
7742         for (rxrcb = NIC_SRAM_RCV_RET_RCB + TG3_BDINFO_SIZE;
7743              rxrcb < limit; rxrcb += TG3_BDINFO_SIZE)
7744                 tg3_write_mem(tp, rxrcb + TG3_BDINFO_MAXLEN_FLAGS,
7745                               BDINFO_FLAGS_DISABLED);
7746
7747         /* Disable interrupts */
7748         tw32_mailbox_f(tp->napi[0].int_mbox, 1);
7749
7750         /* Zero mailbox registers. */
7751         if (tg3_flag(tp, SUPPORT_MSIX)) {
7752                 for (i = 1; i < tp->irq_max; i++) {
7753                         tp->napi[i].tx_prod = 0;
7754                         tp->napi[i].tx_cons = 0;
7755                         if (tg3_flag(tp, ENABLE_TSS))
7756                                 tw32_mailbox(tp->napi[i].prodmbox, 0);
7757                         tw32_rx_mbox(tp->napi[i].consmbox, 0);
7758                         tw32_mailbox_f(tp->napi[i].int_mbox, 1);
7759                 }
7760                 if (!tg3_flag(tp, ENABLE_TSS))
7761                         tw32_mailbox(tp->napi[0].prodmbox, 0);
7762         } else {
7763                 tp->napi[0].tx_prod = 0;
7764                 tp->napi[0].tx_cons = 0;
7765                 tw32_mailbox(tp->napi[0].prodmbox, 0);
7766                 tw32_rx_mbox(tp->napi[0].consmbox, 0);
7767         }
7768
7769         /* Make sure the NIC-based send BD rings are disabled. */
7770         if (!tg3_flag(tp, 5705_PLUS)) {
7771                 u32 mbox = MAILBOX_SNDNIC_PROD_IDX_0 + TG3_64BIT_REG_LOW;
7772                 for (i = 0; i < 16; i++)
7773                         tw32_tx_mbox(mbox + i * 8, 0);
7774         }
7775
7776         txrcb = NIC_SRAM_SEND_RCB;
7777         rxrcb = NIC_SRAM_RCV_RET_RCB;
7778
7779         /* Clear status block in ram. */
7780         memset(tnapi->hw_status, 0, TG3_HW_STATUS_SIZE);
7781
7782         /* Set status block DMA address */
7783         tw32(HOSTCC_STATUS_BLK_HOST_ADDR + TG3_64BIT_REG_HIGH,
7784              ((u64) tnapi->status_mapping >> 32));
7785         tw32(HOSTCC_STATUS_BLK_HOST_ADDR + TG3_64BIT_REG_LOW,
7786              ((u64) tnapi->status_mapping & 0xffffffff));
7787
7788         if (tnapi->tx_ring) {
7789                 tg3_set_bdinfo(tp, txrcb, tnapi->tx_desc_mapping,
7790                                (TG3_TX_RING_SIZE <<
7791                                 BDINFO_FLAGS_MAXLEN_SHIFT),
7792                                NIC_SRAM_TX_BUFFER_DESC);
7793                 txrcb += TG3_BDINFO_SIZE;
7794         }
7795
7796         if (tnapi->rx_rcb) {
7797                 tg3_set_bdinfo(tp, rxrcb, tnapi->rx_rcb_mapping,
7798                                (tp->rx_ret_ring_mask + 1) <<
7799                                 BDINFO_FLAGS_MAXLEN_SHIFT, 0);
7800                 rxrcb += TG3_BDINFO_SIZE;
7801         }
7802
7803         stblk = HOSTCC_STATBLCK_RING1;
7804
7805         for (i = 1, tnapi++; i < tp->irq_cnt; i++, tnapi++) {
7806                 u64 mapping = (u64)tnapi->status_mapping;
7807                 tw32(stblk + TG3_64BIT_REG_HIGH, mapping >> 32);
7808                 tw32(stblk + TG3_64BIT_REG_LOW, mapping & 0xffffffff);
7809
7810                 /* Clear status block in ram. */
7811                 memset(tnapi->hw_status, 0, TG3_HW_STATUS_SIZE);
7812
7813                 if (tnapi->tx_ring) {
7814                         tg3_set_bdinfo(tp, txrcb, tnapi->tx_desc_mapping,
7815                                        (TG3_TX_RING_SIZE <<
7816                                         BDINFO_FLAGS_MAXLEN_SHIFT),
7817                                        NIC_SRAM_TX_BUFFER_DESC);
7818                         txrcb += TG3_BDINFO_SIZE;
7819                 }
7820
7821                 tg3_set_bdinfo(tp, rxrcb, tnapi->rx_rcb_mapping,
7822                                ((tp->rx_ret_ring_mask + 1) <<
7823                                 BDINFO_FLAGS_MAXLEN_SHIFT), 0);
7824
7825                 stblk += 8;
7826                 rxrcb += TG3_BDINFO_SIZE;
7827         }
7828 }
7829
7830 static void tg3_setup_rxbd_thresholds(struct tg3 *tp)
7831 {
7832         u32 val, bdcache_maxcnt, host_rep_thresh, nic_rep_thresh;
7833
7834         if (!tg3_flag(tp, 5750_PLUS) ||
7835             tg3_flag(tp, 5780_CLASS) ||
7836             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5750 ||
7837             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5752)
7838                 bdcache_maxcnt = TG3_SRAM_RX_STD_BDCACHE_SIZE_5700;
7839         else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755 ||
7840                  GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787)
7841                 bdcache_maxcnt = TG3_SRAM_RX_STD_BDCACHE_SIZE_5755;
7842         else
7843                 bdcache_maxcnt = TG3_SRAM_RX_STD_BDCACHE_SIZE_5906;
7844
7845         nic_rep_thresh = min(bdcache_maxcnt / 2, tp->rx_std_max_post);
7846         host_rep_thresh = max_t(u32, tp->rx_pending / 8, 1);
7847
7848         val = min(nic_rep_thresh, host_rep_thresh);
7849         tw32(RCVBDI_STD_THRESH, val);
7850
7851         if (tg3_flag(tp, 57765_PLUS))
7852                 tw32(STD_REPLENISH_LWM, bdcache_maxcnt);
7853
7854         if (!tg3_flag(tp, JUMBO_CAPABLE) || tg3_flag(tp, 5780_CLASS))
7855                 return;
7856
7857         if (!tg3_flag(tp, 5705_PLUS))
7858                 bdcache_maxcnt = TG3_SRAM_RX_JMB_BDCACHE_SIZE_5700;
7859         else
7860                 bdcache_maxcnt = TG3_SRAM_RX_JMB_BDCACHE_SIZE_5717;
7861
7862         host_rep_thresh = max_t(u32, tp->rx_jumbo_pending / 8, 1);
7863
7864         val = min(bdcache_maxcnt / 2, host_rep_thresh);
7865         tw32(RCVBDI_JUMBO_THRESH, val);
7866
7867         if (tg3_flag(tp, 57765_PLUS))
7868                 tw32(JMB_REPLENISH_LWM, bdcache_maxcnt);
7869 }
7870
7871 /* tp->lock is held. */
7872 static int tg3_reset_hw(struct tg3 *tp, int reset_phy)
7873 {
7874         u32 val, rdmac_mode;
7875         int i, err, limit;
7876         struct tg3_rx_prodring_set *tpr = &tp->napi[0].prodring;
7877
7878         tg3_disable_ints(tp);
7879
7880         tg3_stop_fw(tp);
7881
7882         tg3_write_sig_pre_reset(tp, RESET_KIND_INIT);
7883
7884         if (tg3_flag(tp, INIT_COMPLETE))
7885                 tg3_abort_hw(tp, 1);
7886
7887         /* Enable MAC control of LPI */
7888         if (tp->phy_flags & TG3_PHYFLG_EEE_CAP) {
7889                 tw32_f(TG3_CPMU_EEE_LNKIDL_CTRL,
7890                        TG3_CPMU_EEE_LNKIDL_PCIE_NL0 |
7891                        TG3_CPMU_EEE_LNKIDL_UART_IDL);
7892
7893                 tw32_f(TG3_CPMU_EEE_CTRL,
7894                        TG3_CPMU_EEE_CTRL_EXIT_20_1_US);
7895
7896                 val = TG3_CPMU_EEEMD_ERLY_L1_XIT_DET |
7897                       TG3_CPMU_EEEMD_LPI_IN_TX |
7898                       TG3_CPMU_EEEMD_LPI_IN_RX |
7899                       TG3_CPMU_EEEMD_EEE_ENABLE;
7900
7901                 if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5717)
7902                         val |= TG3_CPMU_EEEMD_SND_IDX_DET_EN;
7903
7904                 if (tg3_flag(tp, ENABLE_APE))
7905                         val |= TG3_CPMU_EEEMD_APE_TX_DET_EN;
7906
7907                 tw32_f(TG3_CPMU_EEE_MODE, val);
7908
7909                 tw32_f(TG3_CPMU_EEE_DBTMR1,
7910                        TG3_CPMU_DBTMR1_PCIEXIT_2047US |
7911                        TG3_CPMU_DBTMR1_LNKIDLE_2047US);
7912
7913                 tw32_f(TG3_CPMU_EEE_DBTMR2,
7914                        TG3_CPMU_DBTMR2_APE_TX_2047US |
7915                        TG3_CPMU_DBTMR2_TXIDXEQ_2047US);
7916         }
7917
7918         if (reset_phy)
7919                 tg3_phy_reset(tp);
7920
7921         err = tg3_chip_reset(tp);
7922         if (err)
7923                 return err;
7924
7925         tg3_write_sig_legacy(tp, RESET_KIND_INIT);
7926
7927         if (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5784_AX) {
7928                 val = tr32(TG3_CPMU_CTRL);
7929                 val &= ~(CPMU_CTRL_LINK_AWARE_MODE | CPMU_CTRL_LINK_IDLE_MODE);
7930                 tw32(TG3_CPMU_CTRL, val);
7931
7932                 val = tr32(TG3_CPMU_LSPD_10MB_CLK);
7933                 val &= ~CPMU_LSPD_10MB_MACCLK_MASK;
7934                 val |= CPMU_LSPD_10MB_MACCLK_6_25;
7935                 tw32(TG3_CPMU_LSPD_10MB_CLK, val);
7936
7937                 val = tr32(TG3_CPMU_LNK_AWARE_PWRMD);
7938                 val &= ~CPMU_LNK_AWARE_MACCLK_MASK;
7939                 val |= CPMU_LNK_AWARE_MACCLK_6_25;
7940                 tw32(TG3_CPMU_LNK_AWARE_PWRMD, val);
7941
7942                 val = tr32(TG3_CPMU_HST_ACC);
7943                 val &= ~CPMU_HST_ACC_MACCLK_MASK;
7944                 val |= CPMU_HST_ACC_MACCLK_6_25;
7945                 tw32(TG3_CPMU_HST_ACC, val);
7946         }
7947
7948         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_57780) {
7949                 val = tr32(PCIE_PWR_MGMT_THRESH) & ~PCIE_PWR_MGMT_L1_THRESH_MSK;
7950                 val |= PCIE_PWR_MGMT_EXT_ASPM_TMR_EN |
7951                        PCIE_PWR_MGMT_L1_THRESH_4MS;
7952                 tw32(PCIE_PWR_MGMT_THRESH, val);
7953
7954                 val = tr32(TG3_PCIE_EIDLE_DELAY) & ~TG3_PCIE_EIDLE_DELAY_MASK;
7955                 tw32(TG3_PCIE_EIDLE_DELAY, val | TG3_PCIE_EIDLE_DELAY_13_CLKS);
7956
7957                 tw32(TG3_CORR_ERR_STAT, TG3_CORR_ERR_STAT_CLEAR);
7958
7959                 val = tr32(TG3_PCIE_LNKCTL) & ~TG3_PCIE_LNKCTL_L1_PLL_PD_EN;
7960                 tw32(TG3_PCIE_LNKCTL, val | TG3_PCIE_LNKCTL_L1_PLL_PD_DIS);
7961         }
7962
7963         if (tg3_flag(tp, L1PLLPD_EN)) {
7964                 u32 grc_mode = tr32(GRC_MODE);
7965
7966                 /* Access the lower 1K of PL PCIE block registers. */
7967                 val = grc_mode & ~GRC_MODE_PCIE_PORT_MASK;
7968                 tw32(GRC_MODE, val | GRC_MODE_PCIE_PL_SEL);
7969
7970                 val = tr32(TG3_PCIE_TLDLPL_PORT + TG3_PCIE_PL_LO_PHYCTL1);
7971                 tw32(TG3_PCIE_TLDLPL_PORT + TG3_PCIE_PL_LO_PHYCTL1,
7972                      val | TG3_PCIE_PL_LO_PHYCTL1_L1PLLPD_EN);
7973
7974                 tw32(GRC_MODE, grc_mode);
7975         }
7976
7977         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_57765) {
7978                 if (tp->pci_chip_rev_id == CHIPREV_ID_57765_A0) {
7979                         u32 grc_mode = tr32(GRC_MODE);
7980
7981                         /* Access the lower 1K of PL PCIE block registers. */
7982                         val = grc_mode & ~GRC_MODE_PCIE_PORT_MASK;
7983                         tw32(GRC_MODE, val | GRC_MODE_PCIE_PL_SEL);
7984
7985                         val = tr32(TG3_PCIE_TLDLPL_PORT +
7986                                    TG3_PCIE_PL_LO_PHYCTL5);
7987                         tw32(TG3_PCIE_TLDLPL_PORT + TG3_PCIE_PL_LO_PHYCTL5,
7988                              val | TG3_PCIE_PL_LO_PHYCTL5_DIS_L2CLKREQ);
7989
7990                         tw32(GRC_MODE, grc_mode);
7991                 }
7992
7993                 val = tr32(TG3_CPMU_LSPD_10MB_CLK);
7994                 val &= ~CPMU_LSPD_10MB_MACCLK_MASK;
7995                 val |= CPMU_LSPD_10MB_MACCLK_6_25;
7996                 tw32(TG3_CPMU_LSPD_10MB_CLK, val);
7997         }
7998
7999         /* This works around an issue with Athlon chipsets on
8000          * B3 tigon3 silicon.  This bit has no effect on any
8001          * other revision.  But do not set this on PCI Express
8002          * chips and don't even touch the clocks if the CPMU is present.
8003          */
8004         if (!tg3_flag(tp, CPMU_PRESENT)) {
8005                 if (!tg3_flag(tp, PCI_EXPRESS))
8006                         tp->pci_clock_ctrl |= CLOCK_CTRL_DELAY_PCI_GRANT;
8007                 tw32_f(TG3PCI_CLOCK_CTRL, tp->pci_clock_ctrl);
8008         }
8009
8010         if (tp->pci_chip_rev_id == CHIPREV_ID_5704_A0 &&
8011             tg3_flag(tp, PCIX_MODE)) {
8012                 val = tr32(TG3PCI_PCISTATE);
8013                 val |= PCISTATE_RETRY_SAME_DMA;
8014                 tw32(TG3PCI_PCISTATE, val);
8015         }
8016
8017         if (tg3_flag(tp, ENABLE_APE)) {
8018                 /* Allow reads and writes to the
8019                  * APE register and memory space.
8020                  */
8021                 val = tr32(TG3PCI_PCISTATE);
8022                 val |= PCISTATE_ALLOW_APE_CTLSPC_WR |
8023                        PCISTATE_ALLOW_APE_SHMEM_WR |
8024                        PCISTATE_ALLOW_APE_PSPACE_WR;
8025                 tw32(TG3PCI_PCISTATE, val);
8026         }
8027
8028         if (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5704_BX) {
8029                 /* Enable some hw fixes.  */
8030                 val = tr32(TG3PCI_MSI_DATA);
8031                 val |= (1 << 26) | (1 << 28) | (1 << 29);
8032                 tw32(TG3PCI_MSI_DATA, val);
8033         }
8034
8035         /* Descriptor ring init may make accesses to the
8036          * NIC SRAM area to setup the TX descriptors, so we
8037          * can only do this after the hardware has been
8038          * successfully reset.
8039          */
8040         err = tg3_init_rings(tp);
8041         if (err)
8042                 return err;
8043
8044         if (tg3_flag(tp, 57765_PLUS)) {
8045                 val = tr32(TG3PCI_DMA_RW_CTRL) &
8046                       ~DMA_RWCTRL_DIS_CACHE_ALIGNMENT;
8047                 if (tp->pci_chip_rev_id == CHIPREV_ID_57765_A0)
8048                         val &= ~DMA_RWCTRL_CRDRDR_RDMA_MRRS_MSK;
8049                 if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_57765 &&
8050                     GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5717)
8051                         val |= DMA_RWCTRL_TAGGED_STAT_WA;
8052                 tw32(TG3PCI_DMA_RW_CTRL, val | tp->dma_rwctrl);
8053         } else if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5784 &&
8054                    GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5761) {
8055                 /* This value is determined during the probe time DMA
8056                  * engine test, tg3_test_dma.
8057                  */
8058                 tw32(TG3PCI_DMA_RW_CTRL, tp->dma_rwctrl);
8059         }
8060
8061         tp->grc_mode &= ~(GRC_MODE_HOST_SENDBDS |
8062                           GRC_MODE_4X_NIC_SEND_RINGS |
8063                           GRC_MODE_NO_TX_PHDR_CSUM |
8064                           GRC_MODE_NO_RX_PHDR_CSUM);
8065         tp->grc_mode |= GRC_MODE_HOST_SENDBDS;
8066
8067         /* Pseudo-header checksum is done by hardware logic and not
8068          * the offload processers, so make the chip do the pseudo-
8069          * header checksums on receive.  For transmit it is more
8070          * convenient to do the pseudo-header checksum in software
8071          * as Linux does that on transmit for us in all cases.
8072          */
8073         tp->grc_mode |= GRC_MODE_NO_TX_PHDR_CSUM;
8074
8075         tw32(GRC_MODE,
8076              tp->grc_mode |
8077              (GRC_MODE_IRQ_ON_MAC_ATTN | GRC_MODE_HOST_STACKUP));
8078
8079         /* Setup the timer prescalar register.  Clock is always 66Mhz. */
8080         val = tr32(GRC_MISC_CFG);
8081         val &= ~0xff;
8082         val |= (65 << GRC_MISC_CFG_PRESCALAR_SHIFT);
8083         tw32(GRC_MISC_CFG, val);
8084
8085         /* Initialize MBUF/DESC pool. */
8086         if (tg3_flag(tp, 5750_PLUS)) {
8087                 /* Do nothing.  */
8088         } else if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5705) {
8089                 tw32(BUFMGR_MB_POOL_ADDR, NIC_SRAM_MBUF_POOL_BASE);
8090                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704)
8091                         tw32(BUFMGR_MB_POOL_SIZE, NIC_SRAM_MBUF_POOL_SIZE64);
8092                 else
8093                         tw32(BUFMGR_MB_POOL_SIZE, NIC_SRAM_MBUF_POOL_SIZE96);
8094                 tw32(BUFMGR_DMA_DESC_POOL_ADDR, NIC_SRAM_DMA_DESC_POOL_BASE);
8095                 tw32(BUFMGR_DMA_DESC_POOL_SIZE, NIC_SRAM_DMA_DESC_POOL_SIZE);
8096         } else if (tg3_flag(tp, TSO_CAPABLE)) {
8097                 int fw_len;
8098
8099                 fw_len = tp->fw_len;
8100                 fw_len = (fw_len + (0x80 - 1)) & ~(0x80 - 1);
8101                 tw32(BUFMGR_MB_POOL_ADDR,
8102                      NIC_SRAM_MBUF_POOL_BASE5705 + fw_len);
8103                 tw32(BUFMGR_MB_POOL_SIZE,
8104                      NIC_SRAM_MBUF_POOL_SIZE5705 - fw_len - 0xa00);
8105         }
8106
8107         if (tp->dev->mtu <= ETH_DATA_LEN) {
8108                 tw32(BUFMGR_MB_RDMA_LOW_WATER,
8109                      tp->bufmgr_config.mbuf_read_dma_low_water);
8110                 tw32(BUFMGR_MB_MACRX_LOW_WATER,
8111                      tp->bufmgr_config.mbuf_mac_rx_low_water);
8112                 tw32(BUFMGR_MB_HIGH_WATER,
8113                      tp->bufmgr_config.mbuf_high_water);
8114         } else {
8115                 tw32(BUFMGR_MB_RDMA_LOW_WATER,
8116                      tp->bufmgr_config.mbuf_read_dma_low_water_jumbo);
8117                 tw32(BUFMGR_MB_MACRX_LOW_WATER,
8118                      tp->bufmgr_config.mbuf_mac_rx_low_water_jumbo);
8119                 tw32(BUFMGR_MB_HIGH_WATER,
8120                      tp->bufmgr_config.mbuf_high_water_jumbo);
8121         }
8122         tw32(BUFMGR_DMA_LOW_WATER,
8123              tp->bufmgr_config.dma_low_water);
8124         tw32(BUFMGR_DMA_HIGH_WATER,
8125              tp->bufmgr_config.dma_high_water);
8126
8127         val = BUFMGR_MODE_ENABLE | BUFMGR_MODE_ATTN_ENABLE;
8128         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5719)
8129                 val |= BUFMGR_MODE_NO_TX_UNDERRUN;
8130         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5717 ||
8131             tp->pci_chip_rev_id == CHIPREV_ID_5719_A0 ||
8132             tp->pci_chip_rev_id == CHIPREV_ID_5720_A0)
8133                 val |= BUFMGR_MODE_MBLOW_ATTN_ENAB;
8134         tw32(BUFMGR_MODE, val);
8135         for (i = 0; i < 2000; i++) {
8136                 if (tr32(BUFMGR_MODE) & BUFMGR_MODE_ENABLE)
8137                         break;
8138                 udelay(10);
8139         }
8140         if (i >= 2000) {
8141                 netdev_err(tp->dev, "%s cannot enable BUFMGR\n", __func__);
8142                 return -ENODEV;
8143         }
8144
8145         if (tp->pci_chip_rev_id == CHIPREV_ID_5906_A1)
8146                 tw32(ISO_PKT_TX, (tr32(ISO_PKT_TX) & ~0x3) | 0x2);
8147
8148         tg3_setup_rxbd_thresholds(tp);
8149
8150         /* Initialize TG3_BDINFO's at:
8151          *  RCVDBDI_STD_BD:     standard eth size rx ring
8152          *  RCVDBDI_JUMBO_BD:   jumbo frame rx ring
8153          *  RCVDBDI_MINI_BD:    small frame rx ring (??? does not work)
8154          *
8155          * like so:
8156          *  TG3_BDINFO_HOST_ADDR:       high/low parts of DMA address of ring
8157          *  TG3_BDINFO_MAXLEN_FLAGS:    (rx max buffer size << 16) |
8158          *                              ring attribute flags
8159          *  TG3_BDINFO_NIC_ADDR:        location of descriptors in nic SRAM
8160          *
8161          * Standard receive ring @ NIC_SRAM_RX_BUFFER_DESC, 512 entries.
8162          * Jumbo receive ring @ NIC_SRAM_RX_JUMBO_BUFFER_DESC, 256 entries.
8163          *
8164          * The size of each ring is fixed in the firmware, but the location is
8165          * configurable.
8166          */
8167         tw32(RCVDBDI_STD_BD + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_HIGH,
8168              ((u64) tpr->rx_std_mapping >> 32));
8169         tw32(RCVDBDI_STD_BD + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_LOW,
8170              ((u64) tpr->rx_std_mapping & 0xffffffff));
8171         if (!tg3_flag(tp, 5717_PLUS))
8172                 tw32(RCVDBDI_STD_BD + TG3_BDINFO_NIC_ADDR,
8173                      NIC_SRAM_RX_BUFFER_DESC);
8174
8175         /* Disable the mini ring */
8176         if (!tg3_flag(tp, 5705_PLUS))
8177                 tw32(RCVDBDI_MINI_BD + TG3_BDINFO_MAXLEN_FLAGS,
8178                      BDINFO_FLAGS_DISABLED);
8179
8180         /* Program the jumbo buffer descriptor ring control
8181          * blocks on those devices that have them.
8182          */
8183         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5719 ||
8184             (tg3_flag(tp, JUMBO_CAPABLE) && !tg3_flag(tp, 5780_CLASS))) {
8185
8186                 if (tg3_flag(tp, JUMBO_RING_ENABLE)) {
8187                         tw32(RCVDBDI_JUMBO_BD + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_HIGH,
8188                              ((u64) tpr->rx_jmb_mapping >> 32));
8189                         tw32(RCVDBDI_JUMBO_BD + TG3_BDINFO_HOST_ADDR + TG3_64BIT_REG_LOW,
8190                              ((u64) tpr->rx_jmb_mapping & 0xffffffff));
8191                         val = TG3_RX_JMB_RING_SIZE(tp) <<
8192                               BDINFO_FLAGS_MAXLEN_SHIFT;
8193                         tw32(RCVDBDI_JUMBO_BD + TG3_BDINFO_MAXLEN_FLAGS,
8194                              val | BDINFO_FLAGS_USE_EXT_RECV);
8195                         if (!tg3_flag(tp, USE_JUMBO_BDFLAG) ||
8196                             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_57765)
8197                                 tw32(RCVDBDI_JUMBO_BD + TG3_BDINFO_NIC_ADDR,
8198                                      NIC_SRAM_RX_JUMBO_BUFFER_DESC);
8199                 } else {
8200                         tw32(RCVDBDI_JUMBO_BD + TG3_BDINFO_MAXLEN_FLAGS,
8201                              BDINFO_FLAGS_DISABLED);
8202                 }
8203
8204                 if (tg3_flag(tp, 57765_PLUS)) {
8205                         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_57765)
8206                                 val = TG3_RX_STD_MAX_SIZE_5700;
8207                         else
8208                                 val = TG3_RX_STD_MAX_SIZE_5717;
8209                         val <<= BDINFO_FLAGS_MAXLEN_SHIFT;
8210                         val |= (TG3_RX_STD_DMA_SZ << 2);
8211                 } else
8212                         val = TG3_RX_STD_DMA_SZ << BDINFO_FLAGS_MAXLEN_SHIFT;
8213         } else
8214                 val = TG3_RX_STD_MAX_SIZE_5700 << BDINFO_FLAGS_MAXLEN_SHIFT;
8215
8216         tw32(RCVDBDI_STD_BD + TG3_BDINFO_MAXLEN_FLAGS, val);
8217
8218         tpr->rx_std_prod_idx = tp->rx_pending;
8219         tw32_rx_mbox(TG3_RX_STD_PROD_IDX_REG, tpr->rx_std_prod_idx);
8220
8221         tpr->rx_jmb_prod_idx =
8222                 tg3_flag(tp, JUMBO_RING_ENABLE) ? tp->rx_jumbo_pending : 0;
8223         tw32_rx_mbox(TG3_RX_JMB_PROD_IDX_REG, tpr->rx_jmb_prod_idx);
8224
8225         tg3_rings_reset(tp);
8226
8227         /* Initialize MAC address and backoff seed. */
8228         __tg3_set_mac_addr(tp, 0);
8229
8230         /* MTU + ethernet header + FCS + optional VLAN tag */
8231         tw32(MAC_RX_MTU_SIZE,
8232              tp->dev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN);
8233
8234         /* The slot time is changed by tg3_setup_phy if we
8235          * run at gigabit with half duplex.
8236          */
8237         val = (2 << TX_LENGTHS_IPG_CRS_SHIFT) |
8238               (6 << TX_LENGTHS_IPG_SHIFT) |
8239               (32 << TX_LENGTHS_SLOT_TIME_SHIFT);
8240
8241         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5720)
8242                 val |= tr32(MAC_TX_LENGTHS) &
8243                        (TX_LENGTHS_JMB_FRM_LEN_MSK |
8244                         TX_LENGTHS_CNT_DWN_VAL_MSK);
8245
8246         tw32(MAC_TX_LENGTHS, val);
8247
8248         /* Receive rules. */
8249         tw32(MAC_RCV_RULE_CFG, RCV_RULE_CFG_DEFAULT_CLASS);
8250         tw32(RCVLPC_CONFIG, 0x0181);
8251
8252         /* Calculate RDMAC_MODE setting early, we need it to determine
8253          * the RCVLPC_STATE_ENABLE mask.
8254          */
8255         rdmac_mode = (RDMAC_MODE_ENABLE | RDMAC_MODE_TGTABORT_ENAB |
8256                       RDMAC_MODE_MSTABORT_ENAB | RDMAC_MODE_PARITYERR_ENAB |
8257                       RDMAC_MODE_ADDROFLOW_ENAB | RDMAC_MODE_FIFOOFLOW_ENAB |
8258                       RDMAC_MODE_FIFOURUN_ENAB | RDMAC_MODE_FIFOOREAD_ENAB |
8259                       RDMAC_MODE_LNGREAD_ENAB);
8260
8261         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5717)
8262                 rdmac_mode |= RDMAC_MODE_MULT_DMA_RD_DIS;
8263
8264         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 ||
8265             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5785 ||
8266             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_57780)
8267                 rdmac_mode |= RDMAC_MODE_BD_SBD_CRPT_ENAB |
8268                               RDMAC_MODE_MBUF_RBD_CRPT_ENAB |
8269                               RDMAC_MODE_MBUF_SBD_CRPT_ENAB;
8270
8271         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705 &&
8272             tp->pci_chip_rev_id != CHIPREV_ID_5705_A0) {
8273                 if (tg3_flag(tp, TSO_CAPABLE) &&
8274                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) {
8275                         rdmac_mode |= RDMAC_MODE_FIFO_SIZE_128;
8276                 } else if (!(tr32(TG3PCI_PCISTATE) & PCISTATE_BUS_SPEED_HIGH) &&
8277                            !tg3_flag(tp, IS_5788)) {
8278                         rdmac_mode |= RDMAC_MODE_FIFO_LONG_BURST;
8279                 }
8280         }
8281
8282         if (tg3_flag(tp, PCI_EXPRESS))
8283                 rdmac_mode |= RDMAC_MODE_FIFO_LONG_BURST;
8284
8285         if (tg3_flag(tp, HW_TSO_1) ||
8286             tg3_flag(tp, HW_TSO_2) ||
8287             tg3_flag(tp, HW_TSO_3))
8288                 rdmac_mode |= RDMAC_MODE_IPV4_LSO_EN;
8289
8290         if (tg3_flag(tp, HW_TSO_3) ||
8291             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5785 ||
8292             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_57780)
8293                 rdmac_mode |= RDMAC_MODE_IPV6_LSO_EN;
8294
8295         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5720)
8296                 rdmac_mode |= tr32(RDMAC_MODE) & RDMAC_MODE_H2BNC_VLAN_DET;
8297
8298         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761 ||
8299             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 ||
8300             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5785 ||
8301             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_57780 ||
8302             tg3_flag(tp, 57765_PLUS)) {
8303                 val = tr32(TG3_RDMA_RSRVCTRL_REG);
8304                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5719 ||
8305                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5720) {
8306                         val &= ~(TG3_RDMA_RSRVCTRL_TXMRGN_MASK |
8307                                  TG3_RDMA_RSRVCTRL_FIFO_LWM_MASK |
8308                                  TG3_RDMA_RSRVCTRL_FIFO_HWM_MASK);
8309                         val |= TG3_RDMA_RSRVCTRL_TXMRGN_320B |
8310                                TG3_RDMA_RSRVCTRL_FIFO_LWM_1_5K |
8311                                TG3_RDMA_RSRVCTRL_FIFO_HWM_1_5K;
8312                 }
8313                 tw32(TG3_RDMA_RSRVCTRL_REG,
8314                      val | TG3_RDMA_RSRVCTRL_FIFO_OFLW_FIX);
8315         }
8316
8317         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5719 ||
8318             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5720) {
8319                 val = tr32(TG3_LSO_RD_DMA_CRPTEN_CTRL);
8320                 tw32(TG3_LSO_RD_DMA_CRPTEN_CTRL, val |
8321                      TG3_LSO_RD_DMA_CRPTEN_CTRL_BLEN_BD_4K |
8322                      TG3_LSO_RD_DMA_CRPTEN_CTRL_BLEN_LSO_4K);
8323         }
8324
8325         /* Receive/send statistics. */
8326         if (tg3_flag(tp, 5750_PLUS)) {
8327                 val = tr32(RCVLPC_STATS_ENABLE);
8328                 val &= ~RCVLPC_STATSENAB_DACK_FIX;
8329                 tw32(RCVLPC_STATS_ENABLE, val);
8330         } else if ((rdmac_mode & RDMAC_MODE_FIFO_SIZE_128) &&
8331                    tg3_flag(tp, TSO_CAPABLE)) {
8332                 val = tr32(RCVLPC_STATS_ENABLE);
8333                 val &= ~RCVLPC_STATSENAB_LNGBRST_RFIX;
8334                 tw32(RCVLPC_STATS_ENABLE, val);
8335         } else {
8336                 tw32(RCVLPC_STATS_ENABLE, 0xffffff);
8337         }
8338         tw32(RCVLPC_STATSCTRL, RCVLPC_STATSCTRL_ENABLE);
8339         tw32(SNDDATAI_STATSENAB, 0xffffff);
8340         tw32(SNDDATAI_STATSCTRL,
8341              (SNDDATAI_SCTRL_ENABLE |
8342               SNDDATAI_SCTRL_FASTUPD));
8343
8344         /* Setup host coalescing engine. */
8345         tw32(HOSTCC_MODE, 0);
8346         for (i = 0; i < 2000; i++) {
8347                 if (!(tr32(HOSTCC_MODE) & HOSTCC_MODE_ENABLE))
8348                         break;
8349                 udelay(10);
8350         }
8351
8352         __tg3_set_coalesce(tp, &tp->coal);
8353
8354         if (!tg3_flag(tp, 5705_PLUS)) {
8355                 /* Status/statistics block address.  See tg3_timer,
8356                  * the tg3_periodic_fetch_stats call there, and
8357                  * tg3_get_stats to see how this works for 5705/5750 chips.
8358                  */
8359                 tw32(HOSTCC_STATS_BLK_HOST_ADDR + TG3_64BIT_REG_HIGH,
8360                      ((u64) tp->stats_mapping >> 32));
8361                 tw32(HOSTCC_STATS_BLK_HOST_ADDR + TG3_64BIT_REG_LOW,
8362                      ((u64) tp->stats_mapping & 0xffffffff));
8363                 tw32(HOSTCC_STATS_BLK_NIC_ADDR, NIC_SRAM_STATS_BLK);
8364
8365                 tw32(HOSTCC_STATUS_BLK_NIC_ADDR, NIC_SRAM_STATUS_BLK);
8366
8367                 /* Clear statistics and status block memory areas */
8368                 for (i = NIC_SRAM_STATS_BLK;
8369                      i < NIC_SRAM_STATUS_BLK + TG3_HW_STATUS_SIZE;
8370                      i += sizeof(u32)) {
8371                         tg3_write_mem(tp, i, 0);
8372                         udelay(40);
8373                 }
8374         }
8375
8376         tw32(HOSTCC_MODE, HOSTCC_MODE_ENABLE | tp->coalesce_mode);
8377
8378         tw32(RCVCC_MODE, RCVCC_MODE_ENABLE | RCVCC_MODE_ATTN_ENABLE);
8379         tw32(RCVLPC_MODE, RCVLPC_MODE_ENABLE);
8380         if (!tg3_flag(tp, 5705_PLUS))
8381                 tw32(RCVLSC_MODE, RCVLSC_MODE_ENABLE | RCVLSC_MODE_ATTN_ENABLE);
8382
8383         if (tp->phy_flags & TG3_PHYFLG_MII_SERDES) {
8384                 tp->phy_flags &= ~TG3_PHYFLG_PARALLEL_DETECT;
8385                 /* reset to prevent losing 1st rx packet intermittently */
8386                 tw32_f(MAC_RX_MODE, RX_MODE_RESET);
8387                 udelay(10);
8388         }
8389
8390         if (tg3_flag(tp, ENABLE_APE))
8391                 tp->mac_mode = MAC_MODE_APE_TX_EN | MAC_MODE_APE_RX_EN;
8392         else
8393                 tp->mac_mode = 0;
8394         tp->mac_mode |= MAC_MODE_TXSTAT_ENABLE | MAC_MODE_RXSTAT_ENABLE |
8395                 MAC_MODE_TDE_ENABLE | MAC_MODE_RDE_ENABLE | MAC_MODE_FHDE_ENABLE;
8396         if (!tg3_flag(tp, 5705_PLUS) &&
8397             !(tp->phy_flags & TG3_PHYFLG_PHY_SERDES) &&
8398             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700)
8399                 tp->mac_mode |= MAC_MODE_LINK_POLARITY;
8400         tw32_f(MAC_MODE, tp->mac_mode | MAC_MODE_RXSTAT_CLEAR | MAC_MODE_TXSTAT_CLEAR);
8401         udelay(40);
8402
8403         /* tp->grc_local_ctrl is partially set up during tg3_get_invariants().
8404          * If TG3_FLAG_IS_NIC is zero, we should read the
8405          * register to preserve the GPIO settings for LOMs. The GPIOs,
8406          * whether used as inputs or outputs, are set by boot code after
8407          * reset.
8408          */
8409         if (!tg3_flag(tp, IS_NIC)) {
8410                 u32 gpio_mask;
8411
8412                 gpio_mask = GRC_LCLCTRL_GPIO_OE0 | GRC_LCLCTRL_GPIO_OE1 |
8413                             GRC_LCLCTRL_GPIO_OE2 | GRC_LCLCTRL_GPIO_OUTPUT0 |
8414                             GRC_LCLCTRL_GPIO_OUTPUT1 | GRC_LCLCTRL_GPIO_OUTPUT2;
8415
8416                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5752)
8417                         gpio_mask |= GRC_LCLCTRL_GPIO_OE3 |
8418                                      GRC_LCLCTRL_GPIO_OUTPUT3;
8419
8420                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755)
8421                         gpio_mask |= GRC_LCLCTRL_GPIO_UART_SEL;
8422
8423                 tp->grc_local_ctrl &= ~gpio_mask;
8424                 tp->grc_local_ctrl |= tr32(GRC_LOCAL_CTRL) & gpio_mask;
8425
8426                 /* GPIO1 must be driven high for eeprom write protect */
8427                 if (tg3_flag(tp, EEPROM_WRITE_PROT))
8428                         tp->grc_local_ctrl |= (GRC_LCLCTRL_GPIO_OE1 |
8429                                                GRC_LCLCTRL_GPIO_OUTPUT1);
8430         }
8431         tw32_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl);
8432         udelay(100);
8433
8434         if (tg3_flag(tp, USING_MSIX) && tp->irq_cnt > 1) {
8435                 val = tr32(MSGINT_MODE);
8436                 val |= MSGINT_MODE_MULTIVEC_EN | MSGINT_MODE_ENABLE;
8437                 tw32(MSGINT_MODE, val);
8438         }
8439
8440         if (!tg3_flag(tp, 5705_PLUS)) {
8441                 tw32_f(DMAC_MODE, DMAC_MODE_ENABLE);
8442                 udelay(40);
8443         }
8444
8445         val = (WDMAC_MODE_ENABLE | WDMAC_MODE_TGTABORT_ENAB |
8446                WDMAC_MODE_MSTABORT_ENAB | WDMAC_MODE_PARITYERR_ENAB |
8447                WDMAC_MODE_ADDROFLOW_ENAB | WDMAC_MODE_FIFOOFLOW_ENAB |
8448                WDMAC_MODE_FIFOURUN_ENAB | WDMAC_MODE_FIFOOREAD_ENAB |
8449                WDMAC_MODE_LNGREAD_ENAB);
8450
8451         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705 &&
8452             tp->pci_chip_rev_id != CHIPREV_ID_5705_A0) {
8453                 if (tg3_flag(tp, TSO_CAPABLE) &&
8454                     (tp->pci_chip_rev_id == CHIPREV_ID_5705_A1 ||
8455                      tp->pci_chip_rev_id == CHIPREV_ID_5705_A2)) {
8456                         /* nothing */
8457                 } else if (!(tr32(TG3PCI_PCISTATE) & PCISTATE_BUS_SPEED_HIGH) &&
8458                            !tg3_flag(tp, IS_5788)) {
8459                         val |= WDMAC_MODE_RX_ACCEL;
8460                 }
8461         }
8462
8463         /* Enable host coalescing bug fix */
8464         if (tg3_flag(tp, 5755_PLUS))
8465                 val |= WDMAC_MODE_STATUS_TAG_FIX;
8466
8467         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5785)
8468                 val |= WDMAC_MODE_BURST_ALL_DATA;
8469
8470         tw32_f(WDMAC_MODE, val);
8471         udelay(40);
8472
8473         if (tg3_flag(tp, PCIX_MODE)) {
8474                 u16 pcix_cmd;
8475
8476                 pci_read_config_word(tp->pdev, tp->pcix_cap + PCI_X_CMD,
8477                                      &pcix_cmd);
8478                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703) {
8479                         pcix_cmd &= ~PCI_X_CMD_MAX_READ;
8480                         pcix_cmd |= PCI_X_CMD_READ_2K;
8481                 } else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) {
8482                         pcix_cmd &= ~(PCI_X_CMD_MAX_SPLIT | PCI_X_CMD_MAX_READ);
8483                         pcix_cmd |= PCI_X_CMD_READ_2K;
8484                 }
8485                 pci_write_config_word(tp->pdev, tp->pcix_cap + PCI_X_CMD,
8486                                       pcix_cmd);
8487         }
8488
8489         tw32_f(RDMAC_MODE, rdmac_mode);
8490         udelay(40);
8491
8492         tw32(RCVDCC_MODE, RCVDCC_MODE_ENABLE | RCVDCC_MODE_ATTN_ENABLE);
8493         if (!tg3_flag(tp, 5705_PLUS))
8494                 tw32(MBFREE_MODE, MBFREE_MODE_ENABLE);
8495
8496         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761)
8497                 tw32(SNDDATAC_MODE,
8498                      SNDDATAC_MODE_ENABLE | SNDDATAC_MODE_CDELAY);
8499         else
8500                 tw32(SNDDATAC_MODE, SNDDATAC_MODE_ENABLE);
8501
8502         tw32(SNDBDC_MODE, SNDBDC_MODE_ENABLE | SNDBDC_MODE_ATTN_ENABLE);
8503         tw32(RCVBDI_MODE, RCVBDI_MODE_ENABLE | RCVBDI_MODE_RCB_ATTN_ENAB);
8504         val = RCVDBDI_MODE_ENABLE | RCVDBDI_MODE_INV_RING_SZ;
8505         if (tg3_flag(tp, LRG_PROD_RING_CAP))
8506                 val |= RCVDBDI_MODE_LRG_RING_SZ;
8507         tw32(RCVDBDI_MODE, val);
8508         tw32(SNDDATAI_MODE, SNDDATAI_MODE_ENABLE);
8509         if (tg3_flag(tp, HW_TSO_1) ||
8510             tg3_flag(tp, HW_TSO_2) ||
8511             tg3_flag(tp, HW_TSO_3))
8512                 tw32(SNDDATAI_MODE, SNDDATAI_MODE_ENABLE | 0x8);
8513         val = SNDBDI_MODE_ENABLE | SNDBDI_MODE_ATTN_ENABLE;
8514         if (tg3_flag(tp, ENABLE_TSS))
8515                 val |= SNDBDI_MODE_MULTI_TXQ_EN;
8516         tw32(SNDBDI_MODE, val);
8517         tw32(SNDBDS_MODE, SNDBDS_MODE_ENABLE | SNDBDS_MODE_ATTN_ENABLE);
8518
8519         if (tp->pci_chip_rev_id == CHIPREV_ID_5701_A0) {
8520                 err = tg3_load_5701_a0_firmware_fix(tp);
8521                 if (err)
8522                         return err;
8523         }
8524
8525         if (tg3_flag(tp, TSO_CAPABLE)) {
8526                 err = tg3_load_tso_firmware(tp);
8527                 if (err)
8528                         return err;
8529         }
8530
8531         tp->tx_mode = TX_MODE_ENABLE;
8532
8533         if (tg3_flag(tp, 5755_PLUS) ||
8534             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906)
8535                 tp->tx_mode |= TX_MODE_MBUF_LOCKUP_FIX;
8536
8537         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5720) {
8538                 val = TX_MODE_JMB_FRM_LEN | TX_MODE_CNT_DN_MODE;
8539                 tp->tx_mode &= ~val;
8540                 tp->tx_mode |= tr32(MAC_TX_MODE) & val;
8541         }
8542
8543         tw32_f(MAC_TX_MODE, tp->tx_mode);
8544         udelay(100);
8545
8546         if (tg3_flag(tp, ENABLE_RSS)) {
8547                 u32 reg = MAC_RSS_INDIR_TBL_0;
8548                 u8 *ent = (u8 *)&val;
8549
8550                 /* Setup the indirection table */
8551                 for (i = 0; i < TG3_RSS_INDIR_TBL_SIZE; i++) {
8552                         int idx = i % sizeof(val);
8553
8554                         ent[idx] = i % (tp->irq_cnt - 1);
8555                         if (idx == sizeof(val) - 1) {
8556                                 tw32(reg, val);
8557                                 reg += 4;
8558                         }
8559                 }
8560
8561                 /* Setup the "secret" hash key. */
8562                 tw32(MAC_RSS_HASH_KEY_0, 0x5f865437);
8563                 tw32(MAC_RSS_HASH_KEY_1, 0xe4ac62cc);
8564                 tw32(MAC_RSS_HASH_KEY_2, 0x50103a45);
8565                 tw32(MAC_RSS_HASH_KEY_3, 0x36621985);
8566                 tw32(MAC_RSS_HASH_KEY_4, 0xbf14c0e8);
8567                 tw32(MAC_RSS_HASH_KEY_5, 0x1bc27a1e);
8568                 tw32(MAC_RSS_HASH_KEY_6, 0x84f4b556);
8569                 tw32(MAC_RSS_HASH_KEY_7, 0x094ea6fe);
8570                 tw32(MAC_RSS_HASH_KEY_8, 0x7dda01e7);
8571                 tw32(MAC_RSS_HASH_KEY_9, 0xc04d7481);
8572         }
8573
8574         tp->rx_mode = RX_MODE_ENABLE;
8575         if (tg3_flag(tp, 5755_PLUS))
8576                 tp->rx_mode |= RX_MODE_IPV6_CSUM_ENABLE;
8577
8578         if (tg3_flag(tp, ENABLE_RSS))
8579                 tp->rx_mode |= RX_MODE_RSS_ENABLE |
8580                                RX_MODE_RSS_ITBL_HASH_BITS_7 |
8581                                RX_MODE_RSS_IPV6_HASH_EN |
8582                                RX_MODE_RSS_TCP_IPV6_HASH_EN |
8583                                RX_MODE_RSS_IPV4_HASH_EN |
8584                                RX_MODE_RSS_TCP_IPV4_HASH_EN;
8585
8586         tw32_f(MAC_RX_MODE, tp->rx_mode);
8587         udelay(10);
8588
8589         tw32(MAC_LED_CTRL, tp->led_ctrl);
8590
8591         tw32(MAC_MI_STAT, MAC_MI_STAT_LNKSTAT_ATTN_ENAB);
8592         if (tp->phy_flags & TG3_PHYFLG_PHY_SERDES) {
8593                 tw32_f(MAC_RX_MODE, RX_MODE_RESET);
8594                 udelay(10);
8595         }
8596         tw32_f(MAC_RX_MODE, tp->rx_mode);
8597         udelay(10);
8598
8599         if (tp->phy_flags & TG3_PHYFLG_PHY_SERDES) {
8600                 if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) &&
8601                         !(tp->phy_flags & TG3_PHYFLG_SERDES_PREEMPHASIS)) {
8602                         /* Set drive transmission level to 1.2V  */
8603                         /* only if the signal pre-emphasis bit is not set  */
8604                         val = tr32(MAC_SERDES_CFG);
8605                         val &= 0xfffff000;
8606                         val |= 0x880;
8607                         tw32(MAC_SERDES_CFG, val);
8608                 }
8609                 if (tp->pci_chip_rev_id == CHIPREV_ID_5703_A1)
8610                         tw32(MAC_SERDES_CFG, 0x616000);
8611         }
8612
8613         /* Prevent chip from dropping frames when flow control
8614          * is enabled.
8615          */
8616         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_57765)
8617                 val = 1;
8618         else
8619                 val = 2;
8620         tw32_f(MAC_LOW_WMARK_MAX_RX_FRAME, val);
8621
8622         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704 &&
8623             (tp->phy_flags & TG3_PHYFLG_PHY_SERDES)) {
8624                 /* Use hardware link auto-negotiation */
8625                 tg3_flag_set(tp, HW_AUTONEG);
8626         }
8627
8628         if ((tp->phy_flags & TG3_PHYFLG_MII_SERDES) &&
8629             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714)) {
8630                 u32 tmp;
8631
8632                 tmp = tr32(SERDES_RX_CTRL);
8633                 tw32(SERDES_RX_CTRL, tmp | SERDES_RX_SIG_DETECT);
8634                 tp->grc_local_ctrl &= ~GRC_LCLCTRL_USE_EXT_SIG_DETECT;
8635                 tp->grc_local_ctrl |= GRC_LCLCTRL_USE_SIG_DETECT;
8636                 tw32(GRC_LOCAL_CTRL, tp->grc_local_ctrl);
8637         }
8638
8639         if (!tg3_flag(tp, USE_PHYLIB)) {
8640                 if (tp->phy_flags & TG3_PHYFLG_IS_LOW_POWER) {
8641                         tp->phy_flags &= ~TG3_PHYFLG_IS_LOW_POWER;
8642                         tp->link_config.speed = tp->link_config.orig_speed;
8643                         tp->link_config.duplex = tp->link_config.orig_duplex;
8644                         tp->link_config.autoneg = tp->link_config.orig_autoneg;
8645                 }
8646
8647                 err = tg3_setup_phy(tp, 0);
8648                 if (err)
8649                         return err;
8650
8651                 if (!(tp->phy_flags & TG3_PHYFLG_PHY_SERDES) &&
8652                     !(tp->phy_flags & TG3_PHYFLG_IS_FET)) {
8653                         u32 tmp;
8654
8655                         /* Clear CRC stats. */
8656                         if (!tg3_readphy(tp, MII_TG3_TEST1, &tmp)) {
8657                                 tg3_writephy(tp, MII_TG3_TEST1,
8658                                              tmp | MII_TG3_TEST1_CRC_EN);
8659                                 tg3_readphy(tp, MII_TG3_RXR_COUNTERS, &tmp);
8660                         }
8661                 }
8662         }
8663
8664         __tg3_set_rx_mode(tp->dev);
8665
8666         /* Initialize receive rules. */
8667         tw32(MAC_RCV_RULE_0,  0xc2000000 & RCV_RULE_DISABLE_MASK);
8668         tw32(MAC_RCV_VALUE_0, 0xffffffff & RCV_RULE_DISABLE_MASK);
8669         tw32(MAC_RCV_RULE_1,  0x86000004 & RCV_RULE_DISABLE_MASK);
8670         tw32(MAC_RCV_VALUE_1, 0xffffffff & RCV_RULE_DISABLE_MASK);
8671
8672         if (tg3_flag(tp, 5705_PLUS) && !tg3_flag(tp, 5780_CLASS))
8673                 limit = 8;
8674         else
8675                 limit = 16;
8676         if (tg3_flag(tp, ENABLE_ASF))
8677                 limit -= 4;
8678         switch (limit) {
8679         case 16:
8680                 tw32(MAC_RCV_RULE_15,  0); tw32(MAC_RCV_VALUE_15,  0);
8681         case 15:
8682                 tw32(MAC_RCV_RULE_14,  0); tw32(MAC_RCV_VALUE_14,  0);
8683         case 14:
8684                 tw32(MAC_RCV_RULE_13,  0); tw32(MAC_RCV_VALUE_13,  0);
8685         case 13:
8686                 tw32(MAC_RCV_RULE_12,  0); tw32(MAC_RCV_VALUE_12,  0);
8687         case 12:
8688                 tw32(MAC_RCV_RULE_11,  0); tw32(MAC_RCV_VALUE_11,  0);
8689         case 11:
8690                 tw32(MAC_RCV_RULE_10,  0); tw32(MAC_RCV_VALUE_10,  0);
8691         case 10:
8692                 tw32(MAC_RCV_RULE_9,  0); tw32(MAC_RCV_VALUE_9,  0);
8693         case 9:
8694                 tw32(MAC_RCV_RULE_8,  0); tw32(MAC_RCV_VALUE_8,  0);
8695         case 8:
8696                 tw32(MAC_RCV_RULE_7,  0); tw32(MAC_RCV_VALUE_7,  0);
8697         case 7:
8698                 tw32(MAC_RCV_RULE_6,  0); tw32(MAC_RCV_VALUE_6,  0);
8699         case 6:
8700                 tw32(MAC_RCV_RULE_5,  0); tw32(MAC_RCV_VALUE_5,  0);
8701         case 5:
8702                 tw32(MAC_RCV_RULE_4,  0); tw32(MAC_RCV_VALUE_4,  0);
8703         case 4:
8704                 /* tw32(MAC_RCV_RULE_3,  0); tw32(MAC_RCV_VALUE_3,  0); */
8705         case 3:
8706                 /* tw32(MAC_RCV_RULE_2,  0); tw32(MAC_RCV_VALUE_2,  0); */
8707         case 2:
8708         case 1:
8709
8710         default:
8711                 break;
8712         }
8713
8714         if (tg3_flag(tp, ENABLE_APE))
8715                 /* Write our heartbeat update interval to APE. */
8716                 tg3_ape_write32(tp, TG3_APE_HOST_HEARTBEAT_INT_MS,
8717                                 APE_HOST_HEARTBEAT_INT_DISABLE);
8718
8719         tg3_write_sig_post_reset(tp, RESET_KIND_INIT);
8720
8721         return 0;
8722 }
8723
8724 /* Called at device open time to get the chip ready for
8725  * packet processing.  Invoked with tp->lock held.
8726  */
8727 static int tg3_init_hw(struct tg3 *tp, int reset_phy)
8728 {
8729         tg3_switch_clocks(tp);
8730
8731         tw32(TG3PCI_MEM_WIN_BASE_ADDR, 0);
8732
8733         return tg3_reset_hw(tp, reset_phy);
8734 }
8735
8736 #define TG3_STAT_ADD32(PSTAT, REG) \
8737 do {    u32 __val = tr32(REG); \
8738         (PSTAT)->low += __val; \
8739         if ((PSTAT)->low < __val) \
8740                 (PSTAT)->high += 1; \
8741 } while (0)
8742
8743 static void tg3_periodic_fetch_stats(struct tg3 *tp)
8744 {
8745         struct tg3_hw_stats *sp = tp->hw_stats;
8746
8747         if (!netif_carrier_ok(tp->dev))
8748                 return;
8749
8750         TG3_STAT_ADD32(&sp->tx_octets, MAC_TX_STATS_OCTETS);
8751         TG3_STAT_ADD32(&sp->tx_collisions, MAC_TX_STATS_COLLISIONS);
8752         TG3_STAT_ADD32(&sp->tx_xon_sent, MAC_TX_STATS_XON_SENT);
8753         TG3_STAT_ADD32(&sp->tx_xoff_sent, MAC_TX_STATS_XOFF_SENT);
8754         TG3_STAT_ADD32(&sp->tx_mac_errors, MAC_TX_STATS_MAC_ERRORS);
8755         TG3_STAT_ADD32(&sp->tx_single_collisions, MAC_TX_STATS_SINGLE_COLLISIONS);
8756         TG3_STAT_ADD32(&sp->tx_mult_collisions, MAC_TX_STATS_MULT_COLLISIONS);
8757         TG3_STAT_ADD32(&sp->tx_deferred, MAC_TX_STATS_DEFERRED);
8758         TG3_STAT_ADD32(&sp->tx_excessive_collisions, MAC_TX_STATS_EXCESSIVE_COL);
8759         TG3_STAT_ADD32(&sp->tx_late_collisions, MAC_TX_STATS_LATE_COL);
8760         TG3_STAT_ADD32(&sp->tx_ucast_packets, MAC_TX_STATS_UCAST);
8761         TG3_STAT_ADD32(&sp->tx_mcast_packets, MAC_TX_STATS_MCAST);
8762         TG3_STAT_ADD32(&sp->tx_bcast_packets, MAC_TX_STATS_BCAST);
8763
8764         TG3_STAT_ADD32(&sp->rx_octets, MAC_RX_STATS_OCTETS);
8765         TG3_STAT_ADD32(&sp->rx_fragments, MAC_RX_STATS_FRAGMENTS);
8766         TG3_STAT_ADD32(&sp->rx_ucast_packets, MAC_RX_STATS_UCAST);
8767         TG3_STAT_ADD32(&sp->rx_mcast_packets, MAC_RX_STATS_MCAST);
8768         TG3_STAT_ADD32(&sp->rx_bcast_packets, MAC_RX_STATS_BCAST);
8769         TG3_STAT_ADD32(&sp->rx_fcs_errors, MAC_RX_STATS_FCS_ERRORS);
8770         TG3_STAT_ADD32(&sp->rx_align_errors, MAC_RX_STATS_ALIGN_ERRORS);
8771         TG3_STAT_ADD32(&sp->rx_xon_pause_rcvd, MAC_RX_STATS_XON_PAUSE_RECVD);
8772         TG3_STAT_ADD32(&sp->rx_xoff_pause_rcvd, MAC_RX_STATS_XOFF_PAUSE_RECVD);
8773         TG3_STAT_ADD32(&sp->rx_mac_ctrl_rcvd, MAC_RX_STATS_MAC_CTRL_RECVD);
8774         TG3_STAT_ADD32(&sp->rx_xoff_entered, MAC_RX_STATS_XOFF_ENTERED);
8775         TG3_STAT_ADD32(&sp->rx_frame_too_long_errors, MAC_RX_STATS_FRAME_TOO_LONG);
8776         TG3_STAT_ADD32(&sp->rx_jabbers, MAC_RX_STATS_JABBERS);
8777         TG3_STAT_ADD32(&sp->rx_undersize_packets, MAC_RX_STATS_UNDERSIZE);
8778
8779         TG3_STAT_ADD32(&sp->rxbds_empty, RCVLPC_NO_RCV_BD_CNT);
8780         if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5717) {
8781                 TG3_STAT_ADD32(&sp->rx_discards, RCVLPC_IN_DISCARDS_CNT);
8782         } else {
8783                 u32 val = tr32(HOSTCC_FLOW_ATTN);
8784                 val = (val & HOSTCC_FLOW_ATTN_MBUF_LWM) ? 1 : 0;
8785                 if (val) {
8786                         tw32(HOSTCC_FLOW_ATTN, HOSTCC_FLOW_ATTN_MBUF_LWM);
8787                         sp->rx_discards.low += val;
8788                         if (sp->rx_discards.low < val)
8789                                 sp->rx_discards.high += 1;
8790                 }
8791                 sp->mbuf_lwm_thresh_hit = sp->rx_discards;
8792         }
8793         TG3_STAT_ADD32(&sp->rx_errors, RCVLPC_IN_ERRORS_CNT);
8794 }
8795
8796 static void tg3_timer(unsigned long __opaque)
8797 {
8798         struct tg3 *tp = (struct tg3 *) __opaque;
8799
8800         if (tp->irq_sync)
8801                 goto restart_timer;
8802
8803         spin_lock(&tp->lock);
8804
8805         if (!tg3_flag(tp, TAGGED_STATUS)) {
8806                 /* All of this garbage is because when using non-tagged
8807                  * IRQ status the mailbox/status_block protocol the chip
8808                  * uses with the cpu is race prone.
8809                  */
8810                 if (tp->napi[0].hw_status->status & SD_STATUS_UPDATED) {
8811                         tw32(GRC_LOCAL_CTRL,
8812                              tp->grc_local_ctrl | GRC_LCLCTRL_SETINT);
8813                 } else {
8814                         tw32(HOSTCC_MODE, tp->coalesce_mode |
8815                              HOSTCC_MODE_ENABLE | HOSTCC_MODE_NOW);
8816                 }
8817
8818                 if (!(tr32(WDMAC_MODE) & WDMAC_MODE_ENABLE)) {
8819                         tg3_flag_set(tp, RESTART_TIMER);
8820                         spin_unlock(&tp->lock);
8821                         schedule_work(&tp->reset_task);
8822                         return;
8823                 }
8824         }
8825
8826         /* This part only runs once per second. */
8827         if (!--tp->timer_counter) {
8828                 if (tg3_flag(tp, 5705_PLUS))
8829                         tg3_periodic_fetch_stats(tp);
8830
8831                 if (tp->setlpicnt && !--tp->setlpicnt) {
8832                         u32 val = tr32(TG3_CPMU_EEE_MODE);
8833                         tw32(TG3_CPMU_EEE_MODE,
8834                              val | TG3_CPMU_EEEMD_LPI_ENABLE);
8835                 }
8836
8837                 if (tg3_flag(tp, USE_LINKCHG_REG)) {
8838                         u32 mac_stat;
8839                         int phy_event;
8840
8841                         mac_stat = tr32(MAC_STATUS);
8842
8843                         phy_event = 0;
8844                         if (tp->phy_flags & TG3_PHYFLG_USE_MI_INTERRUPT) {
8845                                 if (mac_stat & MAC_STATUS_MI_INTERRUPT)
8846                                         phy_event = 1;
8847                         } else if (mac_stat & MAC_STATUS_LNKSTATE_CHANGED)
8848                                 phy_event = 1;
8849
8850                         if (phy_event)
8851                                 tg3_setup_phy(tp, 0);
8852                 } else if (tg3_flag(tp, POLL_SERDES)) {
8853                         u32 mac_stat = tr32(MAC_STATUS);
8854                         int need_setup = 0;
8855
8856                         if (netif_carrier_ok(tp->dev) &&
8857                             (mac_stat & MAC_STATUS_LNKSTATE_CHANGED)) {
8858                                 need_setup = 1;
8859                         }
8860                         if (!netif_carrier_ok(tp->dev) &&
8861                             (mac_stat & (MAC_STATUS_PCS_SYNCED |
8862                                          MAC_STATUS_SIGNAL_DET))) {
8863                                 need_setup = 1;
8864                         }
8865                         if (need_setup) {
8866                                 if (!tp->serdes_counter) {
8867                                         tw32_f(MAC_MODE,
8868                                              (tp->mac_mode &
8869                                               ~MAC_MODE_PORT_MODE_MASK));
8870                                         udelay(40);
8871                                         tw32_f(MAC_MODE, tp->mac_mode);
8872                                         udelay(40);
8873                                 }
8874                                 tg3_setup_phy(tp, 0);
8875                         }
8876                 } else if ((tp->phy_flags & TG3_PHYFLG_MII_SERDES) &&
8877                            tg3_flag(tp, 5780_CLASS)) {
8878                         tg3_serdes_parallel_detect(tp);
8879                 }
8880
8881                 tp->timer_counter = tp->timer_multiplier;
8882         }
8883
8884         /* Heartbeat is only sent once every 2 seconds.
8885          *
8886          * The heartbeat is to tell the ASF firmware that the host
8887          * driver is still alive.  In the event that the OS crashes,
8888          * ASF needs to reset the hardware to free up the FIFO space
8889          * that may be filled with rx packets destined for the host.
8890          * If the FIFO is full, ASF will no longer function properly.
8891          *
8892          * Unintended resets have been reported on real time kernels
8893          * where the timer doesn't run on time.  Netpoll will also have
8894          * same problem.
8895          *
8896          * The new FWCMD_NICDRV_ALIVE3 command tells the ASF firmware
8897          * to check the ring condition when the heartbeat is expiring
8898          * before doing the reset.  This will prevent most unintended
8899          * resets.
8900          */
8901         if (!--tp->asf_counter) {
8902                 if (tg3_flag(tp, ENABLE_ASF) && !tg3_flag(tp, ENABLE_APE)) {
8903                         tg3_wait_for_event_ack(tp);
8904
8905                         tg3_write_mem(tp, NIC_SRAM_FW_CMD_MBOX,
8906                                       FWCMD_NICDRV_ALIVE3);
8907                         tg3_write_mem(tp, NIC_SRAM_FW_CMD_LEN_MBOX, 4);
8908                         tg3_write_mem(tp, NIC_SRAM_FW_CMD_DATA_MBOX,
8909                                       TG3_FW_UPDATE_TIMEOUT_SEC);
8910
8911                         tg3_generate_fw_event(tp);
8912                 }
8913                 tp->asf_counter = tp->asf_multiplier;
8914         }
8915
8916         spin_unlock(&tp->lock);
8917
8918 restart_timer:
8919         tp->timer.expires = jiffies + tp->timer_offset;
8920         add_timer(&tp->timer);
8921 }
8922
8923 static int tg3_request_irq(struct tg3 *tp, int irq_num)
8924 {
8925         irq_handler_t fn;
8926         unsigned long flags;
8927         char *name;
8928         struct tg3_napi *tnapi = &tp->napi[irq_num];
8929
8930         if (tp->irq_cnt == 1)
8931                 name = tp->dev->name;
8932         else {
8933                 name = &tnapi->irq_lbl[0];
8934                 snprintf(name, IFNAMSIZ, "%s-%d", tp->dev->name, irq_num);
8935                 name[IFNAMSIZ-1] = 0;
8936         }
8937
8938         if (tg3_flag(tp, USING_MSI) || tg3_flag(tp, USING_MSIX)) {
8939                 fn = tg3_msi;
8940                 if (tg3_flag(tp, 1SHOT_MSI))
8941                         fn = tg3_msi_1shot;
8942                 flags = 0;
8943         } else {
8944                 fn = tg3_interrupt;
8945                 if (tg3_flag(tp, TAGGED_STATUS))
8946                         fn = tg3_interrupt_tagged;
8947                 flags = IRQF_SHARED;
8948         }
8949
8950         return request_irq(tnapi->irq_vec, fn, flags, name, tnapi);
8951 }
8952
8953 static int tg3_test_interrupt(struct tg3 *tp)
8954 {
8955         struct tg3_napi *tnapi = &tp->napi[0];
8956         struct net_device *dev = tp->dev;
8957         int err, i, intr_ok = 0;
8958         u32 val;
8959
8960         if (!netif_running(dev))
8961                 return -ENODEV;
8962
8963         tg3_disable_ints(tp);
8964
8965         free_irq(tnapi->irq_vec, tnapi);
8966
8967         /*
8968          * Turn off MSI one shot mode.  Otherwise this test has no
8969          * observable way to know whether the interrupt was delivered.
8970          */
8971         if (tg3_flag(tp, 57765_PLUS) && tg3_flag(tp, USING_MSI)) {
8972                 val = tr32(MSGINT_MODE) | MSGINT_MODE_ONE_SHOT_DISABLE;
8973                 tw32(MSGINT_MODE, val);
8974         }
8975
8976         err = request_irq(tnapi->irq_vec, tg3_test_isr,
8977                           IRQF_SHARED | IRQF_SAMPLE_RANDOM, dev->name, tnapi);
8978         if (err)
8979                 return err;
8980
8981         tnapi->hw_status->status &= ~SD_STATUS_UPDATED;
8982         tg3_enable_ints(tp);
8983
8984         tw32_f(HOSTCC_MODE, tp->coalesce_mode | HOSTCC_MODE_ENABLE |
8985                tnapi->coal_now);
8986
8987         for (i = 0; i < 5; i++) {
8988                 u32 int_mbox, misc_host_ctrl;
8989
8990                 int_mbox = tr32_mailbox(tnapi->int_mbox);
8991                 misc_host_ctrl = tr32(TG3PCI_MISC_HOST_CTRL);
8992
8993                 if ((int_mbox != 0) ||
8994                     (misc_host_ctrl & MISC_HOST_CTRL_MASK_PCI_INT)) {
8995                         intr_ok = 1;
8996                         break;
8997                 }
8998
8999                 msleep(10);
9000         }
9001
9002         tg3_disable_ints(tp);
9003
9004         free_irq(tnapi->irq_vec, tnapi);
9005
9006         err = tg3_request_irq(tp, 0);
9007
9008         if (err)
9009                 return err;
9010
9011         if (intr_ok) {
9012                 /* Reenable MSI one shot mode. */
9013                 if (tg3_flag(tp, 57765_PLUS) && tg3_flag(tp, USING_MSI)) {
9014                         val = tr32(MSGINT_MODE) & ~MSGINT_MODE_ONE_SHOT_DISABLE;
9015                         tw32(MSGINT_MODE, val);
9016                 }
9017                 return 0;
9018         }
9019
9020         return -EIO;
9021 }
9022
9023 /* Returns 0 if MSI test succeeds or MSI test fails and INTx mode is
9024  * successfully restored
9025  */
9026 static int tg3_test_msi(struct tg3 *tp)
9027 {
9028         int err;
9029         u16 pci_cmd;
9030
9031         if (!tg3_flag(tp, USING_MSI))
9032                 return 0;
9033
9034         /* Turn off SERR reporting in case MSI terminates with Master
9035          * Abort.
9036          */
9037         pci_read_config_word(tp->pdev, PCI_COMMAND, &pci_cmd);
9038         pci_write_config_word(tp->pdev, PCI_COMMAND,
9039                               pci_cmd & ~PCI_COMMAND_SERR);
9040
9041         err = tg3_test_interrupt(tp);
9042
9043         pci_write_config_word(tp->pdev, PCI_COMMAND, pci_cmd);
9044
9045         if (!err)
9046                 return 0;
9047
9048         /* other failures */
9049         if (err != -EIO)
9050                 return err;
9051
9052         /* MSI test failed, go back to INTx mode */
9053         netdev_warn(tp->dev, "No interrupt was generated using MSI. Switching "
9054                     "to INTx mode. Please report this failure to the PCI "
9055                     "maintainer and include system chipset information\n");
9056
9057         free_irq(tp->napi[0].irq_vec, &tp->napi[0]);
9058
9059         pci_disable_msi(tp->pdev);
9060
9061         tg3_flag_clear(tp, USING_MSI);
9062         tp->napi[0].irq_vec = tp->pdev->irq;
9063
9064         err = tg3_request_irq(tp, 0);
9065         if (err)
9066                 return err;
9067
9068         /* Need to reset the chip because the MSI cycle may have terminated
9069          * with Master Abort.
9070          */
9071         tg3_full_lock(tp, 1);
9072
9073         tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
9074         err = tg3_init_hw(tp, 1);
9075
9076         tg3_full_unlock(tp);
9077
9078         if (err)
9079                 free_irq(tp->napi[0].irq_vec, &tp->napi[0]);
9080
9081         return err;
9082 }
9083
9084 static int tg3_request_firmware(struct tg3 *tp)
9085 {
9086         const __be32 *fw_data;
9087
9088         if (request_firmware(&tp->fw, tp->fw_needed, &tp->pdev->dev)) {
9089                 netdev_err(tp->dev, "Failed to load firmware \"%s\"\n",
9090                            tp->fw_needed);
9091                 return -ENOENT;
9092         }
9093
9094         fw_data = (void *)tp->fw->data;
9095
9096         /* Firmware blob starts with version numbers, followed by
9097          * start address and _full_ length including BSS sections
9098          * (which must be longer than the actual data, of course
9099          */
9100
9101         tp->fw_len = be32_to_cpu(fw_data[2]);   /* includes bss */
9102         if (tp->fw_len < (tp->fw->size - 12)) {
9103                 netdev_err(tp->dev, "bogus length %d in \"%s\"\n",
9104                            tp->fw_len, tp->fw_needed);
9105                 release_firmware(tp->fw);
9106                 tp->fw = NULL;
9107                 return -EINVAL;
9108         }
9109
9110         /* We no longer need firmware; we have it. */
9111         tp->fw_needed = NULL;
9112         return 0;
9113 }
9114
9115 static bool tg3_enable_msix(struct tg3 *tp)
9116 {
9117         int i, rc, cpus = num_online_cpus();
9118         struct msix_entry msix_ent[tp->irq_max];
9119
9120         if (cpus == 1)
9121                 /* Just fallback to the simpler MSI mode. */
9122                 return false;
9123
9124         /*
9125          * We want as many rx rings enabled as there are cpus.
9126          * The first MSIX vector only deals with link interrupts, etc,
9127          * so we add one to the number of vectors we are requesting.
9128          */
9129         tp->irq_cnt = min_t(unsigned, cpus + 1, tp->irq_max);
9130
9131         for (i = 0; i < tp->irq_max; i++) {
9132                 msix_ent[i].entry  = i;
9133                 msix_ent[i].vector = 0;
9134         }
9135
9136         rc = pci_enable_msix(tp->pdev, msix_ent, tp->irq_cnt);
9137         if (rc < 0) {
9138                 return false;
9139         } else if (rc != 0) {
9140                 if (pci_enable_msix(tp->pdev, msix_ent, rc))
9141                         return false;
9142                 netdev_notice(tp->dev, "Requested %d MSI-X vectors, received %d\n",
9143                               tp->irq_cnt, rc);
9144                 tp->irq_cnt = rc;
9145         }
9146
9147         for (i = 0; i < tp->irq_max; i++)
9148                 tp->napi[i].irq_vec = msix_ent[i].vector;
9149
9150         netif_set_real_num_tx_queues(tp->dev, 1);
9151         rc = tp->irq_cnt > 1 ? tp->irq_cnt - 1 : 1;
9152         if (netif_set_real_num_rx_queues(tp->dev, rc)) {
9153                 pci_disable_msix(tp->pdev);
9154                 return false;
9155         }
9156
9157         if (tp->irq_cnt > 1) {
9158                 tg3_flag_set(tp, ENABLE_RSS);
9159
9160                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5719 ||
9161                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5720) {
9162                         tg3_flag_set(tp, ENABLE_TSS);
9163                         netif_set_real_num_tx_queues(tp->dev, tp->irq_cnt - 1);
9164                 }
9165         }
9166
9167         return true;
9168 }
9169
9170 static void tg3_ints_init(struct tg3 *tp)
9171 {
9172         if ((tg3_flag(tp, SUPPORT_MSI) || tg3_flag(tp, SUPPORT_MSIX)) &&
9173             !tg3_flag(tp, TAGGED_STATUS)) {
9174                 /* All MSI supporting chips should support tagged
9175                  * status.  Assert that this is the case.
9176                  */
9177                 netdev_warn(tp->dev,
9178                             "MSI without TAGGED_STATUS? Not using MSI\n");
9179                 goto defcfg;
9180         }
9181
9182         if (tg3_flag(tp, SUPPORT_MSIX) && tg3_enable_msix(tp))
9183                 tg3_flag_set(tp, USING_MSIX);
9184         else if (tg3_flag(tp, SUPPORT_MSI) && pci_enable_msi(tp->pdev) == 0)
9185                 tg3_flag_set(tp, USING_MSI);
9186
9187         if (tg3_flag(tp, USING_MSI) || tg3_flag(tp, USING_MSIX)) {
9188                 u32 msi_mode = tr32(MSGINT_MODE);
9189                 if (tg3_flag(tp, USING_MSIX) && tp->irq_cnt > 1)
9190                         msi_mode |= MSGINT_MODE_MULTIVEC_EN;
9191                 tw32(MSGINT_MODE, msi_mode | MSGINT_MODE_ENABLE);
9192         }
9193 defcfg:
9194         if (!tg3_flag(tp, USING_MSIX)) {
9195                 tp->irq_cnt = 1;
9196                 tp->napi[0].irq_vec = tp->pdev->irq;
9197                 netif_set_real_num_tx_queues(tp->dev, 1);
9198                 netif_set_real_num_rx_queues(tp->dev, 1);
9199         }
9200 }
9201
9202 static void tg3_ints_fini(struct tg3 *tp)
9203 {
9204         if (tg3_flag(tp, USING_MSIX))
9205                 pci_disable_msix(tp->pdev);
9206         else if (tg3_flag(tp, USING_MSI))
9207                 pci_disable_msi(tp->pdev);
9208         tg3_flag_clear(tp, USING_MSI);
9209         tg3_flag_clear(tp, USING_MSIX);
9210         tg3_flag_clear(tp, ENABLE_RSS);
9211         tg3_flag_clear(tp, ENABLE_TSS);
9212 }
9213
9214 static int tg3_open(struct net_device *dev)
9215 {
9216         struct tg3 *tp = netdev_priv(dev);
9217         int i, err;
9218
9219         if (tp->fw_needed) {
9220                 err = tg3_request_firmware(tp);
9221                 if (tp->pci_chip_rev_id == CHIPREV_ID_5701_A0) {
9222                         if (err)
9223                                 return err;
9224                 } else if (err) {
9225                         netdev_warn(tp->dev, "TSO capability disabled\n");
9226                         tg3_flag_clear(tp, TSO_CAPABLE);
9227                 } else if (!tg3_flag(tp, TSO_CAPABLE)) {
9228                         netdev_notice(tp->dev, "TSO capability restored\n");
9229                         tg3_flag_set(tp, TSO_CAPABLE);
9230                 }
9231         }
9232
9233         netif_carrier_off(tp->dev);
9234
9235         err = tg3_power_up(tp);
9236         if (err)
9237                 return err;
9238
9239         tg3_full_lock(tp, 0);
9240
9241         tg3_disable_ints(tp);
9242         tg3_flag_clear(tp, INIT_COMPLETE);
9243
9244         tg3_full_unlock(tp);
9245
9246         /*
9247          * Setup interrupts first so we know how
9248          * many NAPI resources to allocate
9249          */
9250         tg3_ints_init(tp);
9251
9252         /* The placement of this call is tied
9253          * to the setup and use of Host TX descriptors.
9254          */
9255         err = tg3_alloc_consistent(tp);
9256         if (err)
9257                 goto err_out1;
9258
9259         tg3_napi_init(tp);
9260
9261         tg3_napi_enable(tp);
9262
9263         for (i = 0; i < tp->irq_cnt; i++) {
9264                 struct tg3_napi *tnapi = &tp->napi[i];
9265                 err = tg3_request_irq(tp, i);
9266                 if (err) {
9267                         for (i--; i >= 0; i--)
9268                                 free_irq(tnapi->irq_vec, tnapi);
9269                         break;
9270                 }
9271         }
9272
9273         if (err)
9274                 goto err_out2;
9275
9276         tg3_full_lock(tp, 0);
9277
9278         err = tg3_init_hw(tp, 1);
9279         if (err) {
9280                 tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
9281                 tg3_free_rings(tp);
9282         } else {
9283                 if (tg3_flag(tp, TAGGED_STATUS))
9284                         tp->timer_offset = HZ;
9285                 else
9286                         tp->timer_offset = HZ / 10;
9287
9288                 BUG_ON(tp->timer_offset > HZ);
9289                 tp->timer_counter = tp->timer_multiplier =
9290                         (HZ / tp->timer_offset);
9291                 tp->asf_counter = tp->asf_multiplier =
9292                         ((HZ / tp->timer_offset) * 2);
9293
9294                 init_timer(&tp->timer);
9295                 tp->timer.expires = jiffies + tp->timer_offset;
9296                 tp->timer.data = (unsigned long) tp;
9297                 tp->timer.function = tg3_timer;
9298         }
9299
9300         tg3_full_unlock(tp);
9301
9302         if (err)
9303                 goto err_out3;
9304
9305         if (tg3_flag(tp, USING_MSI)) {
9306                 err = tg3_test_msi(tp);
9307
9308                 if (err) {
9309                         tg3_full_lock(tp, 0);
9310                         tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
9311                         tg3_free_rings(tp);
9312                         tg3_full_unlock(tp);
9313
9314                         goto err_out2;
9315                 }
9316
9317                 if (!tg3_flag(tp, 57765_PLUS) && tg3_flag(tp, USING_MSI)) {
9318                         u32 val = tr32(PCIE_TRANSACTION_CFG);
9319
9320                         tw32(PCIE_TRANSACTION_CFG,
9321                              val | PCIE_TRANS_CFG_1SHOT_MSI);
9322                 }
9323         }
9324
9325         tg3_phy_start(tp);
9326
9327         tg3_full_lock(tp, 0);
9328
9329         add_timer(&tp->timer);
9330         tg3_flag_set(tp, INIT_COMPLETE);
9331         tg3_enable_ints(tp);
9332
9333         tg3_full_unlock(tp);
9334
9335         netif_tx_start_all_queues(dev);
9336
9337         /*
9338          * Reset loopback feature if it was turned on while the device was down
9339          * make sure that it's installed properly now.
9340          */
9341         if (dev->features & NETIF_F_LOOPBACK)
9342                 tg3_set_loopback(dev, dev->features);
9343
9344         return 0;
9345
9346 err_out3:
9347         for (i = tp->irq_cnt - 1; i >= 0; i--) {
9348                 struct tg3_napi *tnapi = &tp->napi[i];
9349                 free_irq(tnapi->irq_vec, tnapi);
9350         }
9351
9352 err_out2:
9353         tg3_napi_disable(tp);
9354         tg3_napi_fini(tp);
9355         tg3_free_consistent(tp);
9356
9357 err_out1:
9358         tg3_ints_fini(tp);
9359         return err;
9360 }
9361
9362 static struct rtnl_link_stats64 *tg3_get_stats64(struct net_device *,
9363                                                  struct rtnl_link_stats64 *);
9364 static struct tg3_ethtool_stats *tg3_get_estats(struct tg3 *);
9365
9366 static int tg3_close(struct net_device *dev)
9367 {
9368         int i;
9369         struct tg3 *tp = netdev_priv(dev);
9370
9371         tg3_napi_disable(tp);
9372         cancel_work_sync(&tp->reset_task);
9373
9374         netif_tx_stop_all_queues(dev);
9375
9376         del_timer_sync(&tp->timer);
9377
9378         tg3_phy_stop(tp);
9379
9380         tg3_full_lock(tp, 1);
9381
9382         tg3_disable_ints(tp);
9383
9384         tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
9385         tg3_free_rings(tp);
9386         tg3_flag_clear(tp, INIT_COMPLETE);
9387
9388         tg3_full_unlock(tp);
9389
9390         for (i = tp->irq_cnt - 1; i >= 0; i--) {
9391                 struct tg3_napi *tnapi = &tp->napi[i];
9392                 free_irq(tnapi->irq_vec, tnapi);
9393         }
9394
9395         tg3_ints_fini(tp);
9396
9397         tg3_get_stats64(tp->dev, &tp->net_stats_prev);
9398
9399         memcpy(&tp->estats_prev, tg3_get_estats(tp),
9400                sizeof(tp->estats_prev));
9401
9402         tg3_napi_fini(tp);
9403
9404         tg3_free_consistent(tp);
9405
9406         tg3_power_down(tp);
9407
9408         netif_carrier_off(tp->dev);
9409
9410         return 0;
9411 }
9412
9413 static inline u64 get_stat64(tg3_stat64_t *val)
9414 {
9415        return ((u64)val->high << 32) | ((u64)val->low);
9416 }
9417
9418 static u64 calc_crc_errors(struct tg3 *tp)
9419 {
9420         struct tg3_hw_stats *hw_stats = tp->hw_stats;
9421
9422         if (!(tp->phy_flags & TG3_PHYFLG_PHY_SERDES) &&
9423             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
9424              GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701)) {
9425                 u32 val;
9426
9427                 spin_lock_bh(&tp->lock);
9428                 if (!tg3_readphy(tp, MII_TG3_TEST1, &val)) {
9429                         tg3_writephy(tp, MII_TG3_TEST1,
9430                                      val | MII_TG3_TEST1_CRC_EN);
9431                         tg3_readphy(tp, MII_TG3_RXR_COUNTERS, &val);
9432                 } else
9433                         val = 0;
9434                 spin_unlock_bh(&tp->lock);
9435
9436                 tp->phy_crc_errors += val;
9437
9438                 return tp->phy_crc_errors;
9439         }
9440
9441         return get_stat64(&hw_stats->rx_fcs_errors);
9442 }
9443
9444 #define ESTAT_ADD(member) \
9445         estats->member =        old_estats->member + \
9446                                 get_stat64(&hw_stats->member)
9447
9448 static struct tg3_ethtool_stats *tg3_get_estats(struct tg3 *tp)
9449 {
9450         struct tg3_ethtool_stats *estats = &tp->estats;
9451         struct tg3_ethtool_stats *old_estats = &tp->estats_prev;
9452         struct tg3_hw_stats *hw_stats = tp->hw_stats;
9453
9454         if (!hw_stats)
9455                 return old_estats;
9456
9457         ESTAT_ADD(rx_octets);
9458         ESTAT_ADD(rx_fragments);
9459         ESTAT_ADD(rx_ucast_packets);
9460         ESTAT_ADD(rx_mcast_packets);
9461         ESTAT_ADD(rx_bcast_packets);
9462         ESTAT_ADD(rx_fcs_errors);
9463         ESTAT_ADD(rx_align_errors);
9464         ESTAT_ADD(rx_xon_pause_rcvd);
9465         ESTAT_ADD(rx_xoff_pause_rcvd);
9466         ESTAT_ADD(rx_mac_ctrl_rcvd);
9467         ESTAT_ADD(rx_xoff_entered);
9468         ESTAT_ADD(rx_frame_too_long_errors);
9469         ESTAT_ADD(rx_jabbers);
9470         ESTAT_ADD(rx_undersize_packets);
9471         ESTAT_ADD(rx_in_length_errors);
9472         ESTAT_ADD(rx_out_length_errors);
9473         ESTAT_ADD(rx_64_or_less_octet_packets);
9474         ESTAT_ADD(rx_65_to_127_octet_packets);
9475         ESTAT_ADD(rx_128_to_255_octet_packets);
9476         ESTAT_ADD(rx_256_to_511_octet_packets);
9477         ESTAT_ADD(rx_512_to_1023_octet_packets);
9478         ESTAT_ADD(rx_1024_to_1522_octet_packets);
9479         ESTAT_ADD(rx_1523_to_2047_octet_packets);
9480         ESTAT_ADD(rx_2048_to_4095_octet_packets);
9481         ESTAT_ADD(rx_4096_to_8191_octet_packets);
9482         ESTAT_ADD(rx_8192_to_9022_octet_packets);
9483
9484         ESTAT_ADD(tx_octets);
9485         ESTAT_ADD(tx_collisions);
9486         ESTAT_ADD(tx_xon_sent);
9487         ESTAT_ADD(tx_xoff_sent);
9488         ESTAT_ADD(tx_flow_control);
9489         ESTAT_ADD(tx_mac_errors);
9490         ESTAT_ADD(tx_single_collisions);
9491         ESTAT_ADD(tx_mult_collisions);
9492         ESTAT_ADD(tx_deferred);
9493         ESTAT_ADD(tx_excessive_collisions);
9494         ESTAT_ADD(tx_late_collisions);
9495         ESTAT_ADD(tx_collide_2times);
9496         ESTAT_ADD(tx_collide_3times);
9497         ESTAT_ADD(tx_collide_4times);
9498         ESTAT_ADD(tx_collide_5times);
9499         ESTAT_ADD(tx_collide_6times);
9500         ESTAT_ADD(tx_collide_7times);
9501         ESTAT_ADD(tx_collide_8times);
9502         ESTAT_ADD(tx_collide_9times);
9503         ESTAT_ADD(tx_collide_10times);
9504         ESTAT_ADD(tx_collide_11times);
9505         ESTAT_ADD(tx_collide_12times);
9506         ESTAT_ADD(tx_collide_13times);
9507         ESTAT_ADD(tx_collide_14times);
9508         ESTAT_ADD(tx_collide_15times);
9509         ESTAT_ADD(tx_ucast_packets);
9510         ESTAT_ADD(tx_mcast_packets);
9511         ESTAT_ADD(tx_bcast_packets);
9512         ESTAT_ADD(tx_carrier_sense_errors);
9513         ESTAT_ADD(tx_discards);
9514         ESTAT_ADD(tx_errors);
9515
9516         ESTAT_ADD(dma_writeq_full);
9517         ESTAT_ADD(dma_write_prioq_full);
9518         ESTAT_ADD(rxbds_empty);
9519         ESTAT_ADD(rx_discards);
9520         ESTAT_ADD(rx_errors);
9521         ESTAT_ADD(rx_threshold_hit);
9522
9523         ESTAT_ADD(dma_readq_full);
9524         ESTAT_ADD(dma_read_prioq_full);
9525         ESTAT_ADD(tx_comp_queue_full);
9526
9527         ESTAT_ADD(ring_set_send_prod_index);
9528         ESTAT_ADD(ring_status_update);
9529         ESTAT_ADD(nic_irqs);
9530         ESTAT_ADD(nic_avoided_irqs);
9531         ESTAT_ADD(nic_tx_threshold_hit);
9532
9533         return estats;
9534 }
9535
9536 static struct rtnl_link_stats64 *tg3_get_stats64(struct net_device *dev,
9537                                                  struct rtnl_link_stats64 *stats)
9538 {
9539         struct tg3 *tp = netdev_priv(dev);
9540         struct rtnl_link_stats64 *old_stats = &tp->net_stats_prev;
9541         struct tg3_hw_stats *hw_stats = tp->hw_stats;
9542
9543         if (!hw_stats)
9544                 return old_stats;
9545
9546         stats->rx_packets = old_stats->rx_packets +
9547                 get_stat64(&hw_stats->rx_ucast_packets) +
9548                 get_stat64(&hw_stats->rx_mcast_packets) +
9549                 get_stat64(&hw_stats->rx_bcast_packets);
9550
9551         stats->tx_packets = old_stats->tx_packets +
9552                 get_stat64(&hw_stats->tx_ucast_packets) +
9553                 get_stat64(&hw_stats->tx_mcast_packets) +
9554                 get_stat64(&hw_stats->tx_bcast_packets);
9555
9556         stats->rx_bytes = old_stats->rx_bytes +
9557                 get_stat64(&hw_stats->rx_octets);
9558         stats->tx_bytes = old_stats->tx_bytes +
9559                 get_stat64(&hw_stats->tx_octets);
9560
9561         stats->rx_errors = old_stats->rx_errors +
9562                 get_stat64(&hw_stats->rx_errors);
9563         stats->tx_errors = old_stats->tx_errors +
9564                 get_stat64(&hw_stats->tx_errors) +
9565                 get_stat64(&hw_stats->tx_mac_errors) +
9566                 get_stat64(&hw_stats->tx_carrier_sense_errors) +
9567                 get_stat64(&hw_stats->tx_discards);
9568
9569         stats->multicast = old_stats->multicast +
9570                 get_stat64(&hw_stats->rx_mcast_packets);
9571         stats->collisions = old_stats->collisions +
9572                 get_stat64(&hw_stats->tx_collisions);
9573
9574         stats->rx_length_errors = old_stats->rx_length_errors +
9575                 get_stat64(&hw_stats->rx_frame_too_long_errors) +
9576                 get_stat64(&hw_stats->rx_undersize_packets);
9577
9578         stats->rx_over_errors = old_stats->rx_over_errors +
9579                 get_stat64(&hw_stats->rxbds_empty);
9580         stats->rx_frame_errors = old_stats->rx_frame_errors +
9581                 get_stat64(&hw_stats->rx_align_errors);
9582         stats->tx_aborted_errors = old_stats->tx_aborted_errors +
9583                 get_stat64(&hw_stats->tx_discards);
9584         stats->tx_carrier_errors = old_stats->tx_carrier_errors +
9585                 get_stat64(&hw_stats->tx_carrier_sense_errors);
9586
9587         stats->rx_crc_errors = old_stats->rx_crc_errors +
9588                 calc_crc_errors(tp);
9589
9590         stats->rx_missed_errors = old_stats->rx_missed_errors +
9591                 get_stat64(&hw_stats->rx_discards);
9592
9593         stats->rx_dropped = tp->rx_dropped;
9594
9595         return stats;
9596 }
9597
9598 static inline u32 calc_crc(unsigned char *buf, int len)
9599 {
9600         u32 reg;
9601         u32 tmp;
9602         int j, k;
9603
9604         reg = 0xffffffff;
9605
9606         for (j = 0; j < len; j++) {
9607                 reg ^= buf[j];
9608
9609                 for (k = 0; k < 8; k++) {
9610                         tmp = reg & 0x01;
9611
9612                         reg >>= 1;
9613
9614                         if (tmp)
9615                                 reg ^= 0xedb88320;
9616                 }
9617         }
9618
9619         return ~reg;
9620 }
9621
9622 static void tg3_set_multi(struct tg3 *tp, unsigned int accept_all)
9623 {
9624         /* accept or reject all multicast frames */
9625         tw32(MAC_HASH_REG_0, accept_all ? 0xffffffff : 0);
9626         tw32(MAC_HASH_REG_1, accept_all ? 0xffffffff : 0);
9627         tw32(MAC_HASH_REG_2, accept_all ? 0xffffffff : 0);
9628         tw32(MAC_HASH_REG_3, accept_all ? 0xffffffff : 0);
9629 }
9630
9631 static void __tg3_set_rx_mode(struct net_device *dev)
9632 {
9633         struct tg3 *tp = netdev_priv(dev);
9634         u32 rx_mode;
9635
9636         rx_mode = tp->rx_mode & ~(RX_MODE_PROMISC |
9637                                   RX_MODE_KEEP_VLAN_TAG);
9638
9639 #if !defined(CONFIG_VLAN_8021Q) && !defined(CONFIG_VLAN_8021Q_MODULE)
9640         /* When ASF is in use, we always keep the RX_MODE_KEEP_VLAN_TAG
9641          * flag clear.
9642          */
9643         if (!tg3_flag(tp, ENABLE_ASF))
9644                 rx_mode |= RX_MODE_KEEP_VLAN_TAG;
9645 #endif
9646
9647         if (dev->flags & IFF_PROMISC) {
9648                 /* Promiscuous mode. */
9649                 rx_mode |= RX_MODE_PROMISC;
9650         } else if (dev->flags & IFF_ALLMULTI) {
9651                 /* Accept all multicast. */
9652                 tg3_set_multi(tp, 1);
9653         } else if (netdev_mc_empty(dev)) {
9654                 /* Reject all multicast. */
9655                 tg3_set_multi(tp, 0);
9656         } else {
9657                 /* Accept one or more multicast(s). */
9658                 struct netdev_hw_addr *ha;
9659                 u32 mc_filter[4] = { 0, };
9660                 u32 regidx;
9661                 u32 bit;
9662                 u32 crc;
9663
9664                 netdev_for_each_mc_addr(ha, dev) {
9665                         crc = calc_crc(ha->addr, ETH_ALEN);
9666                         bit = ~crc & 0x7f;
9667                         regidx = (bit & 0x60) >> 5;
9668                         bit &= 0x1f;
9669                         mc_filter[regidx] |= (1 << bit);
9670                 }
9671
9672                 tw32(MAC_HASH_REG_0, mc_filter[0]);
9673                 tw32(MAC_HASH_REG_1, mc_filter[1]);
9674                 tw32(MAC_HASH_REG_2, mc_filter[2]);
9675                 tw32(MAC_HASH_REG_3, mc_filter[3]);
9676         }
9677
9678         if (rx_mode != tp->rx_mode) {
9679                 tp->rx_mode = rx_mode;
9680                 tw32_f(MAC_RX_MODE, rx_mode);
9681                 udelay(10);
9682         }
9683 }
9684
9685 static void tg3_set_rx_mode(struct net_device *dev)
9686 {
9687         struct tg3 *tp = netdev_priv(dev);
9688
9689         if (!netif_running(dev))
9690                 return;
9691
9692         tg3_full_lock(tp, 0);
9693         __tg3_set_rx_mode(dev);
9694         tg3_full_unlock(tp);
9695 }
9696
9697 static int tg3_get_regs_len(struct net_device *dev)
9698 {
9699         return TG3_REG_BLK_SIZE;
9700 }
9701
9702 static void tg3_get_regs(struct net_device *dev,
9703                 struct ethtool_regs *regs, void *_p)
9704 {
9705         struct tg3 *tp = netdev_priv(dev);
9706
9707         regs->version = 0;
9708
9709         memset(_p, 0, TG3_REG_BLK_SIZE);
9710
9711         if (tp->phy_flags & TG3_PHYFLG_IS_LOW_POWER)
9712                 return;
9713
9714         tg3_full_lock(tp, 0);
9715
9716         tg3_dump_legacy_regs(tp, (u32 *)_p);
9717
9718         tg3_full_unlock(tp);
9719 }
9720
9721 static int tg3_get_eeprom_len(struct net_device *dev)
9722 {
9723         struct tg3 *tp = netdev_priv(dev);
9724
9725         return tp->nvram_size;
9726 }
9727
9728 static int tg3_get_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom, u8 *data)
9729 {
9730         struct tg3 *tp = netdev_priv(dev);
9731         int ret;
9732         u8  *pd;
9733         u32 i, offset, len, b_offset, b_count;
9734         __be32 val;
9735
9736         if (tg3_flag(tp, NO_NVRAM))
9737                 return -EINVAL;
9738
9739         if (tp->phy_flags & TG3_PHYFLG_IS_LOW_POWER)
9740                 return -EAGAIN;
9741
9742         offset = eeprom->offset;
9743         len = eeprom->len;
9744         eeprom->len = 0;
9745
9746         eeprom->magic = TG3_EEPROM_MAGIC;
9747
9748         if (offset & 3) {
9749                 /* adjustments to start on required 4 byte boundary */
9750                 b_offset = offset & 3;
9751                 b_count = 4 - b_offset;
9752                 if (b_count > len) {
9753                         /* i.e. offset=1 len=2 */
9754                         b_count = len;
9755                 }
9756                 ret = tg3_nvram_read_be32(tp, offset-b_offset, &val);
9757                 if (ret)
9758                         return ret;
9759                 memcpy(data, ((char *)&val) + b_offset, b_count);
9760                 len -= b_count;
9761                 offset += b_count;
9762                 eeprom->len += b_count;
9763         }
9764
9765         /* read bytes up to the last 4 byte boundary */
9766         pd = &data[eeprom->len];
9767         for (i = 0; i < (len - (len & 3)); i += 4) {
9768                 ret = tg3_nvram_read_be32(tp, offset + i, &val);
9769                 if (ret) {
9770                         eeprom->len += i;
9771                         return ret;
9772                 }
9773                 memcpy(pd + i, &val, 4);
9774         }
9775         eeprom->len += i;
9776
9777         if (len & 3) {
9778                 /* read last bytes not ending on 4 byte boundary */
9779                 pd = &data[eeprom->len];
9780                 b_count = len & 3;
9781                 b_offset = offset + len - b_count;
9782                 ret = tg3_nvram_read_be32(tp, b_offset, &val);
9783                 if (ret)
9784                         return ret;
9785                 memcpy(pd, &val, b_count);
9786                 eeprom->len += b_count;
9787         }
9788         return 0;
9789 }
9790
9791 static int tg3_nvram_write_block(struct tg3 *tp, u32 offset, u32 len, u8 *buf);
9792
9793 static int tg3_set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom, u8 *data)
9794 {
9795         struct tg3 *tp = netdev_priv(dev);
9796         int ret;
9797         u32 offset, len, b_offset, odd_len;
9798         u8 *buf;
9799         __be32 start, end;
9800
9801         if (tp->phy_flags & TG3_PHYFLG_IS_LOW_POWER)
9802                 return -EAGAIN;
9803
9804         if (tg3_flag(tp, NO_NVRAM) ||
9805             eeprom->magic != TG3_EEPROM_MAGIC)
9806                 return -EINVAL;
9807
9808         offset = eeprom->offset;
9809         len = eeprom->len;
9810
9811         if ((b_offset = (offset & 3))) {
9812                 /* adjustments to start on required 4 byte boundary */
9813                 ret = tg3_nvram_read_be32(tp, offset-b_offset, &start);
9814                 if (ret)
9815                         return ret;
9816                 len += b_offset;
9817                 offset &= ~3;
9818                 if (len < 4)
9819                         len = 4;
9820         }
9821
9822         odd_len = 0;
9823         if (len & 3) {
9824                 /* adjustments to end on required 4 byte boundary */
9825                 odd_len = 1;
9826                 len = (len + 3) & ~3;
9827                 ret = tg3_nvram_read_be32(tp, offset+len-4, &end);
9828                 if (ret)
9829                         return ret;
9830         }
9831
9832         buf = data;
9833         if (b_offset || odd_len) {
9834                 buf = kmalloc(len, GFP_KERNEL);
9835                 if (!buf)
9836                         return -ENOMEM;
9837                 if (b_offset)
9838                         memcpy(buf, &start, 4);
9839                 if (odd_len)
9840                         memcpy(buf+len-4, &end, 4);
9841                 memcpy(buf + b_offset, data, eeprom->len);
9842         }
9843
9844         ret = tg3_nvram_write_block(tp, offset, len, buf);
9845
9846         if (buf != data)
9847                 kfree(buf);
9848
9849         return ret;
9850 }
9851
9852 static int tg3_get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
9853 {
9854         struct tg3 *tp = netdev_priv(dev);
9855
9856         if (tg3_flag(tp, USE_PHYLIB)) {
9857                 struct phy_device *phydev;
9858                 if (!(tp->phy_flags & TG3_PHYFLG_IS_CONNECTED))
9859                         return -EAGAIN;
9860                 phydev = tp->mdio_bus->phy_map[TG3_PHY_MII_ADDR];
9861                 return phy_ethtool_gset(phydev, cmd);
9862         }
9863
9864         cmd->supported = (SUPPORTED_Autoneg);
9865
9866         if (!(tp->phy_flags & TG3_PHYFLG_10_100_ONLY))
9867                 cmd->supported |= (SUPPORTED_1000baseT_Half |
9868                                    SUPPORTED_1000baseT_Full);
9869
9870         if (!(tp->phy_flags & TG3_PHYFLG_ANY_SERDES)) {
9871                 cmd->supported |= (SUPPORTED_100baseT_Half |
9872                                   SUPPORTED_100baseT_Full |
9873                                   SUPPORTED_10baseT_Half |
9874                                   SUPPORTED_10baseT_Full |
9875                                   SUPPORTED_TP);
9876                 cmd->port = PORT_TP;
9877         } else {
9878                 cmd->supported |= SUPPORTED_FIBRE;
9879                 cmd->port = PORT_FIBRE;
9880         }
9881
9882         cmd->advertising = tp->link_config.advertising;
9883         if (netif_running(dev)) {
9884                 ethtool_cmd_speed_set(cmd, tp->link_config.active_speed);
9885                 cmd->duplex = tp->link_config.active_duplex;
9886         } else {
9887                 ethtool_cmd_speed_set(cmd, SPEED_INVALID);
9888                 cmd->duplex = DUPLEX_INVALID;
9889         }
9890         cmd->phy_address = tp->phy_addr;
9891         cmd->transceiver = XCVR_INTERNAL;
9892         cmd->autoneg = tp->link_config.autoneg;
9893         cmd->maxtxpkt = 0;
9894         cmd->maxrxpkt = 0;
9895         return 0;
9896 }
9897
9898 static int tg3_set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
9899 {
9900         struct tg3 *tp = netdev_priv(dev);
9901         u32 speed = ethtool_cmd_speed(cmd);
9902
9903         if (tg3_flag(tp, USE_PHYLIB)) {
9904                 struct phy_device *phydev;
9905                 if (!(tp->phy_flags & TG3_PHYFLG_IS_CONNECTED))
9906                         return -EAGAIN;
9907                 phydev = tp->mdio_bus->phy_map[TG3_PHY_MII_ADDR];
9908                 return phy_ethtool_sset(phydev, cmd);
9909         }
9910
9911         if (cmd->autoneg != AUTONEG_ENABLE &&
9912             cmd->autoneg != AUTONEG_DISABLE)
9913                 return -EINVAL;
9914
9915         if (cmd->autoneg == AUTONEG_DISABLE &&
9916             cmd->duplex != DUPLEX_FULL &&
9917             cmd->duplex != DUPLEX_HALF)
9918                 return -EINVAL;
9919
9920         if (cmd->autoneg == AUTONEG_ENABLE) {
9921                 u32 mask = ADVERTISED_Autoneg |
9922                            ADVERTISED_Pause |
9923                            ADVERTISED_Asym_Pause;
9924
9925                 if (!(tp->phy_flags & TG3_PHYFLG_10_100_ONLY))
9926                         mask |= ADVERTISED_1000baseT_Half |
9927                                 ADVERTISED_1000baseT_Full;
9928
9929                 if (!(tp->phy_flags & TG3_PHYFLG_ANY_SERDES))
9930                         mask |= ADVERTISED_100baseT_Half |
9931                                 ADVERTISED_100baseT_Full |
9932                                 ADVERTISED_10baseT_Half |
9933                                 ADVERTISED_10baseT_Full |
9934                                 ADVERTISED_TP;
9935                 else
9936                         mask |= ADVERTISED_FIBRE;
9937
9938                 if (cmd->advertising & ~mask)
9939                         return -EINVAL;
9940
9941                 mask &= (ADVERTISED_1000baseT_Half |
9942                          ADVERTISED_1000baseT_Full |
9943                          ADVERTISED_100baseT_Half |
9944                          ADVERTISED_100baseT_Full |
9945                          ADVERTISED_10baseT_Half |
9946                          ADVERTISED_10baseT_Full);
9947
9948                 cmd->advertising &= mask;
9949         } else {
9950                 if (tp->phy_flags & TG3_PHYFLG_ANY_SERDES) {
9951                         if (speed != SPEED_1000)
9952                                 return -EINVAL;
9953
9954                         if (cmd->duplex != DUPLEX_FULL)
9955                                 return -EINVAL;
9956                 } else {
9957                         if (speed != SPEED_100 &&
9958                             speed != SPEED_10)
9959                                 return -EINVAL;
9960                 }
9961         }
9962
9963         tg3_full_lock(tp, 0);
9964
9965         tp->link_config.autoneg = cmd->autoneg;
9966         if (cmd->autoneg == AUTONEG_ENABLE) {
9967                 tp->link_config.advertising = (cmd->advertising |
9968                                               ADVERTISED_Autoneg);
9969                 tp->link_config.speed = SPEED_INVALID;
9970                 tp->link_config.duplex = DUPLEX_INVALID;
9971         } else {
9972                 tp->link_config.advertising = 0;
9973                 tp->link_config.speed = speed;
9974                 tp->link_config.duplex = cmd->duplex;
9975         }
9976
9977         tp->link_config.orig_speed = tp->link_config.speed;
9978         tp->link_config.orig_duplex = tp->link_config.duplex;
9979         tp->link_config.orig_autoneg = tp->link_config.autoneg;
9980
9981         if (netif_running(dev))
9982                 tg3_setup_phy(tp, 1);
9983
9984         tg3_full_unlock(tp);
9985
9986         return 0;
9987 }
9988
9989 static void tg3_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
9990 {
9991         struct tg3 *tp = netdev_priv(dev);
9992
9993         strcpy(info->driver, DRV_MODULE_NAME);
9994         strcpy(info->version, DRV_MODULE_VERSION);
9995         strcpy(info->fw_version, tp->fw_ver);
9996         strcpy(info->bus_info, pci_name(tp->pdev));
9997 }
9998
9999 static void tg3_get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
10000 {
10001         struct tg3 *tp = netdev_priv(dev);
10002
10003         if (tg3_flag(tp, WOL_CAP) && device_can_wakeup(&tp->pdev->dev))
10004                 wol->supported = WAKE_MAGIC;
10005         else
10006                 wol->supported = 0;
10007         wol->wolopts = 0;
10008         if (tg3_flag(tp, WOL_ENABLE) && device_can_wakeup(&tp->pdev->dev))
10009                 wol->wolopts = WAKE_MAGIC;
10010         memset(&wol->sopass, 0, sizeof(wol->sopass));
10011 }
10012
10013 static int tg3_set_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
10014 {
10015         struct tg3 *tp = netdev_priv(dev);
10016         struct device *dp = &tp->pdev->dev;
10017
10018         if (wol->wolopts & ~WAKE_MAGIC)
10019                 return -EINVAL;
10020         if ((wol->wolopts & WAKE_MAGIC) &&
10021             !(tg3_flag(tp, WOL_CAP) && device_can_wakeup(dp)))
10022                 return -EINVAL;
10023
10024         device_set_wakeup_enable(dp, wol->wolopts & WAKE_MAGIC);
10025
10026         spin_lock_bh(&tp->lock);
10027         if (device_may_wakeup(dp))
10028                 tg3_flag_set(tp, WOL_ENABLE);
10029         else
10030                 tg3_flag_clear(tp, WOL_ENABLE);
10031         spin_unlock_bh(&tp->lock);
10032
10033         return 0;
10034 }
10035
10036 static u32 tg3_get_msglevel(struct net_device *dev)
10037 {
10038         struct tg3 *tp = netdev_priv(dev);
10039         return tp->msg_enable;
10040 }
10041
10042 static void tg3_set_msglevel(struct net_device *dev, u32 value)
10043 {
10044         struct tg3 *tp = netdev_priv(dev);
10045         tp->msg_enable = value;
10046 }
10047
10048 static int tg3_nway_reset(struct net_device *dev)
10049 {
10050         struct tg3 *tp = netdev_priv(dev);
10051         int r;
10052
10053         if (!netif_running(dev))
10054                 return -EAGAIN;
10055
10056         if (tp->phy_flags & TG3_PHYFLG_PHY_SERDES)
10057                 return -EINVAL;
10058
10059         if (tg3_flag(tp, USE_PHYLIB)) {
10060                 if (!(tp->phy_flags & TG3_PHYFLG_IS_CONNECTED))
10061                         return -EAGAIN;
10062                 r = phy_start_aneg(tp->mdio_bus->phy_map[TG3_PHY_MII_ADDR]);
10063         } else {
10064                 u32 bmcr;
10065
10066                 spin_lock_bh(&tp->lock);
10067                 r = -EINVAL;
10068                 tg3_readphy(tp, MII_BMCR, &bmcr);
10069                 if (!tg3_readphy(tp, MII_BMCR, &bmcr) &&
10070                     ((bmcr & BMCR_ANENABLE) ||
10071                      (tp->phy_flags & TG3_PHYFLG_PARALLEL_DETECT))) {
10072                         tg3_writephy(tp, MII_BMCR, bmcr | BMCR_ANRESTART |
10073                                                    BMCR_ANENABLE);
10074                         r = 0;
10075                 }
10076                 spin_unlock_bh(&tp->lock);
10077         }
10078
10079         return r;
10080 }
10081
10082 static void tg3_get_ringparam(struct net_device *dev, struct ethtool_ringparam *ering)
10083 {
10084         struct tg3 *tp = netdev_priv(dev);
10085
10086         ering->rx_max_pending = tp->rx_std_ring_mask;
10087         ering->rx_mini_max_pending = 0;
10088         if (tg3_flag(tp, JUMBO_RING_ENABLE))
10089                 ering->rx_jumbo_max_pending = tp->rx_jmb_ring_mask;
10090         else
10091                 ering->rx_jumbo_max_pending = 0;
10092
10093         ering->tx_max_pending = TG3_TX_RING_SIZE - 1;
10094
10095         ering->rx_pending = tp->rx_pending;
10096         ering->rx_mini_pending = 0;
10097         if (tg3_flag(tp, JUMBO_RING_ENABLE))
10098                 ering->rx_jumbo_pending = tp->rx_jumbo_pending;
10099         else
10100                 ering->rx_jumbo_pending = 0;
10101
10102         ering->tx_pending = tp->napi[0].tx_pending;
10103 }
10104
10105 static int tg3_set_ringparam(struct net_device *dev, struct ethtool_ringparam *ering)
10106 {
10107         struct tg3 *tp = netdev_priv(dev);
10108         int i, irq_sync = 0, err = 0;
10109
10110         if ((ering->rx_pending > tp->rx_std_ring_mask) ||
10111             (ering->rx_jumbo_pending > tp->rx_jmb_ring_mask) ||
10112             (ering->tx_pending > TG3_TX_RING_SIZE - 1) ||
10113             (ering->tx_pending <= MAX_SKB_FRAGS) ||
10114             (tg3_flag(tp, TSO_BUG) &&
10115              (ering->tx_pending <= (MAX_SKB_FRAGS * 3))))
10116                 return -EINVAL;
10117
10118         if (netif_running(dev)) {
10119                 tg3_phy_stop(tp);
10120                 tg3_netif_stop(tp);
10121                 irq_sync = 1;
10122         }
10123
10124         tg3_full_lock(tp, irq_sync);
10125
10126         tp->rx_pending = ering->rx_pending;
10127
10128         if (tg3_flag(tp, MAX_RXPEND_64) &&
10129             tp->rx_pending > 63)
10130                 tp->rx_pending = 63;
10131         tp->rx_jumbo_pending = ering->rx_jumbo_pending;
10132
10133         for (i = 0; i < tp->irq_max; i++)
10134                 tp->napi[i].tx_pending = ering->tx_pending;
10135
10136         if (netif_running(dev)) {
10137                 tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
10138                 err = tg3_restart_hw(tp, 1);
10139                 if (!err)
10140                         tg3_netif_start(tp);
10141         }
10142
10143         tg3_full_unlock(tp);
10144
10145         if (irq_sync && !err)
10146                 tg3_phy_start(tp);
10147
10148         return err;
10149 }
10150
10151 static void tg3_get_pauseparam(struct net_device *dev, struct ethtool_pauseparam *epause)
10152 {
10153         struct tg3 *tp = netdev_priv(dev);
10154
10155         epause->autoneg = !!tg3_flag(tp, PAUSE_AUTONEG);
10156
10157         if (tp->link_config.active_flowctrl & FLOW_CTRL_RX)
10158                 epause->rx_pause = 1;
10159         else
10160                 epause->rx_pause = 0;
10161
10162         if (tp->link_config.active_flowctrl & FLOW_CTRL_TX)
10163                 epause->tx_pause = 1;
10164         else
10165                 epause->tx_pause = 0;
10166 }
10167
10168 static int tg3_set_pauseparam(struct net_device *dev, struct ethtool_pauseparam *epause)
10169 {
10170         struct tg3 *tp = netdev_priv(dev);
10171         int err = 0;
10172
10173         if (tg3_flag(tp, USE_PHYLIB)) {
10174                 u32 newadv;
10175                 struct phy_device *phydev;
10176
10177                 phydev = tp->mdio_bus->phy_map[TG3_PHY_MII_ADDR];
10178
10179                 if (!(phydev->supported & SUPPORTED_Pause) ||
10180                     (!(phydev->supported & SUPPORTED_Asym_Pause) &&
10181                      (epause->rx_pause != epause->tx_pause)))
10182                         return -EINVAL;
10183
10184                 tp->link_config.flowctrl = 0;
10185                 if (epause->rx_pause) {
10186                         tp->link_config.flowctrl |= FLOW_CTRL_RX;
10187
10188                         if (epause->tx_pause) {
10189                                 tp->link_config.flowctrl |= FLOW_CTRL_TX;
10190                                 newadv = ADVERTISED_Pause;
10191                         } else
10192                                 newadv = ADVERTISED_Pause |
10193                                          ADVERTISED_Asym_Pause;
10194                 } else if (epause->tx_pause) {
10195                         tp->link_config.flowctrl |= FLOW_CTRL_TX;
10196                         newadv = ADVERTISED_Asym_Pause;
10197                 } else
10198                         newadv = 0;
10199
10200                 if (epause->autoneg)
10201                         tg3_flag_set(tp, PAUSE_AUTONEG);
10202                 else
10203                         tg3_flag_clear(tp, PAUSE_AUTONEG);
10204
10205                 if (tp->phy_flags & TG3_PHYFLG_IS_CONNECTED) {
10206                         u32 oldadv = phydev->advertising &
10207                                      (ADVERTISED_Pause | ADVERTISED_Asym_Pause);
10208                         if (oldadv != newadv) {
10209                                 phydev->advertising &=
10210                                         ~(ADVERTISED_Pause |
10211                                           ADVERTISED_Asym_Pause);
10212                                 phydev->advertising |= newadv;
10213                                 if (phydev->autoneg) {
10214                                         /*
10215                                          * Always renegotiate the link to
10216                                          * inform our link partner of our
10217                                          * flow control settings, even if the
10218                                          * flow control is forced.  Let
10219                                          * tg3_adjust_link() do the final
10220                                          * flow control setup.
10221                                          */
10222                                         return phy_start_aneg(phydev);
10223                                 }
10224                         }
10225
10226                         if (!epause->autoneg)
10227                                 tg3_setup_flow_control(tp, 0, 0);
10228                 } else {
10229                         tp->link_config.orig_advertising &=
10230                                         ~(ADVERTISED_Pause |
10231                                           ADVERTISED_Asym_Pause);
10232                         tp->link_config.orig_advertising |= newadv;
10233                 }
10234         } else {
10235                 int irq_sync = 0;
10236
10237                 if (netif_running(dev)) {
10238                         tg3_netif_stop(tp);
10239                         irq_sync = 1;
10240                 }
10241
10242                 tg3_full_lock(tp, irq_sync);
10243
10244                 if (epause->autoneg)
10245                         tg3_flag_set(tp, PAUSE_AUTONEG);
10246                 else
10247                         tg3_flag_clear(tp, PAUSE_AUTONEG);
10248                 if (epause->rx_pause)
10249                         tp->link_config.flowctrl |= FLOW_CTRL_RX;
10250                 else
10251                         tp->link_config.flowctrl &= ~FLOW_CTRL_RX;
10252                 if (epause->tx_pause)
10253                         tp->link_config.flowctrl |= FLOW_CTRL_TX;
10254                 else
10255                         tp->link_config.flowctrl &= ~FLOW_CTRL_TX;
10256
10257                 if (netif_running(dev)) {
10258                         tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
10259                         err = tg3_restart_hw(tp, 1);
10260                         if (!err)
10261                                 tg3_netif_start(tp);
10262                 }
10263
10264                 tg3_full_unlock(tp);
10265         }
10266
10267         return err;
10268 }
10269
10270 static int tg3_get_sset_count(struct net_device *dev, int sset)
10271 {
10272         switch (sset) {
10273         case ETH_SS_TEST:
10274                 return TG3_NUM_TEST;
10275         case ETH_SS_STATS:
10276                 return TG3_NUM_STATS;
10277         default:
10278                 return -EOPNOTSUPP;
10279         }
10280 }
10281
10282 static void tg3_get_strings(struct net_device *dev, u32 stringset, u8 *buf)
10283 {
10284         switch (stringset) {
10285         case ETH_SS_STATS:
10286                 memcpy(buf, &ethtool_stats_keys, sizeof(ethtool_stats_keys));
10287                 break;
10288         case ETH_SS_TEST:
10289                 memcpy(buf, &ethtool_test_keys, sizeof(ethtool_test_keys));
10290                 break;
10291         default:
10292                 WARN_ON(1);     /* we need a WARN() */
10293                 break;
10294         }
10295 }
10296
10297 static int tg3_set_phys_id(struct net_device *dev,
10298                             enum ethtool_phys_id_state state)
10299 {
10300         struct tg3 *tp = netdev_priv(dev);
10301
10302         if (!netif_running(tp->dev))
10303                 return -EAGAIN;
10304
10305         switch (state) {
10306         case ETHTOOL_ID_ACTIVE:
10307                 return 1;       /* cycle on/off once per second */
10308
10309         case ETHTOOL_ID_ON:
10310                 tw32(MAC_LED_CTRL, LED_CTRL_LNKLED_OVERRIDE |
10311                      LED_CTRL_1000MBPS_ON |
10312                      LED_CTRL_100MBPS_ON |
10313                      LED_CTRL_10MBPS_ON |
10314                      LED_CTRL_TRAFFIC_OVERRIDE |
10315                      LED_CTRL_TRAFFIC_BLINK |
10316                      LED_CTRL_TRAFFIC_LED);
10317                 break;
10318
10319         case ETHTOOL_ID_OFF:
10320                 tw32(MAC_LED_CTRL, LED_CTRL_LNKLED_OVERRIDE |
10321                      LED_CTRL_TRAFFIC_OVERRIDE);
10322                 break;
10323
10324         case ETHTOOL_ID_INACTIVE:
10325                 tw32(MAC_LED_CTRL, tp->led_ctrl);
10326                 break;
10327         }
10328
10329         return 0;
10330 }
10331
10332 static void tg3_get_ethtool_stats(struct net_device *dev,
10333                                    struct ethtool_stats *estats, u64 *tmp_stats)
10334 {
10335         struct tg3 *tp = netdev_priv(dev);
10336         memcpy(tmp_stats, tg3_get_estats(tp), sizeof(tp->estats));
10337 }
10338
10339 static __be32 * tg3_vpd_readblock(struct tg3 *tp)
10340 {
10341         int i;
10342         __be32 *buf;
10343         u32 offset = 0, len = 0;
10344         u32 magic, val;
10345
10346         if (tg3_flag(tp, NO_NVRAM) || tg3_nvram_read(tp, 0, &magic))
10347                 return NULL;
10348
10349         if (magic == TG3_EEPROM_MAGIC) {
10350                 for (offset = TG3_NVM_DIR_START;
10351                      offset < TG3_NVM_DIR_END;
10352                      offset += TG3_NVM_DIRENT_SIZE) {
10353                         if (tg3_nvram_read(tp, offset, &val))
10354                                 return NULL;
10355
10356                         if ((val >> TG3_NVM_DIRTYPE_SHIFT) ==
10357                             TG3_NVM_DIRTYPE_EXTVPD)
10358                                 break;
10359                 }
10360
10361                 if (offset != TG3_NVM_DIR_END) {
10362                         len = (val & TG3_NVM_DIRTYPE_LENMSK) * 4;
10363                         if (tg3_nvram_read(tp, offset + 4, &offset))
10364                                 return NULL;
10365
10366                         offset = tg3_nvram_logical_addr(tp, offset);
10367                 }
10368         }
10369
10370         if (!offset || !len) {
10371                 offset = TG3_NVM_VPD_OFF;
10372                 len = TG3_NVM_VPD_LEN;
10373         }
10374
10375         buf = kmalloc(len, GFP_KERNEL);
10376         if (buf == NULL)
10377                 return NULL;
10378
10379         if (magic == TG3_EEPROM_MAGIC) {
10380                 for (i = 0; i < len; i += 4) {
10381                         /* The data is in little-endian format in NVRAM.
10382                          * Use the big-endian read routines to preserve
10383                          * the byte order as it exists in NVRAM.
10384                          */
10385                         if (tg3_nvram_read_be32(tp, offset + i, &buf[i/4]))
10386                                 goto error;
10387                 }
10388         } else {
10389                 u8 *ptr;
10390                 ssize_t cnt;
10391                 unsigned int pos = 0;
10392
10393                 ptr = (u8 *)&buf[0];
10394                 for (i = 0; pos < len && i < 3; i++, pos += cnt, ptr += cnt) {
10395                         cnt = pci_read_vpd(tp->pdev, pos,
10396                                            len - pos, ptr);
10397                         if (cnt == -ETIMEDOUT || cnt == -EINTR)
10398                                 cnt = 0;
10399                         else if (cnt < 0)
10400                                 goto error;
10401                 }
10402                 if (pos != len)
10403                         goto error;
10404         }
10405
10406         return buf;
10407
10408 error:
10409         kfree(buf);
10410         return NULL;
10411 }
10412
10413 #define NVRAM_TEST_SIZE 0x100
10414 #define NVRAM_SELFBOOT_FORMAT1_0_SIZE   0x14
10415 #define NVRAM_SELFBOOT_FORMAT1_2_SIZE   0x18
10416 #define NVRAM_SELFBOOT_FORMAT1_3_SIZE   0x1c
10417 #define NVRAM_SELFBOOT_HW_SIZE 0x20
10418 #define NVRAM_SELFBOOT_DATA_SIZE 0x1c
10419
10420 static int tg3_test_nvram(struct tg3 *tp)
10421 {
10422         u32 csum, magic;
10423         __be32 *buf;
10424         int i, j, k, err = 0, size;
10425
10426         if (tg3_flag(tp, NO_NVRAM))
10427                 return 0;
10428
10429         if (tg3_nvram_read(tp, 0, &magic) != 0)
10430                 return -EIO;
10431
10432         if (magic == TG3_EEPROM_MAGIC)
10433                 size = NVRAM_TEST_SIZE;
10434         else if ((magic & TG3_EEPROM_MAGIC_FW_MSK) == TG3_EEPROM_MAGIC_FW) {
10435                 if ((magic & TG3_EEPROM_SB_FORMAT_MASK) ==
10436                     TG3_EEPROM_SB_FORMAT_1) {
10437                         switch (magic & TG3_EEPROM_SB_REVISION_MASK) {
10438                         case TG3_EEPROM_SB_REVISION_0:
10439                                 size = NVRAM_SELFBOOT_FORMAT1_0_SIZE;
10440                                 break;
10441                         case TG3_EEPROM_SB_REVISION_2:
10442                                 size = NVRAM_SELFBOOT_FORMAT1_2_SIZE;
10443                                 break;
10444                         case TG3_EEPROM_SB_REVISION_3:
10445                                 size = NVRAM_SELFBOOT_FORMAT1_3_SIZE;
10446                                 break;
10447                         default:
10448                                 return 0;
10449                         }
10450                 } else
10451                         return 0;
10452         } else if ((magic & TG3_EEPROM_MAGIC_HW_MSK) == TG3_EEPROM_MAGIC_HW)
10453                 size = NVRAM_SELFBOOT_HW_SIZE;
10454         else
10455                 return -EIO;
10456
10457         buf = kmalloc(size, GFP_KERNEL);
10458         if (buf == NULL)
10459                 return -ENOMEM;
10460
10461         err = -EIO;
10462         for (i = 0, j = 0; i < size; i += 4, j++) {
10463                 err = tg3_nvram_read_be32(tp, i, &buf[j]);
10464                 if (err)
10465                         break;
10466         }
10467         if (i < size)
10468                 goto out;
10469
10470         /* Selfboot format */
10471         magic = be32_to_cpu(buf[0]);
10472         if ((magic & TG3_EEPROM_MAGIC_FW_MSK) ==
10473             TG3_EEPROM_MAGIC_FW) {
10474                 u8 *buf8 = (u8 *) buf, csum8 = 0;
10475
10476                 if ((magic & TG3_EEPROM_SB_REVISION_MASK) ==
10477                     TG3_EEPROM_SB_REVISION_2) {
10478                         /* For rev 2, the csum doesn't include the MBA. */
10479                         for (i = 0; i < TG3_EEPROM_SB_F1R2_MBA_OFF; i++)
10480                                 csum8 += buf8[i];
10481                         for (i = TG3_EEPROM_SB_F1R2_MBA_OFF + 4; i < size; i++)
10482                                 csum8 += buf8[i];
10483                 } else {
10484                         for (i = 0; i < size; i++)
10485                                 csum8 += buf8[i];
10486                 }
10487
10488                 if (csum8 == 0) {
10489                         err = 0;
10490                         goto out;
10491                 }
10492
10493                 err = -EIO;
10494                 goto out;
10495         }
10496
10497         if ((magic & TG3_EEPROM_MAGIC_HW_MSK) ==
10498             TG3_EEPROM_MAGIC_HW) {
10499                 u8 data[NVRAM_SELFBOOT_DATA_SIZE];
10500                 u8 parity[NVRAM_SELFBOOT_DATA_SIZE];
10501                 u8 *buf8 = (u8 *) buf;
10502
10503                 /* Separate the parity bits and the data bytes.  */
10504                 for (i = 0, j = 0, k = 0; i < NVRAM_SELFBOOT_HW_SIZE; i++) {
10505                         if ((i == 0) || (i == 8)) {
10506                                 int l;
10507                                 u8 msk;
10508
10509                                 for (l = 0, msk = 0x80; l < 7; l++, msk >>= 1)
10510                                         parity[k++] = buf8[i] & msk;
10511                                 i++;
10512                         } else if (i == 16) {
10513                                 int l;
10514                                 u8 msk;
10515
10516                                 for (l = 0, msk = 0x20; l < 6; l++, msk >>= 1)
10517                                         parity[k++] = buf8[i] & msk;
10518                                 i++;
10519
10520                                 for (l = 0, msk = 0x80; l < 8; l++, msk >>= 1)
10521                                         parity[k++] = buf8[i] & msk;
10522                                 i++;
10523                         }
10524                         data[j++] = buf8[i];
10525                 }
10526
10527                 err = -EIO;
10528                 for (i = 0; i < NVRAM_SELFBOOT_DATA_SIZE; i++) {
10529                         u8 hw8 = hweight8(data[i]);
10530
10531                         if ((hw8 & 0x1) && parity[i])
10532                                 goto out;
10533                         else if (!(hw8 & 0x1) && !parity[i])
10534                                 goto out;
10535                 }
10536                 err = 0;
10537                 goto out;
10538         }
10539
10540         err = -EIO;
10541
10542         /* Bootstrap checksum at offset 0x10 */
10543         csum = calc_crc((unsigned char *) buf, 0x10);
10544         if (csum != le32_to_cpu(buf[0x10/4]))
10545                 goto out;
10546
10547         /* Manufacturing block starts at offset 0x74, checksum at 0xfc */
10548         csum = calc_crc((unsigned char *) &buf[0x74/4], 0x88);
10549         if (csum != le32_to_cpu(buf[0xfc/4]))
10550                 goto out;
10551
10552         kfree(buf);
10553
10554         buf = tg3_vpd_readblock(tp);
10555         if (!buf)
10556                 return -ENOMEM;
10557
10558         i = pci_vpd_find_tag((u8 *)buf, 0, TG3_NVM_VPD_LEN,
10559                              PCI_VPD_LRDT_RO_DATA);
10560         if (i > 0) {
10561                 j = pci_vpd_lrdt_size(&((u8 *)buf)[i]);
10562                 if (j < 0)
10563                         goto out;
10564
10565                 if (i + PCI_VPD_LRDT_TAG_SIZE + j > TG3_NVM_VPD_LEN)
10566                         goto out;
10567
10568                 i += PCI_VPD_LRDT_TAG_SIZE;
10569                 j = pci_vpd_find_info_keyword((u8 *)buf, i, j,
10570                                               PCI_VPD_RO_KEYWORD_CHKSUM);
10571                 if (j > 0) {
10572                         u8 csum8 = 0;
10573
10574                         j += PCI_VPD_INFO_FLD_HDR_SIZE;
10575
10576                         for (i = 0; i <= j; i++)
10577                                 csum8 += ((u8 *)buf)[i];
10578
10579                         if (csum8)
10580                                 goto out;
10581                 }
10582         }
10583
10584         err = 0;
10585
10586 out:
10587         kfree(buf);
10588         return err;
10589 }
10590
10591 #define TG3_SERDES_TIMEOUT_SEC  2
10592 #define TG3_COPPER_TIMEOUT_SEC  6
10593
10594 static int tg3_test_link(struct tg3 *tp)
10595 {
10596         int i, max;
10597
10598         if (!netif_running(tp->dev))
10599                 return -ENODEV;
10600
10601         if (tp->phy_flags & TG3_PHYFLG_ANY_SERDES)
10602                 max = TG3_SERDES_TIMEOUT_SEC;
10603         else
10604                 max = TG3_COPPER_TIMEOUT_SEC;
10605
10606         for (i = 0; i < max; i++) {
10607                 if (netif_carrier_ok(tp->dev))
10608                         return 0;
10609
10610                 if (msleep_interruptible(1000))
10611                         break;
10612         }
10613
10614         return -EIO;
10615 }
10616
10617 /* Only test the commonly used registers */
10618 static int tg3_test_registers(struct tg3 *tp)
10619 {
10620         int i, is_5705, is_5750;
10621         u32 offset, read_mask, write_mask, val, save_val, read_val;
10622         static struct {
10623                 u16 offset;
10624                 u16 flags;
10625 #define TG3_FL_5705     0x1
10626 #define TG3_FL_NOT_5705 0x2
10627 #define TG3_FL_NOT_5788 0x4
10628 #define TG3_FL_NOT_5750 0x8
10629                 u32 read_mask;
10630                 u32 write_mask;
10631         } reg_tbl[] = {
10632                 /* MAC Control Registers */
10633                 { MAC_MODE, TG3_FL_NOT_5705,
10634                         0x00000000, 0x00ef6f8c },
10635                 { MAC_MODE, TG3_FL_5705,
10636                         0x00000000, 0x01ef6b8c },
10637                 { MAC_STATUS, TG3_FL_NOT_5705,
10638                         0x03800107, 0x00000000 },
10639                 { MAC_STATUS, TG3_FL_5705,
10640                         0x03800100, 0x00000000 },
10641                 { MAC_ADDR_0_HIGH, 0x0000,
10642                         0x00000000, 0x0000ffff },
10643                 { MAC_ADDR_0_LOW, 0x0000,
10644                         0x00000000, 0xffffffff },
10645                 { MAC_RX_MTU_SIZE, 0x0000,
10646                         0x00000000, 0x0000ffff },
10647                 { MAC_TX_MODE, 0x0000,
10648                         0x00000000, 0x00000070 },
10649                 { MAC_TX_LENGTHS, 0x0000,
10650                         0x00000000, 0x00003fff },
10651                 { MAC_RX_MODE, TG3_FL_NOT_5705,
10652                         0x00000000, 0x000007fc },
10653                 { MAC_RX_MODE, TG3_FL_5705,
10654                         0x00000000, 0x000007dc },
10655                 { MAC_HASH_REG_0, 0x0000,
10656                         0x00000000, 0xffffffff },
10657                 { MAC_HASH_REG_1, 0x0000,
10658                         0x00000000, 0xffffffff },
10659                 { MAC_HASH_REG_2, 0x0000,
10660                         0x00000000, 0xffffffff },
10661                 { MAC_HASH_REG_3, 0x0000,
10662                         0x00000000, 0xffffffff },
10663
10664                 /* Receive Data and Receive BD Initiator Control Registers. */
10665                 { RCVDBDI_JUMBO_BD+0, TG3_FL_NOT_5705,
10666                         0x00000000, 0xffffffff },
10667                 { RCVDBDI_JUMBO_BD+4, TG3_FL_NOT_5705,
10668                         0x00000000, 0xffffffff },
10669                 { RCVDBDI_JUMBO_BD+8, TG3_FL_NOT_5705,
10670                         0x00000000, 0x00000003 },
10671                 { RCVDBDI_JUMBO_BD+0xc, TG3_FL_NOT_5705,
10672                         0x00000000, 0xffffffff },
10673                 { RCVDBDI_STD_BD+0, 0x0000,
10674                         0x00000000, 0xffffffff },
10675                 { RCVDBDI_STD_BD+4, 0x0000,
10676                         0x00000000, 0xffffffff },
10677                 { RCVDBDI_STD_BD+8, 0x0000,
10678                         0x00000000, 0xffff0002 },
10679                 { RCVDBDI_STD_BD+0xc, 0x0000,
10680                         0x00000000, 0xffffffff },
10681
10682                 /* Receive BD Initiator Control Registers. */
10683                 { RCVBDI_STD_THRESH, TG3_FL_NOT_5705,
10684                         0x00000000, 0xffffffff },
10685                 { RCVBDI_STD_THRESH, TG3_FL_5705,
10686                         0x00000000, 0x000003ff },
10687                 { RCVBDI_JUMBO_THRESH, TG3_FL_NOT_5705,
10688                         0x00000000, 0xffffffff },
10689
10690                 /* Host Coalescing Control Registers. */
10691                 { HOSTCC_MODE, TG3_FL_NOT_5705,
10692                         0x00000000, 0x00000004 },
10693                 { HOSTCC_MODE, TG3_FL_5705,
10694                         0x00000000, 0x000000f6 },
10695                 { HOSTCC_RXCOL_TICKS, TG3_FL_NOT_5705,
10696                         0x00000000, 0xffffffff },
10697                 { HOSTCC_RXCOL_TICKS, TG3_FL_5705,
10698                         0x00000000, 0x000003ff },
10699                 { HOSTCC_TXCOL_TICKS, TG3_FL_NOT_5705,
10700                         0x00000000, 0xffffffff },
10701                 { HOSTCC_TXCOL_TICKS, TG3_FL_5705,
10702                         0x00000000, 0x000003ff },
10703                 { HOSTCC_RXMAX_FRAMES, TG3_FL_NOT_5705,
10704                         0x00000000, 0xffffffff },
10705                 { HOSTCC_RXMAX_FRAMES, TG3_FL_5705 | TG3_FL_NOT_5788,
10706                         0x00000000, 0x000000ff },
10707                 { HOSTCC_TXMAX_FRAMES, TG3_FL_NOT_5705,
10708                         0x00000000, 0xffffffff },
10709                 { HOSTCC_TXMAX_FRAMES, TG3_FL_5705 | TG3_FL_NOT_5788,
10710                         0x00000000, 0x000000ff },
10711                 { HOSTCC_RXCOAL_TICK_INT, TG3_FL_NOT_5705,
10712                         0x00000000, 0xffffffff },
10713                 { HOSTCC_TXCOAL_TICK_INT, TG3_FL_NOT_5705,
10714                         0x00000000, 0xffffffff },
10715                 { HOSTCC_RXCOAL_MAXF_INT, TG3_FL_NOT_5705,
10716                         0x00000000, 0xffffffff },
10717                 { HOSTCC_RXCOAL_MAXF_INT, TG3_FL_5705 | TG3_FL_NOT_5788,
10718                         0x00000000, 0x000000ff },
10719                 { HOSTCC_TXCOAL_MAXF_INT, TG3_FL_NOT_5705,
10720                         0x00000000, 0xffffffff },
10721                 { HOSTCC_TXCOAL_MAXF_INT, TG3_FL_5705 | TG3_FL_NOT_5788,
10722                         0x00000000, 0x000000ff },
10723                 { HOSTCC_STAT_COAL_TICKS, TG3_FL_NOT_5705,
10724                         0x00000000, 0xffffffff },
10725                 { HOSTCC_STATS_BLK_HOST_ADDR, TG3_FL_NOT_5705,
10726                         0x00000000, 0xffffffff },
10727                 { HOSTCC_STATS_BLK_HOST_ADDR+4, TG3_FL_NOT_5705,
10728                         0x00000000, 0xffffffff },
10729                 { HOSTCC_STATUS_BLK_HOST_ADDR, 0x0000,
10730                         0x00000000, 0xffffffff },
10731                 { HOSTCC_STATUS_BLK_HOST_ADDR+4, 0x0000,
10732                         0x00000000, 0xffffffff },
10733                 { HOSTCC_STATS_BLK_NIC_ADDR, 0x0000,
10734                         0xffffffff, 0x00000000 },
10735                 { HOSTCC_STATUS_BLK_NIC_ADDR, 0x0000,
10736                         0xffffffff, 0x00000000 },
10737
10738                 /* Buffer Manager Control Registers. */
10739                 { BUFMGR_MB_POOL_ADDR, TG3_FL_NOT_5750,
10740                         0x00000000, 0x007fff80 },
10741                 { BUFMGR_MB_POOL_SIZE, TG3_FL_NOT_5750,
10742                         0x00000000, 0x007fffff },
10743                 { BUFMGR_MB_RDMA_LOW_WATER, 0x0000,
10744                         0x00000000, 0x0000003f },
10745                 { BUFMGR_MB_MACRX_LOW_WATER, 0x0000,
10746                         0x00000000, 0x000001ff },
10747                 { BUFMGR_MB_HIGH_WATER, 0x0000,
10748                         0x00000000, 0x000001ff },
10749                 { BUFMGR_DMA_DESC_POOL_ADDR, TG3_FL_NOT_5705,
10750                         0xffffffff, 0x00000000 },
10751                 { BUFMGR_DMA_DESC_POOL_SIZE, TG3_FL_NOT_5705,
10752                         0xffffffff, 0x00000000 },
10753
10754                 /* Mailbox Registers */
10755                 { GRCMBOX_RCVSTD_PROD_IDX+4, 0x0000,
10756                         0x00000000, 0x000001ff },
10757                 { GRCMBOX_RCVJUMBO_PROD_IDX+4, TG3_FL_NOT_5705,
10758                         0x00000000, 0x000001ff },
10759                 { GRCMBOX_RCVRET_CON_IDX_0+4, 0x0000,
10760                         0x00000000, 0x000007ff },
10761                 { GRCMBOX_SNDHOST_PROD_IDX_0+4, 0x0000,
10762                         0x00000000, 0x000001ff },
10763
10764                 { 0xffff, 0x0000, 0x00000000, 0x00000000 },
10765         };
10766
10767         is_5705 = is_5750 = 0;
10768         if (tg3_flag(tp, 5705_PLUS)) {
10769                 is_5705 = 1;
10770                 if (tg3_flag(tp, 5750_PLUS))
10771                         is_5750 = 1;
10772         }
10773
10774         for (i = 0; reg_tbl[i].offset != 0xffff; i++) {
10775                 if (is_5705 && (reg_tbl[i].flags & TG3_FL_NOT_5705))
10776                         continue;
10777
10778                 if (!is_5705 && (reg_tbl[i].flags & TG3_FL_5705))
10779                         continue;
10780
10781                 if (tg3_flag(tp, IS_5788) &&
10782                     (reg_tbl[i].flags & TG3_FL_NOT_5788))
10783                         continue;
10784
10785                 if (is_5750 && (reg_tbl[i].flags & TG3_FL_NOT_5750))
10786                         continue;
10787
10788                 offset = (u32) reg_tbl[i].offset;
10789                 read_mask = reg_tbl[i].read_mask;
10790                 write_mask = reg_tbl[i].write_mask;
10791
10792                 /* Save the original register content */
10793                 save_val = tr32(offset);
10794
10795                 /* Determine the read-only value. */
10796                 read_val = save_val & read_mask;
10797
10798                 /* Write zero to the register, then make sure the read-only bits
10799                  * are not changed and the read/write bits are all zeros.
10800                  */
10801                 tw32(offset, 0);
10802
10803                 val = tr32(offset);
10804
10805                 /* Test the read-only and read/write bits. */
10806                 if (((val & read_mask) != read_val) || (val & write_mask))
10807                         goto out;
10808
10809                 /* Write ones to all the bits defined by RdMask and WrMask, then
10810                  * make sure the read-only bits are not changed and the
10811                  * read/write bits are all ones.
10812                  */
10813                 tw32(offset, read_mask | write_mask);
10814
10815                 val = tr32(offset);
10816
10817                 /* Test the read-only bits. */
10818                 if ((val & read_mask) != read_val)
10819                         goto out;
10820
10821                 /* Test the read/write bits. */
10822                 if ((val & write_mask) != write_mask)
10823                         goto out;
10824
10825                 tw32(offset, save_val);
10826         }
10827
10828         return 0;
10829
10830 out:
10831         if (netif_msg_hw(tp))
10832                 netdev_err(tp->dev,
10833                            "Register test failed at offset %x\n", offset);
10834         tw32(offset, save_val);
10835         return -EIO;
10836 }
10837
10838 static int tg3_do_mem_test(struct tg3 *tp, u32 offset, u32 len)
10839 {
10840         static const u32 test_pattern[] = { 0x00000000, 0xffffffff, 0xaa55a55a };
10841         int i;
10842         u32 j;
10843
10844         for (i = 0; i < ARRAY_SIZE(test_pattern); i++) {
10845                 for (j = 0; j < len; j += 4) {
10846                         u32 val;
10847
10848                         tg3_write_mem(tp, offset + j, test_pattern[i]);
10849                         tg3_read_mem(tp, offset + j, &val);
10850                         if (val != test_pattern[i])
10851                                 return -EIO;
10852                 }
10853         }
10854         return 0;
10855 }
10856
10857 static int tg3_test_memory(struct tg3 *tp)
10858 {
10859         static struct mem_entry {
10860                 u32 offset;
10861                 u32 len;
10862         } mem_tbl_570x[] = {
10863                 { 0x00000000, 0x00b50},
10864                 { 0x00002000, 0x1c000},
10865                 { 0xffffffff, 0x00000}
10866         }, mem_tbl_5705[] = {
10867                 { 0x00000100, 0x0000c},
10868                 { 0x00000200, 0x00008},
10869                 { 0x00004000, 0x00800},
10870                 { 0x00006000, 0x01000},
10871                 { 0x00008000, 0x02000},
10872                 { 0x00010000, 0x0e000},
10873                 { 0xffffffff, 0x00000}
10874         }, mem_tbl_5755[] = {
10875                 { 0x00000200, 0x00008},
10876                 { 0x00004000, 0x00800},
10877                 { 0x00006000, 0x00800},
10878                 { 0x00008000, 0x02000},
10879                 { 0x00010000, 0x0c000},
10880                 { 0xffffffff, 0x00000}
10881         }, mem_tbl_5906[] = {
10882                 { 0x00000200, 0x00008},
10883                 { 0x00004000, 0x00400},
10884                 { 0x00006000, 0x00400},
10885                 { 0x00008000, 0x01000},
10886                 { 0x00010000, 0x01000},
10887                 { 0xffffffff, 0x00000}
10888         }, mem_tbl_5717[] = {
10889                 { 0x00000200, 0x00008},
10890                 { 0x00010000, 0x0a000},
10891                 { 0x00020000, 0x13c00},
10892                 { 0xffffffff, 0x00000}
10893         }, mem_tbl_57765[] = {
10894                 { 0x00000200, 0x00008},
10895                 { 0x00004000, 0x00800},
10896                 { 0x00006000, 0x09800},
10897                 { 0x00010000, 0x0a000},
10898                 { 0xffffffff, 0x00000}
10899         };
10900         struct mem_entry *mem_tbl;
10901         int err = 0;
10902         int i;
10903
10904         if (tg3_flag(tp, 5717_PLUS))
10905                 mem_tbl = mem_tbl_5717;
10906         else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_57765)
10907                 mem_tbl = mem_tbl_57765;
10908         else if (tg3_flag(tp, 5755_PLUS))
10909                 mem_tbl = mem_tbl_5755;
10910         else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906)
10911                 mem_tbl = mem_tbl_5906;
10912         else if (tg3_flag(tp, 5705_PLUS))
10913                 mem_tbl = mem_tbl_5705;
10914         else
10915                 mem_tbl = mem_tbl_570x;
10916
10917         for (i = 0; mem_tbl[i].offset != 0xffffffff; i++) {
10918                 err = tg3_do_mem_test(tp, mem_tbl[i].offset, mem_tbl[i].len);
10919                 if (err)
10920                         break;
10921         }
10922
10923         return err;
10924 }
10925
10926 #define TG3_MAC_LOOPBACK        0
10927 #define TG3_PHY_LOOPBACK        1
10928 #define TG3_TSO_LOOPBACK        2
10929
10930 #define TG3_TSO_MSS             500
10931
10932 #define TG3_TSO_IP_HDR_LEN      20
10933 #define TG3_TSO_TCP_HDR_LEN     20
10934 #define TG3_TSO_TCP_OPT_LEN     12
10935
10936 static const u8 tg3_tso_header[] = {
10937 0x08, 0x00,
10938 0x45, 0x00, 0x00, 0x00,
10939 0x00, 0x00, 0x40, 0x00,
10940 0x40, 0x06, 0x00, 0x00,
10941 0x0a, 0x00, 0x00, 0x01,
10942 0x0a, 0x00, 0x00, 0x02,
10943 0x0d, 0x00, 0xe0, 0x00,
10944 0x00, 0x00, 0x01, 0x00,
10945 0x00, 0x00, 0x02, 0x00,
10946 0x80, 0x10, 0x10, 0x00,
10947 0x14, 0x09, 0x00, 0x00,
10948 0x01, 0x01, 0x08, 0x0a,
10949 0x11, 0x11, 0x11, 0x11,
10950 0x11, 0x11, 0x11, 0x11,
10951 };
10952
10953 static int tg3_run_loopback(struct tg3 *tp, u32 pktsz, int loopback_mode)
10954 {
10955         u32 mac_mode, rx_start_idx, rx_idx, tx_idx, opaque_key;
10956         u32 base_flags = 0, mss = 0, desc_idx, coal_now, data_off, val;
10957         struct sk_buff *skb, *rx_skb;
10958         u8 *tx_data;
10959         dma_addr_t map;
10960         int num_pkts, tx_len, rx_len, i, err;
10961         struct tg3_rx_buffer_desc *desc;
10962         struct tg3_napi *tnapi, *rnapi;
10963         struct tg3_rx_prodring_set *tpr = &tp->napi[0].prodring;
10964
10965         tnapi = &tp->napi[0];
10966         rnapi = &tp->napi[0];
10967         if (tp->irq_cnt > 1) {
10968                 if (tg3_flag(tp, ENABLE_RSS))
10969                         rnapi = &tp->napi[1];
10970                 if (tg3_flag(tp, ENABLE_TSS))
10971                         tnapi = &tp->napi[1];
10972         }
10973         coal_now = tnapi->coal_now | rnapi->coal_now;
10974
10975         if (loopback_mode == TG3_MAC_LOOPBACK) {
10976                 /* HW errata - mac loopback fails in some cases on 5780.
10977                  * Normal traffic and PHY loopback are not affected by
10978                  * errata.  Also, the MAC loopback test is deprecated for
10979                  * all newer ASIC revisions.
10980                  */
10981                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5780 ||
10982                     tg3_flag(tp, CPMU_PRESENT))
10983                         return 0;
10984
10985                 mac_mode = tp->mac_mode &
10986                            ~(MAC_MODE_PORT_MODE_MASK | MAC_MODE_HALF_DUPLEX);
10987                 mac_mode |= MAC_MODE_PORT_INT_LPBACK;
10988                 if (!tg3_flag(tp, 5705_PLUS))
10989                         mac_mode |= MAC_MODE_LINK_POLARITY;
10990                 if (tp->phy_flags & TG3_PHYFLG_10_100_ONLY)
10991                         mac_mode |= MAC_MODE_PORT_MODE_MII;
10992                 else
10993                         mac_mode |= MAC_MODE_PORT_MODE_GMII;
10994                 tw32(MAC_MODE, mac_mode);
10995         } else {
10996                 if (tp->phy_flags & TG3_PHYFLG_IS_FET) {
10997                         tg3_phy_fet_toggle_apd(tp, false);
10998                         val = BMCR_LOOPBACK | BMCR_FULLDPLX | BMCR_SPEED100;
10999                 } else
11000                         val = BMCR_LOOPBACK | BMCR_FULLDPLX | BMCR_SPEED1000;
11001
11002                 tg3_phy_toggle_automdix(tp, 0);
11003
11004                 tg3_writephy(tp, MII_BMCR, val);
11005                 udelay(40);
11006
11007                 mac_mode = tp->mac_mode &
11008                            ~(MAC_MODE_PORT_MODE_MASK | MAC_MODE_HALF_DUPLEX);
11009                 if (tp->phy_flags & TG3_PHYFLG_IS_FET) {
11010                         tg3_writephy(tp, MII_TG3_FET_PTEST,
11011                                      MII_TG3_FET_PTEST_FRC_TX_LINK |
11012                                      MII_TG3_FET_PTEST_FRC_TX_LOCK);
11013                         /* The write needs to be flushed for the AC131 */
11014                         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5785)
11015                                 tg3_readphy(tp, MII_TG3_FET_PTEST, &val);
11016                         mac_mode |= MAC_MODE_PORT_MODE_MII;
11017                 } else
11018                         mac_mode |= MAC_MODE_PORT_MODE_GMII;
11019
11020                 /* reset to prevent losing 1st rx packet intermittently */
11021                 if (tp->phy_flags & TG3_PHYFLG_MII_SERDES) {
11022                         tw32_f(MAC_RX_MODE, RX_MODE_RESET);
11023                         udelay(10);
11024                         tw32_f(MAC_RX_MODE, tp->rx_mode);
11025                 }
11026                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700) {
11027                         u32 masked_phy_id = tp->phy_id & TG3_PHY_ID_MASK;
11028                         if (masked_phy_id == TG3_PHY_ID_BCM5401)
11029                                 mac_mode &= ~MAC_MODE_LINK_POLARITY;
11030                         else if (masked_phy_id == TG3_PHY_ID_BCM5411)
11031                                 mac_mode |= MAC_MODE_LINK_POLARITY;
11032                         tg3_writephy(tp, MII_TG3_EXT_CTRL,
11033                                      MII_TG3_EXT_CTRL_LNK3_LED_MODE);
11034                 }
11035                 tw32(MAC_MODE, mac_mode);
11036
11037                 /* Wait for link */
11038                 for (i = 0; i < 100; i++) {
11039                         if (tr32(MAC_TX_STATUS) & TX_STATUS_LINK_UP)
11040                                 break;
11041                         mdelay(1);
11042                 }
11043         }
11044
11045         err = -EIO;
11046
11047         tx_len = pktsz;
11048         skb = netdev_alloc_skb(tp->dev, tx_len);
11049         if (!skb)
11050                 return -ENOMEM;
11051
11052         tx_data = skb_put(skb, tx_len);
11053         memcpy(tx_data, tp->dev->dev_addr, 6);
11054         memset(tx_data + 6, 0x0, 8);
11055
11056         tw32(MAC_RX_MTU_SIZE, tx_len + ETH_FCS_LEN);
11057
11058         if (loopback_mode == TG3_TSO_LOOPBACK) {
11059                 struct iphdr *iph = (struct iphdr *)&tx_data[ETH_HLEN];
11060
11061                 u32 hdr_len = TG3_TSO_IP_HDR_LEN + TG3_TSO_TCP_HDR_LEN +
11062                               TG3_TSO_TCP_OPT_LEN;
11063
11064                 memcpy(tx_data + ETH_ALEN * 2, tg3_tso_header,
11065                        sizeof(tg3_tso_header));
11066                 mss = TG3_TSO_MSS;
11067
11068                 val = tx_len - ETH_ALEN * 2 - sizeof(tg3_tso_header);
11069                 num_pkts = DIV_ROUND_UP(val, TG3_TSO_MSS);
11070
11071                 /* Set the total length field in the IP header */
11072                 iph->tot_len = htons((u16)(mss + hdr_len));
11073
11074                 base_flags = (TXD_FLAG_CPU_PRE_DMA |
11075                               TXD_FLAG_CPU_POST_DMA);
11076
11077                 if (tg3_flag(tp, HW_TSO_1) ||
11078                     tg3_flag(tp, HW_TSO_2) ||
11079                     tg3_flag(tp, HW_TSO_3)) {
11080                         struct tcphdr *th;
11081                         val = ETH_HLEN + TG3_TSO_IP_HDR_LEN;
11082                         th = (struct tcphdr *)&tx_data[val];
11083                         th->check = 0;
11084                 } else
11085                         base_flags |= TXD_FLAG_TCPUDP_CSUM;
11086
11087                 if (tg3_flag(tp, HW_TSO_3)) {
11088                         mss |= (hdr_len & 0xc) << 12;
11089                         if (hdr_len & 0x10)
11090                                 base_flags |= 0x00000010;
11091                         base_flags |= (hdr_len & 0x3e0) << 5;
11092                 } else if (tg3_flag(tp, HW_TSO_2))
11093                         mss |= hdr_len << 9;
11094                 else if (tg3_flag(tp, HW_TSO_1) ||
11095                          GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) {
11096                         mss |= (TG3_TSO_TCP_OPT_LEN << 9);
11097                 } else {
11098                         base_flags |= (TG3_TSO_TCP_OPT_LEN << 10);
11099                 }
11100
11101                 data_off = ETH_ALEN * 2 + sizeof(tg3_tso_header);
11102         } else {
11103                 num_pkts = 1;
11104                 data_off = ETH_HLEN;
11105         }
11106
11107         for (i = data_off; i < tx_len; i++)
11108                 tx_data[i] = (u8) (i & 0xff);
11109
11110         map = pci_map_single(tp->pdev, skb->data, tx_len, PCI_DMA_TODEVICE);
11111         if (pci_dma_mapping_error(tp->pdev, map)) {
11112                 dev_kfree_skb(skb);
11113                 return -EIO;
11114         }
11115
11116         tw32_f(HOSTCC_MODE, tp->coalesce_mode | HOSTCC_MODE_ENABLE |
11117                rnapi->coal_now);
11118
11119         udelay(10);
11120
11121         rx_start_idx = rnapi->hw_status->idx[0].rx_producer;
11122
11123         tg3_set_txd(tnapi, tnapi->tx_prod, map, tx_len,
11124                     base_flags, (mss << 1) | 1);
11125
11126         tnapi->tx_prod++;
11127
11128         tw32_tx_mbox(tnapi->prodmbox, tnapi->tx_prod);
11129         tr32_mailbox(tnapi->prodmbox);
11130
11131         udelay(10);
11132
11133         /* 350 usec to allow enough time on some 10/100 Mbps devices.  */
11134         for (i = 0; i < 35; i++) {
11135                 tw32_f(HOSTCC_MODE, tp->coalesce_mode | HOSTCC_MODE_ENABLE |
11136                        coal_now);
11137
11138                 udelay(10);
11139
11140                 tx_idx = tnapi->hw_status->idx[0].tx_consumer;
11141                 rx_idx = rnapi->hw_status->idx[0].rx_producer;
11142                 if ((tx_idx == tnapi->tx_prod) &&
11143                     (rx_idx == (rx_start_idx + num_pkts)))
11144                         break;
11145         }
11146
11147         pci_unmap_single(tp->pdev, map, tx_len, PCI_DMA_TODEVICE);
11148         dev_kfree_skb(skb);
11149
11150         if (tx_idx != tnapi->tx_prod)
11151                 goto out;
11152
11153         if (rx_idx != rx_start_idx + num_pkts)
11154                 goto out;
11155
11156         val = data_off;
11157         while (rx_idx != rx_start_idx) {
11158                 desc = &rnapi->rx_rcb[rx_start_idx++];
11159                 desc_idx = desc->opaque & RXD_OPAQUE_INDEX_MASK;
11160                 opaque_key = desc->opaque & RXD_OPAQUE_RING_MASK;
11161
11162                 if ((desc->err_vlan & RXD_ERR_MASK) != 0 &&
11163                     (desc->err_vlan != RXD_ERR_ODD_NIBBLE_RCVD_MII))
11164                         goto out;
11165
11166                 rx_len = ((desc->idx_len & RXD_LEN_MASK) >> RXD_LEN_SHIFT)
11167                          - ETH_FCS_LEN;
11168
11169                 if (loopback_mode != TG3_TSO_LOOPBACK) {
11170                         if (rx_len != tx_len)
11171                                 goto out;
11172
11173                         if (pktsz <= TG3_RX_STD_DMA_SZ - ETH_FCS_LEN) {
11174                                 if (opaque_key != RXD_OPAQUE_RING_STD)
11175                                         goto out;
11176                         } else {
11177                                 if (opaque_key != RXD_OPAQUE_RING_JUMBO)
11178                                         goto out;
11179                         }
11180                 } else if ((desc->type_flags & RXD_FLAG_TCPUDP_CSUM) &&
11181                            (desc->ip_tcp_csum & RXD_TCPCSUM_MASK)
11182                             >> RXD_TCPCSUM_SHIFT == 0xffff) {
11183                         goto out;
11184                 }
11185
11186                 if (opaque_key == RXD_OPAQUE_RING_STD) {
11187                         rx_skb = tpr->rx_std_buffers[desc_idx].skb;
11188                         map = dma_unmap_addr(&tpr->rx_std_buffers[desc_idx],
11189                                              mapping);
11190                 } else if (opaque_key == RXD_OPAQUE_RING_JUMBO) {
11191                         rx_skb = tpr->rx_jmb_buffers[desc_idx].skb;
11192                         map = dma_unmap_addr(&tpr->rx_jmb_buffers[desc_idx],
11193                                              mapping);
11194                 } else
11195                         goto out;
11196
11197                 pci_dma_sync_single_for_cpu(tp->pdev, map, rx_len,
11198                                             PCI_DMA_FROMDEVICE);
11199
11200                 for (i = data_off; i < rx_len; i++, val++) {
11201                         if (*(rx_skb->data + i) != (u8) (val & 0xff))
11202                                 goto out;
11203                 }
11204         }
11205
11206         err = 0;
11207
11208         /* tg3_free_rings will unmap and free the rx_skb */
11209 out:
11210         return err;
11211 }
11212
11213 #define TG3_STD_LOOPBACK_FAILED         1
11214 #define TG3_JMB_LOOPBACK_FAILED         2
11215 #define TG3_TSO_LOOPBACK_FAILED         4
11216
11217 #define TG3_MAC_LOOPBACK_SHIFT          0
11218 #define TG3_PHY_LOOPBACK_SHIFT          4
11219 #define TG3_LOOPBACK_FAILED             0x00000077
11220
11221 static int tg3_test_loopback(struct tg3 *tp)
11222 {
11223         int err = 0;
11224         u32 eee_cap, cpmuctrl = 0;
11225
11226         if (!netif_running(tp->dev))
11227                 return TG3_LOOPBACK_FAILED;
11228
11229         eee_cap = tp->phy_flags & TG3_PHYFLG_EEE_CAP;
11230         tp->phy_flags &= ~TG3_PHYFLG_EEE_CAP;
11231
11232         err = tg3_reset_hw(tp, 1);
11233         if (err) {
11234                 err = TG3_LOOPBACK_FAILED;
11235                 goto done;
11236         }
11237
11238         if (tg3_flag(tp, ENABLE_RSS)) {
11239                 int i;
11240
11241                 /* Reroute all rx packets to the 1st queue */
11242                 for (i = MAC_RSS_INDIR_TBL_0;
11243                      i < MAC_RSS_INDIR_TBL_0 + TG3_RSS_INDIR_TBL_SIZE; i += 4)
11244                         tw32(i, 0x0);
11245         }
11246
11247         /* Turn off gphy autopowerdown. */
11248         if (tp->phy_flags & TG3_PHYFLG_ENABLE_APD)
11249                 tg3_phy_toggle_apd(tp, false);
11250
11251         if (tg3_flag(tp, CPMU_PRESENT)) {
11252                 int i;
11253                 u32 status;
11254
11255                 tw32(TG3_CPMU_MUTEX_REQ, CPMU_MUTEX_REQ_DRIVER);
11256
11257                 /* Wait for up to 40 microseconds to acquire lock. */
11258                 for (i = 0; i < 4; i++) {
11259                         status = tr32(TG3_CPMU_MUTEX_GNT);
11260                         if (status == CPMU_MUTEX_GNT_DRIVER)
11261                                 break;
11262                         udelay(10);
11263                 }
11264
11265                 if (status != CPMU_MUTEX_GNT_DRIVER) {
11266                         err = TG3_LOOPBACK_FAILED;
11267                         goto done;
11268                 }
11269
11270                 /* Turn off link-based power management. */
11271                 cpmuctrl = tr32(TG3_CPMU_CTRL);
11272                 tw32(TG3_CPMU_CTRL,
11273                      cpmuctrl & ~(CPMU_CTRL_LINK_SPEED_MODE |
11274                                   CPMU_CTRL_LINK_AWARE_MODE));
11275         }
11276
11277         if (tg3_run_loopback(tp, ETH_FRAME_LEN, TG3_MAC_LOOPBACK))
11278                 err |= TG3_STD_LOOPBACK_FAILED << TG3_MAC_LOOPBACK_SHIFT;
11279
11280         if (tg3_flag(tp, JUMBO_RING_ENABLE) &&
11281             tg3_run_loopback(tp, 9000 + ETH_HLEN, TG3_MAC_LOOPBACK))
11282                 err |= TG3_JMB_LOOPBACK_FAILED << TG3_MAC_LOOPBACK_SHIFT;
11283
11284         if (tg3_flag(tp, CPMU_PRESENT)) {
11285                 tw32(TG3_CPMU_CTRL, cpmuctrl);
11286
11287                 /* Release the mutex */
11288                 tw32(TG3_CPMU_MUTEX_GNT, CPMU_MUTEX_GNT_DRIVER);
11289         }
11290
11291         if (!(tp->phy_flags & TG3_PHYFLG_PHY_SERDES) &&
11292             !tg3_flag(tp, USE_PHYLIB)) {
11293                 if (tg3_run_loopback(tp, ETH_FRAME_LEN, TG3_PHY_LOOPBACK))
11294                         err |= TG3_STD_LOOPBACK_FAILED <<
11295                                TG3_PHY_LOOPBACK_SHIFT;
11296                 if (tg3_flag(tp, TSO_CAPABLE) &&
11297                     tg3_run_loopback(tp, ETH_FRAME_LEN, TG3_TSO_LOOPBACK))
11298                         err |= TG3_TSO_LOOPBACK_FAILED <<
11299                                TG3_PHY_LOOPBACK_SHIFT;
11300                 if (tg3_flag(tp, JUMBO_RING_ENABLE) &&
11301                     tg3_run_loopback(tp, 9000 + ETH_HLEN, TG3_PHY_LOOPBACK))
11302                         err |= TG3_JMB_LOOPBACK_FAILED <<
11303                                TG3_PHY_LOOPBACK_SHIFT;
11304         }
11305
11306         /* Re-enable gphy autopowerdown. */
11307         if (tp->phy_flags & TG3_PHYFLG_ENABLE_APD)
11308                 tg3_phy_toggle_apd(tp, true);
11309
11310 done:
11311         tp->phy_flags |= eee_cap;
11312
11313         return err;
11314 }
11315
11316 static void tg3_self_test(struct net_device *dev, struct ethtool_test *etest,
11317                           u64 *data)
11318 {
11319         struct tg3 *tp = netdev_priv(dev);
11320
11321         if (tp->phy_flags & TG3_PHYFLG_IS_LOW_POWER)
11322                 tg3_power_up(tp);
11323
11324         memset(data, 0, sizeof(u64) * TG3_NUM_TEST);
11325
11326         if (tg3_test_nvram(tp) != 0) {
11327                 etest->flags |= ETH_TEST_FL_FAILED;
11328                 data[0] = 1;
11329         }
11330         if (tg3_test_link(tp) != 0) {
11331                 etest->flags |= ETH_TEST_FL_FAILED;
11332                 data[1] = 1;
11333         }
11334         if (etest->flags & ETH_TEST_FL_OFFLINE) {
11335                 int err, err2 = 0, irq_sync = 0;
11336
11337                 if (netif_running(dev)) {
11338                         tg3_phy_stop(tp);
11339                         tg3_netif_stop(tp);
11340                         irq_sync = 1;
11341                 }
11342
11343                 tg3_full_lock(tp, irq_sync);
11344
11345                 tg3_halt(tp, RESET_KIND_SUSPEND, 1);
11346                 err = tg3_nvram_lock(tp);
11347                 tg3_halt_cpu(tp, RX_CPU_BASE);
11348                 if (!tg3_flag(tp, 5705_PLUS))
11349                         tg3_halt_cpu(tp, TX_CPU_BASE);
11350                 if (!err)
11351                         tg3_nvram_unlock(tp);
11352
11353                 if (tp->phy_flags & TG3_PHYFLG_MII_SERDES)
11354                         tg3_phy_reset(tp);
11355
11356                 if (tg3_test_registers(tp) != 0) {
11357                         etest->flags |= ETH_TEST_FL_FAILED;
11358                         data[2] = 1;
11359                 }
11360                 if (tg3_test_memory(tp) != 0) {
11361                         etest->flags |= ETH_TEST_FL_FAILED;
11362                         data[3] = 1;
11363                 }
11364                 if ((data[4] = tg3_test_loopback(tp)) != 0)
11365                         etest->flags |= ETH_TEST_FL_FAILED;
11366
11367                 tg3_full_unlock(tp);
11368
11369                 if (tg3_test_interrupt(tp) != 0) {
11370                         etest->flags |= ETH_TEST_FL_FAILED;
11371                         data[5] = 1;
11372                 }
11373
11374                 tg3_full_lock(tp, 0);
11375
11376                 tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
11377                 if (netif_running(dev)) {
11378                         tg3_flag_set(tp, INIT_COMPLETE);
11379                         err2 = tg3_restart_hw(tp, 1);
11380                         if (!err2)
11381                                 tg3_netif_start(tp);
11382                 }
11383
11384                 tg3_full_unlock(tp);
11385
11386                 if (irq_sync && !err2)
11387                         tg3_phy_start(tp);
11388         }
11389         if (tp->phy_flags & TG3_PHYFLG_IS_LOW_POWER)
11390                 tg3_power_down(tp);
11391
11392 }
11393
11394 static int tg3_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
11395 {
11396         struct mii_ioctl_data *data = if_mii(ifr);
11397         struct tg3 *tp = netdev_priv(dev);
11398         int err;
11399
11400         if (tg3_flag(tp, USE_PHYLIB)) {
11401                 struct phy_device *phydev;
11402                 if (!(tp->phy_flags & TG3_PHYFLG_IS_CONNECTED))
11403                         return -EAGAIN;
11404                 phydev = tp->mdio_bus->phy_map[TG3_PHY_MII_ADDR];
11405                 return phy_mii_ioctl(phydev, ifr, cmd);
11406         }
11407
11408         switch (cmd) {
11409         case SIOCGMIIPHY:
11410                 data->phy_id = tp->phy_addr;
11411
11412                 /* fallthru */
11413         case SIOCGMIIREG: {
11414                 u32 mii_regval;
11415
11416                 if (tp->phy_flags & TG3_PHYFLG_PHY_SERDES)
11417                         break;                  /* We have no PHY */
11418
11419                 if (!netif_running(dev))
11420                         return -EAGAIN;
11421
11422                 spin_lock_bh(&tp->lock);
11423                 err = tg3_readphy(tp, data->reg_num & 0x1f, &mii_regval);
11424                 spin_unlock_bh(&tp->lock);
11425
11426                 data->val_out = mii_regval;
11427
11428                 return err;
11429         }
11430
11431         case SIOCSMIIREG:
11432                 if (tp->phy_flags & TG3_PHYFLG_PHY_SERDES)
11433                         break;                  /* We have no PHY */
11434
11435                 if (!netif_running(dev))
11436                         return -EAGAIN;
11437
11438                 spin_lock_bh(&tp->lock);
11439                 err = tg3_writephy(tp, data->reg_num & 0x1f, data->val_in);
11440                 spin_unlock_bh(&tp->lock);
11441
11442                 return err;
11443
11444         default:
11445                 /* do nothing */
11446                 break;
11447         }
11448         return -EOPNOTSUPP;
11449 }
11450
11451 static int tg3_get_coalesce(struct net_device *dev, struct ethtool_coalesce *ec)
11452 {
11453         struct tg3 *tp = netdev_priv(dev);
11454
11455         memcpy(ec, &tp->coal, sizeof(*ec));
11456         return 0;
11457 }
11458
11459 static int tg3_set_coalesce(struct net_device *dev, struct ethtool_coalesce *ec)
11460 {
11461         struct tg3 *tp = netdev_priv(dev);
11462         u32 max_rxcoal_tick_int = 0, max_txcoal_tick_int = 0;
11463         u32 max_stat_coal_ticks = 0, min_stat_coal_ticks = 0;
11464
11465         if (!tg3_flag(tp, 5705_PLUS)) {
11466                 max_rxcoal_tick_int = MAX_RXCOAL_TICK_INT;
11467                 max_txcoal_tick_int = MAX_TXCOAL_TICK_INT;
11468                 max_stat_coal_ticks = MAX_STAT_COAL_TICKS;
11469                 min_stat_coal_ticks = MIN_STAT_COAL_TICKS;
11470         }
11471
11472         if ((ec->rx_coalesce_usecs > MAX_RXCOL_TICKS) ||
11473             (ec->tx_coalesce_usecs > MAX_TXCOL_TICKS) ||
11474             (ec->rx_max_coalesced_frames > MAX_RXMAX_FRAMES) ||
11475             (ec->tx_max_coalesced_frames > MAX_TXMAX_FRAMES) ||
11476             (ec->rx_coalesce_usecs_irq > max_rxcoal_tick_int) ||
11477             (ec->tx_coalesce_usecs_irq > max_txcoal_tick_int) ||
11478             (ec->rx_max_coalesced_frames_irq > MAX_RXCOAL_MAXF_INT) ||
11479             (ec->tx_max_coalesced_frames_irq > MAX_TXCOAL_MAXF_INT) ||
11480             (ec->stats_block_coalesce_usecs > max_stat_coal_ticks) ||
11481             (ec->stats_block_coalesce_usecs < min_stat_coal_ticks))
11482                 return -EINVAL;
11483
11484         /* No rx interrupts will be generated if both are zero */
11485         if ((ec->rx_coalesce_usecs == 0) &&
11486             (ec->rx_max_coalesced_frames == 0))
11487                 return -EINVAL;
11488
11489         /* No tx interrupts will be generated if both are zero */
11490         if ((ec->tx_coalesce_usecs == 0) &&
11491             (ec->tx_max_coalesced_frames == 0))
11492                 return -EINVAL;
11493
11494         /* Only copy relevant parameters, ignore all others. */
11495         tp->coal.rx_coalesce_usecs = ec->rx_coalesce_usecs;
11496         tp->coal.tx_coalesce_usecs = ec->tx_coalesce_usecs;
11497         tp->coal.rx_max_coalesced_frames = ec->rx_max_coalesced_frames;
11498         tp->coal.tx_max_coalesced_frames = ec->tx_max_coalesced_frames;
11499         tp->coal.rx_coalesce_usecs_irq = ec->rx_coalesce_usecs_irq;
11500         tp->coal.tx_coalesce_usecs_irq = ec->tx_coalesce_usecs_irq;
11501         tp->coal.rx_max_coalesced_frames_irq = ec->rx_max_coalesced_frames_irq;
11502         tp->coal.tx_max_coalesced_frames_irq = ec->tx_max_coalesced_frames_irq;
11503         tp->coal.stats_block_coalesce_usecs = ec->stats_block_coalesce_usecs;
11504
11505         if (netif_running(dev)) {
11506                 tg3_full_lock(tp, 0);
11507                 __tg3_set_coalesce(tp, &tp->coal);
11508                 tg3_full_unlock(tp);
11509         }
11510         return 0;
11511 }
11512
11513 static const struct ethtool_ops tg3_ethtool_ops = {
11514         .get_settings           = tg3_get_settings,
11515         .set_settings           = tg3_set_settings,
11516         .get_drvinfo            = tg3_get_drvinfo,
11517         .get_regs_len           = tg3_get_regs_len,
11518         .get_regs               = tg3_get_regs,
11519         .get_wol                = tg3_get_wol,
11520         .set_wol                = tg3_set_wol,
11521         .get_msglevel           = tg3_get_msglevel,
11522         .set_msglevel           = tg3_set_msglevel,
11523         .nway_reset             = tg3_nway_reset,
11524         .get_link               = ethtool_op_get_link,
11525         .get_eeprom_len         = tg3_get_eeprom_len,
11526         .get_eeprom             = tg3_get_eeprom,
11527         .set_eeprom             = tg3_set_eeprom,
11528         .get_ringparam          = tg3_get_ringparam,
11529         .set_ringparam          = tg3_set_ringparam,
11530         .get_pauseparam         = tg3_get_pauseparam,
11531         .set_pauseparam         = tg3_set_pauseparam,
11532         .self_test              = tg3_self_test,
11533         .get_strings            = tg3_get_strings,
11534         .set_phys_id            = tg3_set_phys_id,
11535         .get_ethtool_stats      = tg3_get_ethtool_stats,
11536         .get_coalesce           = tg3_get_coalesce,
11537         .set_coalesce           = tg3_set_coalesce,
11538         .get_sset_count         = tg3_get_sset_count,
11539 };
11540
11541 static void __devinit tg3_get_eeprom_size(struct tg3 *tp)
11542 {
11543         u32 cursize, val, magic;
11544
11545         tp->nvram_size = EEPROM_CHIP_SIZE;
11546
11547         if (tg3_nvram_read(tp, 0, &magic) != 0)
11548                 return;
11549
11550         if ((magic != TG3_EEPROM_MAGIC) &&
11551             ((magic & TG3_EEPROM_MAGIC_FW_MSK) != TG3_EEPROM_MAGIC_FW) &&
11552             ((magic & TG3_EEPROM_MAGIC_HW_MSK) != TG3_EEPROM_MAGIC_HW))
11553                 return;
11554
11555         /*
11556          * Size the chip by reading offsets at increasing powers of two.
11557          * When we encounter our validation signature, we know the addressing
11558          * has wrapped around, and thus have our chip size.
11559          */
11560         cursize = 0x10;
11561
11562         while (cursize < tp->nvram_size) {
11563                 if (tg3_nvram_read(tp, cursize, &val) != 0)
11564                         return;
11565
11566                 if (val == magic)
11567                         break;
11568
11569                 cursize <<= 1;
11570         }
11571
11572         tp->nvram_size = cursize;
11573 }
11574
11575 static void __devinit tg3_get_nvram_size(struct tg3 *tp)
11576 {
11577         u32 val;
11578
11579         if (tg3_flag(tp, NO_NVRAM) || tg3_nvram_read(tp, 0, &val) != 0)
11580                 return;
11581
11582         /* Selfboot format */
11583         if (val != TG3_EEPROM_MAGIC) {
11584                 tg3_get_eeprom_size(tp);
11585                 return;
11586         }
11587
11588         if (tg3_nvram_read(tp, 0xf0, &val) == 0) {
11589                 if (val != 0) {
11590                         /* This is confusing.  We want to operate on the
11591                          * 16-bit value at offset 0xf2.  The tg3_nvram_read()
11592                          * call will read from NVRAM and byteswap the data
11593                          * according to the byteswapping settings for all
11594                          * other register accesses.  This ensures the data we
11595                          * want will always reside in the lower 16-bits.
11596                          * However, the data in NVRAM is in LE format, which
11597                          * means the data from the NVRAM read will always be
11598                          * opposite the endianness of the CPU.  The 16-bit
11599                          * byteswap then brings the data to CPU endianness.
11600                          */
11601                         tp->nvram_size = swab16((u16)(val & 0x0000ffff)) * 1024;
11602                         return;
11603                 }
11604         }
11605         tp->nvram_size = TG3_NVRAM_SIZE_512KB;
11606 }
11607
11608 static void __devinit tg3_get_nvram_info(struct tg3 *tp)
11609 {
11610         u32 nvcfg1;
11611
11612         nvcfg1 = tr32(NVRAM_CFG1);
11613         if (nvcfg1 & NVRAM_CFG1_FLASHIF_ENAB) {
11614                 tg3_flag_set(tp, FLASH);
11615         } else {
11616                 nvcfg1 &= ~NVRAM_CFG1_COMPAT_BYPASS;
11617                 tw32(NVRAM_CFG1, nvcfg1);
11618         }
11619
11620         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5750) ||
11621             tg3_flag(tp, 5780_CLASS)) {
11622                 switch (nvcfg1 & NVRAM_CFG1_VENDOR_MASK) {
11623                 case FLASH_VENDOR_ATMEL_FLASH_BUFFERED:
11624                         tp->nvram_jedecnum = JEDEC_ATMEL;
11625                         tp->nvram_pagesize = ATMEL_AT45DB0X1B_PAGE_SIZE;
11626                         tg3_flag_set(tp, NVRAM_BUFFERED);
11627                         break;
11628                 case FLASH_VENDOR_ATMEL_FLASH_UNBUFFERED:
11629                         tp->nvram_jedecnum = JEDEC_ATMEL;
11630                         tp->nvram_pagesize = ATMEL_AT25F512_PAGE_SIZE;
11631                         break;
11632                 case FLASH_VENDOR_ATMEL_EEPROM:
11633                         tp->nvram_jedecnum = JEDEC_ATMEL;
11634                         tp->nvram_pagesize = ATMEL_AT24C512_CHIP_SIZE;
11635                         tg3_flag_set(tp, NVRAM_BUFFERED);
11636                         break;
11637                 case FLASH_VENDOR_ST:
11638                         tp->nvram_jedecnum = JEDEC_ST;
11639                         tp->nvram_pagesize = ST_M45PEX0_PAGE_SIZE;
11640                         tg3_flag_set(tp, NVRAM_BUFFERED);
11641                         break;
11642                 case FLASH_VENDOR_SAIFUN:
11643                         tp->nvram_jedecnum = JEDEC_SAIFUN;
11644                         tp->nvram_pagesize = SAIFUN_SA25F0XX_PAGE_SIZE;
11645                         break;
11646                 case FLASH_VENDOR_SST_SMALL:
11647                 case FLASH_VENDOR_SST_LARGE:
11648                         tp->nvram_jedecnum = JEDEC_SST;
11649                         tp->nvram_pagesize = SST_25VF0X0_PAGE_SIZE;
11650                         break;
11651                 }
11652         } else {
11653                 tp->nvram_jedecnum = JEDEC_ATMEL;
11654                 tp->nvram_pagesize = ATMEL_AT45DB0X1B_PAGE_SIZE;
11655                 tg3_flag_set(tp, NVRAM_BUFFERED);
11656         }
11657 }
11658
11659 static void __devinit tg3_nvram_get_pagesize(struct tg3 *tp, u32 nvmcfg1)
11660 {
11661         switch (nvmcfg1 & NVRAM_CFG1_5752PAGE_SIZE_MASK) {
11662         case FLASH_5752PAGE_SIZE_256:
11663                 tp->nvram_pagesize = 256;
11664                 break;
11665         case FLASH_5752PAGE_SIZE_512:
11666                 tp->nvram_pagesize = 512;
11667                 break;
11668         case FLASH_5752PAGE_SIZE_1K:
11669                 tp->nvram_pagesize = 1024;
11670                 break;
11671         case FLASH_5752PAGE_SIZE_2K:
11672                 tp->nvram_pagesize = 2048;
11673                 break;
11674         case FLASH_5752PAGE_SIZE_4K:
11675                 tp->nvram_pagesize = 4096;
11676                 break;
11677         case FLASH_5752PAGE_SIZE_264:
11678                 tp->nvram_pagesize = 264;
11679                 break;
11680         case FLASH_5752PAGE_SIZE_528:
11681                 tp->nvram_pagesize = 528;
11682                 break;
11683         }
11684 }
11685
11686 static void __devinit tg3_get_5752_nvram_info(struct tg3 *tp)
11687 {
11688         u32 nvcfg1;
11689
11690         nvcfg1 = tr32(NVRAM_CFG1);
11691
11692         /* NVRAM protection for TPM */
11693         if (nvcfg1 & (1 << 27))
11694                 tg3_flag_set(tp, PROTECTED_NVRAM);
11695
11696         switch (nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK) {
11697         case FLASH_5752VENDOR_ATMEL_EEPROM_64KHZ:
11698         case FLASH_5752VENDOR_ATMEL_EEPROM_376KHZ:
11699                 tp->nvram_jedecnum = JEDEC_ATMEL;
11700                 tg3_flag_set(tp, NVRAM_BUFFERED);
11701                 break;
11702         case FLASH_5752VENDOR_ATMEL_FLASH_BUFFERED:
11703                 tp->nvram_jedecnum = JEDEC_ATMEL;
11704                 tg3_flag_set(tp, NVRAM_BUFFERED);
11705                 tg3_flag_set(tp, FLASH);
11706                 break;
11707         case FLASH_5752VENDOR_ST_M45PE10:
11708         case FLASH_5752VENDOR_ST_M45PE20:
11709         case FLASH_5752VENDOR_ST_M45PE40:
11710                 tp->nvram_jedecnum = JEDEC_ST;
11711                 tg3_flag_set(tp, NVRAM_BUFFERED);
11712                 tg3_flag_set(tp, FLASH);
11713                 break;
11714         }
11715
11716         if (tg3_flag(tp, FLASH)) {
11717                 tg3_nvram_get_pagesize(tp, nvcfg1);
11718         } else {
11719                 /* For eeprom, set pagesize to maximum eeprom size */
11720                 tp->nvram_pagesize = ATMEL_AT24C512_CHIP_SIZE;
11721
11722                 nvcfg1 &= ~NVRAM_CFG1_COMPAT_BYPASS;
11723                 tw32(NVRAM_CFG1, nvcfg1);
11724         }
11725 }
11726
11727 static void __devinit tg3_get_5755_nvram_info(struct tg3 *tp)
11728 {
11729         u32 nvcfg1, protect = 0;
11730
11731         nvcfg1 = tr32(NVRAM_CFG1);
11732
11733         /* NVRAM protection for TPM */
11734         if (nvcfg1 & (1 << 27)) {
11735                 tg3_flag_set(tp, PROTECTED_NVRAM);
11736                 protect = 1;
11737         }
11738
11739         nvcfg1 &= NVRAM_CFG1_5752VENDOR_MASK;
11740         switch (nvcfg1) {
11741         case FLASH_5755VENDOR_ATMEL_FLASH_1:
11742         case FLASH_5755VENDOR_ATMEL_FLASH_2:
11743         case FLASH_5755VENDOR_ATMEL_FLASH_3:
11744         case FLASH_5755VENDOR_ATMEL_FLASH_5:
11745                 tp->nvram_jedecnum = JEDEC_ATMEL;
11746                 tg3_flag_set(tp, NVRAM_BUFFERED);
11747                 tg3_flag_set(tp, FLASH);
11748                 tp->nvram_pagesize = 264;
11749                 if (nvcfg1 == FLASH_5755VENDOR_ATMEL_FLASH_1 ||
11750                     nvcfg1 == FLASH_5755VENDOR_ATMEL_FLASH_5)
11751                         tp->nvram_size = (protect ? 0x3e200 :
11752                                           TG3_NVRAM_SIZE_512KB);
11753                 else if (nvcfg1 == FLASH_5755VENDOR_ATMEL_FLASH_2)
11754                         tp->nvram_size = (protect ? 0x1f200 :
11755                                           TG3_NVRAM_SIZE_256KB);
11756                 else
11757                         tp->nvram_size = (protect ? 0x1f200 :
11758                                           TG3_NVRAM_SIZE_128KB);
11759                 break;
11760         case FLASH_5752VENDOR_ST_M45PE10:
11761         case FLASH_5752VENDOR_ST_M45PE20:
11762         case FLASH_5752VENDOR_ST_M45PE40:
11763                 tp->nvram_jedecnum = JEDEC_ST;
11764                 tg3_flag_set(tp, NVRAM_BUFFERED);
11765                 tg3_flag_set(tp, FLASH);
11766                 tp->nvram_pagesize = 256;
11767                 if (nvcfg1 == FLASH_5752VENDOR_ST_M45PE10)
11768                         tp->nvram_size = (protect ?
11769                                           TG3_NVRAM_SIZE_64KB :
11770                                           TG3_NVRAM_SIZE_128KB);
11771                 else if (nvcfg1 == FLASH_5752VENDOR_ST_M45PE20)
11772                         tp->nvram_size = (protect ?
11773                                           TG3_NVRAM_SIZE_64KB :
11774                                           TG3_NVRAM_SIZE_256KB);
11775                 else
11776                         tp->nvram_size = (protect ?
11777                                           TG3_NVRAM_SIZE_128KB :
11778                                           TG3_NVRAM_SIZE_512KB);
11779                 break;
11780         }
11781 }
11782
11783 static void __devinit tg3_get_5787_nvram_info(struct tg3 *tp)
11784 {
11785         u32 nvcfg1;
11786
11787         nvcfg1 = tr32(NVRAM_CFG1);
11788
11789         switch (nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK) {
11790         case FLASH_5787VENDOR_ATMEL_EEPROM_64KHZ:
11791         case FLASH_5787VENDOR_ATMEL_EEPROM_376KHZ:
11792         case FLASH_5787VENDOR_MICRO_EEPROM_64KHZ:
11793         case FLASH_5787VENDOR_MICRO_EEPROM_376KHZ:
11794                 tp->nvram_jedecnum = JEDEC_ATMEL;
11795                 tg3_flag_set(tp, NVRAM_BUFFERED);
11796                 tp->nvram_pagesize = ATMEL_AT24C512_CHIP_SIZE;
11797
11798                 nvcfg1 &= ~NVRAM_CFG1_COMPAT_BYPASS;
11799                 tw32(NVRAM_CFG1, nvcfg1);
11800                 break;
11801         case FLASH_5752VENDOR_ATMEL_FLASH_BUFFERED:
11802         case FLASH_5755VENDOR_ATMEL_FLASH_1:
11803         case FLASH_5755VENDOR_ATMEL_FLASH_2:
11804         case FLASH_5755VENDOR_ATMEL_FLASH_3:
11805                 tp->nvram_jedecnum = JEDEC_ATMEL;
11806                 tg3_flag_set(tp, NVRAM_BUFFERED);
11807                 tg3_flag_set(tp, FLASH);
11808                 tp->nvram_pagesize = 264;
11809                 break;
11810         case FLASH_5752VENDOR_ST_M45PE10:
11811         case FLASH_5752VENDOR_ST_M45PE20:
11812         case FLASH_5752VENDOR_ST_M45PE40:
11813                 tp->nvram_jedecnum = JEDEC_ST;
11814                 tg3_flag_set(tp, NVRAM_BUFFERED);
11815                 tg3_flag_set(tp, FLASH);
11816                 tp->nvram_pagesize = 256;
11817                 break;
11818         }
11819 }
11820
11821 static void __devinit tg3_get_5761_nvram_info(struct tg3 *tp)
11822 {
11823         u32 nvcfg1, protect = 0;
11824
11825         nvcfg1 = tr32(NVRAM_CFG1);
11826
11827         /* NVRAM protection for TPM */
11828         if (nvcfg1 & (1 << 27)) {
11829                 tg3_flag_set(tp, PROTECTED_NVRAM);
11830                 protect = 1;
11831         }
11832
11833         nvcfg1 &= NVRAM_CFG1_5752VENDOR_MASK;
11834         switch (nvcfg1) {
11835         case FLASH_5761VENDOR_ATMEL_ADB021D:
11836         case FLASH_5761VENDOR_ATMEL_ADB041D:
11837         case FLASH_5761VENDOR_ATMEL_ADB081D:
11838         case FLASH_5761VENDOR_ATMEL_ADB161D:
11839         case FLASH_5761VENDOR_ATMEL_MDB021D:
11840         case FLASH_5761VENDOR_ATMEL_MDB041D:
11841         case FLASH_5761VENDOR_ATMEL_MDB081D:
11842         case FLASH_5761VENDOR_ATMEL_MDB161D:
11843                 tp->nvram_jedecnum = JEDEC_ATMEL;
11844                 tg3_flag_set(tp, NVRAM_BUFFERED);
11845                 tg3_flag_set(tp, FLASH);
11846                 tg3_flag_set(tp, NO_NVRAM_ADDR_TRANS);
11847                 tp->nvram_pagesize = 256;
11848                 break;
11849         case FLASH_5761VENDOR_ST_A_M45PE20:
11850         case FLASH_5761VENDOR_ST_A_M45PE40:
11851         case FLASH_5761VENDOR_ST_A_M45PE80:
11852         case FLASH_5761VENDOR_ST_A_M45PE16:
11853         case FLASH_5761VENDOR_ST_M_M45PE20:
11854         case FLASH_5761VENDOR_ST_M_M45PE40:
11855         case FLASH_5761VENDOR_ST_M_M45PE80:
11856         case FLASH_5761VENDOR_ST_M_M45PE16:
11857                 tp->nvram_jedecnum = JEDEC_ST;
11858                 tg3_flag_set(tp, NVRAM_BUFFERED);
11859                 tg3_flag_set(tp, FLASH);
11860                 tp->nvram_pagesize = 256;
11861                 break;
11862         }
11863
11864         if (protect) {
11865                 tp->nvram_size = tr32(NVRAM_ADDR_LOCKOUT);
11866         } else {
11867                 switch (nvcfg1) {
11868                 case FLASH_5761VENDOR_ATMEL_ADB161D:
11869                 case FLASH_5761VENDOR_ATMEL_MDB161D:
11870                 case FLASH_5761VENDOR_ST_A_M45PE16:
11871                 case FLASH_5761VENDOR_ST_M_M45PE16:
11872                         tp->nvram_size = TG3_NVRAM_SIZE_2MB;
11873                         break;
11874                 case FLASH_5761VENDOR_ATMEL_ADB081D:
11875                 case FLASH_5761VENDOR_ATMEL_MDB081D:
11876                 case FLASH_5761VENDOR_ST_A_M45PE80:
11877                 case FLASH_5761VENDOR_ST_M_M45PE80:
11878                         tp->nvram_size = TG3_NVRAM_SIZE_1MB;
11879                         break;
11880                 case FLASH_5761VENDOR_ATMEL_ADB041D:
11881                 case FLASH_5761VENDOR_ATMEL_MDB041D:
11882                 case FLASH_5761VENDOR_ST_A_M45PE40:
11883                 case FLASH_5761VENDOR_ST_M_M45PE40:
11884                         tp->nvram_size = TG3_NVRAM_SIZE_512KB;
11885                         break;
11886                 case FLASH_5761VENDOR_ATMEL_ADB021D:
11887                 case FLASH_5761VENDOR_ATMEL_MDB021D:
11888                 case FLASH_5761VENDOR_ST_A_M45PE20:
11889                 case FLASH_5761VENDOR_ST_M_M45PE20:
11890                         tp->nvram_size = TG3_NVRAM_SIZE_256KB;
11891                         break;
11892                 }
11893         }
11894 }
11895
11896 static void __devinit tg3_get_5906_nvram_info(struct tg3 *tp)
11897 {
11898         tp->nvram_jedecnum = JEDEC_ATMEL;
11899         tg3_flag_set(tp, NVRAM_BUFFERED);
11900         tp->nvram_pagesize = ATMEL_AT24C512_CHIP_SIZE;
11901 }
11902
11903 static void __devinit tg3_get_57780_nvram_info(struct tg3 *tp)
11904 {
11905         u32 nvcfg1;
11906
11907         nvcfg1 = tr32(NVRAM_CFG1);
11908
11909         switch (nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK) {
11910         case FLASH_5787VENDOR_ATMEL_EEPROM_376KHZ:
11911         case FLASH_5787VENDOR_MICRO_EEPROM_376KHZ:
11912                 tp->nvram_jedecnum = JEDEC_ATMEL;
11913                 tg3_flag_set(tp, NVRAM_BUFFERED);
11914                 tp->nvram_pagesize = ATMEL_AT24C512_CHIP_SIZE;
11915
11916                 nvcfg1 &= ~NVRAM_CFG1_COMPAT_BYPASS;
11917                 tw32(NVRAM_CFG1, nvcfg1);
11918                 return;
11919         case FLASH_5752VENDOR_ATMEL_FLASH_BUFFERED:
11920         case FLASH_57780VENDOR_ATMEL_AT45DB011D:
11921         case FLASH_57780VENDOR_ATMEL_AT45DB011B:
11922         case FLASH_57780VENDOR_ATMEL_AT45DB021D:
11923         case FLASH_57780VENDOR_ATMEL_AT45DB021B:
11924         case FLASH_57780VENDOR_ATMEL_AT45DB041D:
11925         case FLASH_57780VENDOR_ATMEL_AT45DB041B:
11926                 tp->nvram_jedecnum = JEDEC_ATMEL;
11927                 tg3_flag_set(tp, NVRAM_BUFFERED);
11928                 tg3_flag_set(tp, FLASH);
11929
11930                 switch (nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK) {
11931                 case FLASH_5752VENDOR_ATMEL_FLASH_BUFFERED:
11932                 case FLASH_57780VENDOR_ATMEL_AT45DB011D:
11933                 case FLASH_57780VENDOR_ATMEL_AT45DB011B:
11934                         tp->nvram_size = TG3_NVRAM_SIZE_128KB;
11935                         break;
11936                 case FLASH_57780VENDOR_ATMEL_AT45DB021D:
11937                 case FLASH_57780VENDOR_ATMEL_AT45DB021B:
11938                         tp->nvram_size = TG3_NVRAM_SIZE_256KB;
11939                         break;
11940                 case FLASH_57780VENDOR_ATMEL_AT45DB041D:
11941                 case FLASH_57780VENDOR_ATMEL_AT45DB041B:
11942                         tp->nvram_size = TG3_NVRAM_SIZE_512KB;
11943                         break;
11944                 }
11945                 break;
11946         case FLASH_5752VENDOR_ST_M45PE10:
11947         case FLASH_5752VENDOR_ST_M45PE20:
11948         case FLASH_5752VENDOR_ST_M45PE40:
11949                 tp->nvram_jedecnum = JEDEC_ST;
11950                 tg3_flag_set(tp, NVRAM_BUFFERED);
11951                 tg3_flag_set(tp, FLASH);
11952
11953                 switch (nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK) {
11954                 case FLASH_5752VENDOR_ST_M45PE10:
11955                         tp->nvram_size = TG3_NVRAM_SIZE_128KB;
11956                         break;
11957                 case FLASH_5752VENDOR_ST_M45PE20:
11958                         tp->nvram_size = TG3_NVRAM_SIZE_256KB;
11959                         break;
11960                 case FLASH_5752VENDOR_ST_M45PE40:
11961                         tp->nvram_size = TG3_NVRAM_SIZE_512KB;
11962                         break;
11963                 }
11964                 break;
11965         default:
11966                 tg3_flag_set(tp, NO_NVRAM);
11967                 return;
11968         }
11969
11970         tg3_nvram_get_pagesize(tp, nvcfg1);
11971         if (tp->nvram_pagesize != 264 && tp->nvram_pagesize != 528)
11972                 tg3_flag_set(tp, NO_NVRAM_ADDR_TRANS);
11973 }
11974
11975
11976 static void __devinit tg3_get_5717_nvram_info(struct tg3 *tp)
11977 {
11978         u32 nvcfg1;
11979
11980         nvcfg1 = tr32(NVRAM_CFG1);
11981
11982         switch (nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK) {
11983         case FLASH_5717VENDOR_ATMEL_EEPROM:
11984         case FLASH_5717VENDOR_MICRO_EEPROM:
11985                 tp->nvram_jedecnum = JEDEC_ATMEL;
11986                 tg3_flag_set(tp, NVRAM_BUFFERED);
11987                 tp->nvram_pagesize = ATMEL_AT24C512_CHIP_SIZE;
11988
11989                 nvcfg1 &= ~NVRAM_CFG1_COMPAT_BYPASS;
11990                 tw32(NVRAM_CFG1, nvcfg1);
11991                 return;
11992         case FLASH_5717VENDOR_ATMEL_MDB011D:
11993         case FLASH_5717VENDOR_ATMEL_ADB011B:
11994         case FLASH_5717VENDOR_ATMEL_ADB011D:
11995         case FLASH_5717VENDOR_ATMEL_MDB021D:
11996         case FLASH_5717VENDOR_ATMEL_ADB021B:
11997         case FLASH_5717VENDOR_ATMEL_ADB021D:
11998         case FLASH_5717VENDOR_ATMEL_45USPT:
11999                 tp->nvram_jedecnum = JEDEC_ATMEL;
12000                 tg3_flag_set(tp, NVRAM_BUFFERED);
12001                 tg3_flag_set(tp, FLASH);
12002
12003                 switch (nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK) {
12004                 case FLASH_5717VENDOR_ATMEL_MDB021D:
12005                         /* Detect size with tg3_nvram_get_size() */
12006                         break;
12007                 case FLASH_5717VENDOR_ATMEL_ADB021B:
12008                 case FLASH_5717VENDOR_ATMEL_ADB021D:
12009                         tp->nvram_size = TG3_NVRAM_SIZE_256KB;
12010                         break;
12011                 default:
12012                         tp->nvram_size = TG3_NVRAM_SIZE_128KB;
12013                         break;
12014                 }
12015                 break;
12016         case FLASH_5717VENDOR_ST_M_M25PE10:
12017         case FLASH_5717VENDOR_ST_A_M25PE10:
12018         case FLASH_5717VENDOR_ST_M_M45PE10:
12019         case FLASH_5717VENDOR_ST_A_M45PE10:
12020         case FLASH_5717VENDOR_ST_M_M25PE20:
12021         case FLASH_5717VENDOR_ST_A_M25PE20:
12022         case FLASH_5717VENDOR_ST_M_M45PE20:
12023         case FLASH_5717VENDOR_ST_A_M45PE20:
12024         case FLASH_5717VENDOR_ST_25USPT:
12025         case FLASH_5717VENDOR_ST_45USPT:
12026                 tp->nvram_jedecnum = JEDEC_ST;
12027                 tg3_flag_set(tp, NVRAM_BUFFERED);
12028                 tg3_flag_set(tp, FLASH);
12029
12030                 switch (nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK) {
12031                 case FLASH_5717VENDOR_ST_M_M25PE20:
12032                 case FLASH_5717VENDOR_ST_M_M45PE20:
12033                         /* Detect size with tg3_nvram_get_size() */
12034                         break;
12035                 case FLASH_5717VENDOR_ST_A_M25PE20:
12036                 case FLASH_5717VENDOR_ST_A_M45PE20:
12037                         tp->nvram_size = TG3_NVRAM_SIZE_256KB;
12038                         break;
12039                 default:
12040                         tp->nvram_size = TG3_NVRAM_SIZE_128KB;
12041                         break;
12042                 }
12043                 break;
12044         default:
12045                 tg3_flag_set(tp, NO_NVRAM);
12046                 return;
12047         }
12048
12049         tg3_nvram_get_pagesize(tp, nvcfg1);
12050         if (tp->nvram_pagesize != 264 && tp->nvram_pagesize != 528)
12051                 tg3_flag_set(tp, NO_NVRAM_ADDR_TRANS);
12052 }
12053
12054 static void __devinit tg3_get_5720_nvram_info(struct tg3 *tp)
12055 {
12056         u32 nvcfg1, nvmpinstrp;
12057
12058         nvcfg1 = tr32(NVRAM_CFG1);
12059         nvmpinstrp = nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK;
12060
12061         switch (nvmpinstrp) {
12062         case FLASH_5720_EEPROM_HD:
12063         case FLASH_5720_EEPROM_LD:
12064                 tp->nvram_jedecnum = JEDEC_ATMEL;
12065                 tg3_flag_set(tp, NVRAM_BUFFERED);
12066
12067                 nvcfg1 &= ~NVRAM_CFG1_COMPAT_BYPASS;
12068                 tw32(NVRAM_CFG1, nvcfg1);
12069                 if (nvmpinstrp == FLASH_5720_EEPROM_HD)
12070                         tp->nvram_pagesize = ATMEL_AT24C512_CHIP_SIZE;
12071                 else
12072                         tp->nvram_pagesize = ATMEL_AT24C02_CHIP_SIZE;
12073                 return;
12074         case FLASH_5720VENDOR_M_ATMEL_DB011D:
12075         case FLASH_5720VENDOR_A_ATMEL_DB011B:
12076         case FLASH_5720VENDOR_A_ATMEL_DB011D:
12077         case FLASH_5720VENDOR_M_ATMEL_DB021D:
12078         case FLASH_5720VENDOR_A_ATMEL_DB021B:
12079         case FLASH_5720VENDOR_A_ATMEL_DB021D:
12080         case FLASH_5720VENDOR_M_ATMEL_DB041D:
12081         case FLASH_5720VENDOR_A_ATMEL_DB041B:
12082         case FLASH_5720VENDOR_A_ATMEL_DB041D:
12083         case FLASH_5720VENDOR_M_ATMEL_DB081D:
12084         case FLASH_5720VENDOR_A_ATMEL_DB081D:
12085         case FLASH_5720VENDOR_ATMEL_45USPT:
12086                 tp->nvram_jedecnum = JEDEC_ATMEL;
12087                 tg3_flag_set(tp, NVRAM_BUFFERED);
12088                 tg3_flag_set(tp, FLASH);
12089
12090                 switch (nvmpinstrp) {
12091                 case FLASH_5720VENDOR_M_ATMEL_DB021D:
12092                 case FLASH_5720VENDOR_A_ATMEL_DB021B:
12093                 case FLASH_5720VENDOR_A_ATMEL_DB021D:
12094                         tp->nvram_size = TG3_NVRAM_SIZE_256KB;
12095                         break;
12096                 case FLASH_5720VENDOR_M_ATMEL_DB041D:
12097                 case FLASH_5720VENDOR_A_ATMEL_DB041B:
12098                 case FLASH_5720VENDOR_A_ATMEL_DB041D:
12099                         tp->nvram_size = TG3_NVRAM_SIZE_512KB;
12100                         break;
12101                 case FLASH_5720VENDOR_M_ATMEL_DB081D:
12102                 case FLASH_5720VENDOR_A_ATMEL_DB081D:
12103                         tp->nvram_size = TG3_NVRAM_SIZE_1MB;
12104                         break;
12105                 default:
12106                         tp->nvram_size = TG3_NVRAM_SIZE_128KB;
12107                         break;
12108                 }
12109                 break;
12110         case FLASH_5720VENDOR_M_ST_M25PE10:
12111         case FLASH_5720VENDOR_M_ST_M45PE10:
12112         case FLASH_5720VENDOR_A_ST_M25PE10:
12113         case FLASH_5720VENDOR_A_ST_M45PE10:
12114         case FLASH_5720VENDOR_M_ST_M25PE20:
12115         case FLASH_5720VENDOR_M_ST_M45PE20:
12116         case FLASH_5720VENDOR_A_ST_M25PE20:
12117         case FLASH_5720VENDOR_A_ST_M45PE20:
12118         case FLASH_5720VENDOR_M_ST_M25PE40:
12119         case FLASH_5720VENDOR_M_ST_M45PE40:
12120         case FLASH_5720VENDOR_A_ST_M25PE40:
12121         case FLASH_5720VENDOR_A_ST_M45PE40:
12122         case FLASH_5720VENDOR_M_ST_M25PE80:
12123         case FLASH_5720VENDOR_M_ST_M45PE80:
12124         case FLASH_5720VENDOR_A_ST_M25PE80:
12125         case FLASH_5720VENDOR_A_ST_M45PE80:
12126         case FLASH_5720VENDOR_ST_25USPT:
12127         case FLASH_5720VENDOR_ST_45USPT:
12128                 tp->nvram_jedecnum = JEDEC_ST;
12129                 tg3_flag_set(tp, NVRAM_BUFFERED);
12130                 tg3_flag_set(tp, FLASH);
12131
12132                 switch (nvmpinstrp) {
12133                 case FLASH_5720VENDOR_M_ST_M25PE20:
12134                 case FLASH_5720VENDOR_M_ST_M45PE20:
12135                 case FLASH_5720VENDOR_A_ST_M25PE20:
12136                 case FLASH_5720VENDOR_A_ST_M45PE20:
12137                         tp->nvram_size = TG3_NVRAM_SIZE_256KB;
12138                         break;
12139                 case FLASH_5720VENDOR_M_ST_M25PE40:
12140                 case FLASH_5720VENDOR_M_ST_M45PE40:
12141                 case FLASH_5720VENDOR_A_ST_M25PE40:
12142                 case FLASH_5720VENDOR_A_ST_M45PE40:
12143                         tp->nvram_size = TG3_NVRAM_SIZE_512KB;
12144                         break;
12145                 case FLASH_5720VENDOR_M_ST_M25PE80:
12146                 case FLASH_5720VENDOR_M_ST_M45PE80:
12147                 case FLASH_5720VENDOR_A_ST_M25PE80:
12148                 case FLASH_5720VENDOR_A_ST_M45PE80:
12149                         tp->nvram_size = TG3_NVRAM_SIZE_1MB;
12150                         break;
12151                 default:
12152                         tp->nvram_size = TG3_NVRAM_SIZE_128KB;
12153                         break;
12154                 }
12155                 break;
12156         default:
12157                 tg3_flag_set(tp, NO_NVRAM);
12158                 return;
12159         }
12160
12161         tg3_nvram_get_pagesize(tp, nvcfg1);
12162         if (tp->nvram_pagesize != 264 && tp->nvram_pagesize != 528)
12163                 tg3_flag_set(tp, NO_NVRAM_ADDR_TRANS);
12164 }
12165
12166 /* Chips other than 5700/5701 use the NVRAM for fetching info. */
12167 static void __devinit tg3_nvram_init(struct tg3 *tp)
12168 {
12169         tw32_f(GRC_EEPROM_ADDR,
12170              (EEPROM_ADDR_FSM_RESET |
12171               (EEPROM_DEFAULT_CLOCK_PERIOD <<
12172                EEPROM_ADDR_CLKPERD_SHIFT)));
12173
12174         msleep(1);
12175
12176         /* Enable seeprom accesses. */
12177         tw32_f(GRC_LOCAL_CTRL,
12178              tr32(GRC_LOCAL_CTRL) | GRC_LCLCTRL_AUTO_SEEPROM);
12179         udelay(100);
12180
12181         if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700 &&
12182             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5701) {
12183                 tg3_flag_set(tp, NVRAM);
12184
12185                 if (tg3_nvram_lock(tp)) {
12186                         netdev_warn(tp->dev,
12187                                     "Cannot get nvram lock, %s failed\n",
12188                                     __func__);
12189                         return;
12190                 }
12191                 tg3_enable_nvram_access(tp);
12192
12193                 tp->nvram_size = 0;
12194
12195                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5752)
12196                         tg3_get_5752_nvram_info(tp);
12197                 else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755)
12198                         tg3_get_5755_nvram_info(tp);
12199                 else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787 ||
12200                          GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 ||
12201                          GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5785)
12202                         tg3_get_5787_nvram_info(tp);
12203                 else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761)
12204                         tg3_get_5761_nvram_info(tp);
12205                 else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906)
12206                         tg3_get_5906_nvram_info(tp);
12207                 else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_57780 ||
12208                          GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_57765)
12209                         tg3_get_57780_nvram_info(tp);
12210                 else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5717 ||
12211                          GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5719)
12212                         tg3_get_5717_nvram_info(tp);
12213                 else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5720)
12214                         tg3_get_5720_nvram_info(tp);
12215                 else
12216                         tg3_get_nvram_info(tp);
12217
12218                 if (tp->nvram_size == 0)
12219                         tg3_get_nvram_size(tp);
12220
12221                 tg3_disable_nvram_access(tp);
12222                 tg3_nvram_unlock(tp);
12223
12224         } else {
12225                 tg3_flag_clear(tp, NVRAM);
12226                 tg3_flag_clear(tp, NVRAM_BUFFERED);
12227
12228                 tg3_get_eeprom_size(tp);
12229         }
12230 }
12231
12232 static int tg3_nvram_write_block_using_eeprom(struct tg3 *tp,
12233                                     u32 offset, u32 len, u8 *buf)
12234 {
12235         int i, j, rc = 0;
12236         u32 val;
12237
12238         for (i = 0; i < len; i += 4) {
12239                 u32 addr;
12240                 __be32 data;
12241
12242                 addr = offset + i;
12243
12244                 memcpy(&data, buf + i, 4);
12245
12246                 /*
12247                  * The SEEPROM interface expects the data to always be opposite
12248                  * the native endian format.  We accomplish this by reversing
12249                  * all the operations that would have been performed on the
12250                  * data from a call to tg3_nvram_read_be32().
12251                  */
12252                 tw32(GRC_EEPROM_DATA, swab32(be32_to_cpu(data)));
12253
12254                 val = tr32(GRC_EEPROM_ADDR);
12255                 tw32(GRC_EEPROM_ADDR, val | EEPROM_ADDR_COMPLETE);
12256
12257                 val &= ~(EEPROM_ADDR_ADDR_MASK | EEPROM_ADDR_DEVID_MASK |
12258                         EEPROM_ADDR_READ);
12259                 tw32(GRC_EEPROM_ADDR, val |
12260                         (0 << EEPROM_ADDR_DEVID_SHIFT) |
12261                         (addr & EEPROM_ADDR_ADDR_MASK) |
12262                         EEPROM_ADDR_START |
12263                         EEPROM_ADDR_WRITE);
12264
12265                 for (j = 0; j < 1000; j++) {
12266                         val = tr32(GRC_EEPROM_ADDR);
12267
12268                         if (val & EEPROM_ADDR_COMPLETE)
12269                                 break;
12270                         msleep(1);
12271                 }
12272                 if (!(val & EEPROM_ADDR_COMPLETE)) {
12273                         rc = -EBUSY;
12274                         break;
12275                 }
12276         }
12277
12278         return rc;
12279 }
12280
12281 /* offset and length are dword aligned */
12282 static int tg3_nvram_write_block_unbuffered(struct tg3 *tp, u32 offset, u32 len,
12283                 u8 *buf)
12284 {
12285         int ret = 0;
12286         u32 pagesize = tp->nvram_pagesize;
12287         u32 pagemask = pagesize - 1;
12288         u32 nvram_cmd;
12289         u8 *tmp;
12290
12291         tmp = kmalloc(pagesize, GFP_KERNEL);
12292         if (tmp == NULL)
12293                 return -ENOMEM;
12294
12295         while (len) {
12296                 int j;
12297                 u32 phy_addr, page_off, size;
12298
12299                 phy_addr = offset & ~pagemask;
12300
12301                 for (j = 0; j < pagesize; j += 4) {
12302                         ret = tg3_nvram_read_be32(tp, phy_addr + j,
12303                                                   (__be32 *) (tmp + j));
12304                         if (ret)
12305                                 break;
12306                 }
12307                 if (ret)
12308                         break;
12309
12310                 page_off = offset & pagemask;
12311                 size = pagesize;
12312                 if (len < size)
12313                         size = len;
12314
12315                 len -= size;
12316
12317                 memcpy(tmp + page_off, buf, size);
12318
12319                 offset = offset + (pagesize - page_off);
12320
12321                 tg3_enable_nvram_access(tp);
12322
12323                 /*
12324                  * Before we can erase the flash page, we need
12325                  * to issue a special "write enable" command.
12326                  */
12327                 nvram_cmd = NVRAM_CMD_WREN | NVRAM_CMD_GO | NVRAM_CMD_DONE;
12328
12329                 if (tg3_nvram_exec_cmd(tp, nvram_cmd))
12330                         break;
12331
12332                 /* Erase the target page */
12333                 tw32(NVRAM_ADDR, phy_addr);
12334
12335                 nvram_cmd = NVRAM_CMD_GO | NVRAM_CMD_DONE | NVRAM_CMD_WR |
12336                         NVRAM_CMD_FIRST | NVRAM_CMD_LAST | NVRAM_CMD_ERASE;
12337
12338                 if (tg3_nvram_exec_cmd(tp, nvram_cmd))
12339                         break;
12340
12341                 /* Issue another write enable to start the write. */
12342                 nvram_cmd = NVRAM_CMD_WREN | NVRAM_CMD_GO | NVRAM_CMD_DONE;
12343
12344                 if (tg3_nvram_exec_cmd(tp, nvram_cmd))
12345                         break;
12346
12347                 for (j = 0; j < pagesize; j += 4) {
12348                         __be32 data;
12349
12350                         data = *((__be32 *) (tmp + j));
12351
12352                         tw32(NVRAM_WRDATA, be32_to_cpu(data));
12353
12354                         tw32(NVRAM_ADDR, phy_addr + j);
12355
12356                         nvram_cmd = NVRAM_CMD_GO | NVRAM_CMD_DONE |
12357                                 NVRAM_CMD_WR;
12358
12359                         if (j == 0)
12360                                 nvram_cmd |= NVRAM_CMD_FIRST;
12361                         else if (j == (pagesize - 4))
12362                                 nvram_cmd |= NVRAM_CMD_LAST;
12363
12364                         if ((ret = tg3_nvram_exec_cmd(tp, nvram_cmd)))
12365                                 break;
12366                 }
12367                 if (ret)
12368                         break;
12369         }
12370
12371         nvram_cmd = NVRAM_CMD_WRDI | NVRAM_CMD_GO | NVRAM_CMD_DONE;
12372         tg3_nvram_exec_cmd(tp, nvram_cmd);
12373
12374         kfree(tmp);
12375
12376         return ret;
12377 }
12378
12379 /* offset and length are dword aligned */
12380 static int tg3_nvram_write_block_buffered(struct tg3 *tp, u32 offset, u32 len,
12381                 u8 *buf)
12382 {
12383         int i, ret = 0;
12384
12385         for (i = 0; i < len; i += 4, offset += 4) {
12386                 u32 page_off, phy_addr, nvram_cmd;
12387                 __be32 data;
12388
12389                 memcpy(&data, buf + i, 4);
12390                 tw32(NVRAM_WRDATA, be32_to_cpu(data));
12391
12392                 page_off = offset % tp->nvram_pagesize;
12393
12394                 phy_addr = tg3_nvram_phys_addr(tp, offset);
12395
12396                 tw32(NVRAM_ADDR, phy_addr);
12397
12398                 nvram_cmd = NVRAM_CMD_GO | NVRAM_CMD_DONE | NVRAM_CMD_WR;
12399
12400                 if (page_off == 0 || i == 0)
12401                         nvram_cmd |= NVRAM_CMD_FIRST;
12402                 if (page_off == (tp->nvram_pagesize - 4))
12403                         nvram_cmd |= NVRAM_CMD_LAST;
12404
12405                 if (i == (len - 4))
12406                         nvram_cmd |= NVRAM_CMD_LAST;
12407
12408                 if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5752 &&
12409                     !tg3_flag(tp, 5755_PLUS) &&
12410                     (tp->nvram_jedecnum == JEDEC_ST) &&
12411                     (nvram_cmd & NVRAM_CMD_FIRST)) {
12412
12413                         if ((ret = tg3_nvram_exec_cmd(tp,
12414                                 NVRAM_CMD_WREN | NVRAM_CMD_GO |
12415                                 NVRAM_CMD_DONE)))
12416
12417                                 break;
12418                 }
12419                 if (!tg3_flag(tp, FLASH)) {
12420                         /* We always do complete word writes to eeprom. */
12421                         nvram_cmd |= (NVRAM_CMD_FIRST | NVRAM_CMD_LAST);
12422                 }
12423
12424                 if ((ret = tg3_nvram_exec_cmd(tp, nvram_cmd)))
12425                         break;
12426         }
12427         return ret;
12428 }
12429
12430 /* offset and length are dword aligned */
12431 static int tg3_nvram_write_block(struct tg3 *tp, u32 offset, u32 len, u8 *buf)
12432 {
12433         int ret;
12434
12435         if (tg3_flag(tp, EEPROM_WRITE_PROT)) {
12436                 tw32_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl &
12437                        ~GRC_LCLCTRL_GPIO_OUTPUT1);
12438                 udelay(40);
12439         }
12440
12441         if (!tg3_flag(tp, NVRAM)) {
12442                 ret = tg3_nvram_write_block_using_eeprom(tp, offset, len, buf);
12443         } else {
12444                 u32 grc_mode;
12445
12446                 ret = tg3_nvram_lock(tp);
12447                 if (ret)
12448                         return ret;
12449
12450                 tg3_enable_nvram_access(tp);
12451                 if (tg3_flag(tp, 5750_PLUS) && !tg3_flag(tp, PROTECTED_NVRAM))
12452                         tw32(NVRAM_WRITE1, 0x406);
12453
12454                 grc_mode = tr32(GRC_MODE);
12455                 tw32(GRC_MODE, grc_mode | GRC_MODE_NVRAM_WR_ENABLE);
12456
12457                 if (tg3_flag(tp, NVRAM_BUFFERED) || !tg3_flag(tp, FLASH)) {
12458                         ret = tg3_nvram_write_block_buffered(tp, offset, len,
12459                                 buf);
12460                 } else {
12461                         ret = tg3_nvram_write_block_unbuffered(tp, offset, len,
12462                                 buf);
12463                 }
12464
12465                 grc_mode = tr32(GRC_MODE);
12466                 tw32(GRC_MODE, grc_mode & ~GRC_MODE_NVRAM_WR_ENABLE);
12467
12468                 tg3_disable_nvram_access(tp);
12469                 tg3_nvram_unlock(tp);
12470         }
12471
12472         if (tg3_flag(tp, EEPROM_WRITE_PROT)) {
12473                 tw32_f(GRC_LOCAL_CTRL, tp->grc_local_ctrl);
12474                 udelay(40);
12475         }
12476
12477         return ret;
12478 }
12479
12480 struct subsys_tbl_ent {
12481         u16 subsys_vendor, subsys_devid;
12482         u32 phy_id;
12483 };
12484
12485 static struct subsys_tbl_ent subsys_id_to_phy_id[] __devinitdata = {
12486         /* Broadcom boards. */
12487         { TG3PCI_SUBVENDOR_ID_BROADCOM,
12488           TG3PCI_SUBDEVICE_ID_BROADCOM_95700A6, TG3_PHY_ID_BCM5401 },
12489         { TG3PCI_SUBVENDOR_ID_BROADCOM,
12490           TG3PCI_SUBDEVICE_ID_BROADCOM_95701A5, TG3_PHY_ID_BCM5701 },
12491         { TG3PCI_SUBVENDOR_ID_BROADCOM,
12492           TG3PCI_SUBDEVICE_ID_BROADCOM_95700T6, TG3_PHY_ID_BCM8002 },
12493         { TG3PCI_SUBVENDOR_ID_BROADCOM,
12494           TG3PCI_SUBDEVICE_ID_BROADCOM_95700A9, 0 },
12495         { TG3PCI_SUBVENDOR_ID_BROADCOM,
12496           TG3PCI_SUBDEVICE_ID_BROADCOM_95701T1, TG3_PHY_ID_BCM5701 },
12497         { TG3PCI_SUBVENDOR_ID_BROADCOM,
12498           TG3PCI_SUBDEVICE_ID_BROADCOM_95701T8, TG3_PHY_ID_BCM5701 },
12499         { TG3PCI_SUBVENDOR_ID_BROADCOM,
12500           TG3PCI_SUBDEVICE_ID_BROADCOM_95701A7, 0 },
12501         { TG3PCI_SUBVENDOR_ID_BROADCOM,
12502           TG3PCI_SUBDEVICE_ID_BROADCOM_95701A10, TG3_PHY_ID_BCM5701 },
12503         { TG3PCI_SUBVENDOR_ID_BROADCOM,
12504           TG3PCI_SUBDEVICE_ID_BROADCOM_95701A12, TG3_PHY_ID_BCM5701 },
12505         { TG3PCI_SUBVENDOR_ID_BROADCOM,
12506           TG3PCI_SUBDEVICE_ID_BROADCOM_95703AX1, TG3_PHY_ID_BCM5703 },
12507         { TG3PCI_SUBVENDOR_ID_BROADCOM,
12508           TG3PCI_SUBDEVICE_ID_BROADCOM_95703AX2, TG3_PHY_ID_BCM5703 },
12509
12510         /* 3com boards. */
12511         { TG3PCI_SUBVENDOR_ID_3COM,
12512           TG3PCI_SUBDEVICE_ID_3COM_3C996T, TG3_PHY_ID_BCM5401 },
12513         { TG3PCI_SUBVENDOR_ID_3COM,
12514           TG3PCI_SUBDEVICE_ID_3COM_3C996BT, TG3_PHY_ID_BCM5701 },
12515         { TG3PCI_SUBVENDOR_ID_3COM,
12516           TG3PCI_SUBDEVICE_ID_3COM_3C996SX, 0 },
12517         { TG3PCI_SUBVENDOR_ID_3COM,
12518           TG3PCI_SUBDEVICE_ID_3COM_3C1000T, TG3_PHY_ID_BCM5701 },
12519         { TG3PCI_SUBVENDOR_ID_3COM,
12520           TG3PCI_SUBDEVICE_ID_3COM_3C940BR01, TG3_PHY_ID_BCM5701 },
12521
12522         /* DELL boards. */
12523         { TG3PCI_SUBVENDOR_ID_DELL,
12524           TG3PCI_SUBDEVICE_ID_DELL_VIPER, TG3_PHY_ID_BCM5401 },
12525         { TG3PCI_SUBVENDOR_ID_DELL,
12526           TG3PCI_SUBDEVICE_ID_DELL_JAGUAR, TG3_PHY_ID_BCM5401 },
12527         { TG3PCI_SUBVENDOR_ID_DELL,
12528           TG3PCI_SUBDEVICE_ID_DELL_MERLOT, TG3_PHY_ID_BCM5411 },
12529         { TG3PCI_SUBVENDOR_ID_DELL,
12530           TG3PCI_SUBDEVICE_ID_DELL_SLIM_MERLOT, TG3_PHY_ID_BCM5411 },
12531
12532         /* Compaq boards. */
12533         { TG3PCI_SUBVENDOR_ID_COMPAQ,
12534           TG3PCI_SUBDEVICE_ID_COMPAQ_BANSHEE, TG3_PHY_ID_BCM5701 },
12535         { TG3PCI_SUBVENDOR_ID_COMPAQ,
12536           TG3PCI_SUBDEVICE_ID_COMPAQ_BANSHEE_2, TG3_PHY_ID_BCM5701 },
12537         { TG3PCI_SUBVENDOR_ID_COMPAQ,
12538           TG3PCI_SUBDEVICE_ID_COMPAQ_CHANGELING, 0 },
12539         { TG3PCI_SUBVENDOR_ID_COMPAQ,
12540           TG3PCI_SUBDEVICE_ID_COMPAQ_NC7780, TG3_PHY_ID_BCM5701 },
12541         { TG3PCI_SUBVENDOR_ID_COMPAQ,
12542           TG3PCI_SUBDEVICE_ID_COMPAQ_NC7780_2, TG3_PHY_ID_BCM5701 },
12543
12544         /* IBM boards. */
12545         { TG3PCI_SUBVENDOR_ID_IBM,
12546           TG3PCI_SUBDEVICE_ID_IBM_5703SAX2, 0 }
12547 };
12548
12549 static struct subsys_tbl_ent * __devinit tg3_lookup_by_subsys(struct tg3 *tp)
12550 {
12551         int i;
12552
12553         for (i = 0; i < ARRAY_SIZE(subsys_id_to_phy_id); i++) {
12554                 if ((subsys_id_to_phy_id[i].subsys_vendor ==
12555                      tp->pdev->subsystem_vendor) &&
12556                     (subsys_id_to_phy_id[i].subsys_devid ==
12557                      tp->pdev->subsystem_device))
12558                         return &subsys_id_to_phy_id[i];
12559         }
12560         return NULL;
12561 }
12562
12563 static void __devinit tg3_get_eeprom_hw_cfg(struct tg3 *tp)
12564 {
12565         u32 val;
12566         u16 pmcsr;
12567
12568         /* On some early chips the SRAM cannot be accessed in D3hot state,
12569          * so need make sure we're in D0.
12570          */
12571         pci_read_config_word(tp->pdev, tp->pm_cap + PCI_PM_CTRL, &pmcsr);
12572         pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
12573         pci_write_config_word(tp->pdev, tp->pm_cap + PCI_PM_CTRL, pmcsr);
12574         msleep(1);
12575
12576         /* Make sure register accesses (indirect or otherwise)
12577          * will function correctly.
12578          */
12579         pci_write_config_dword(tp->pdev, TG3PCI_MISC_HOST_CTRL,
12580                                tp->misc_host_ctrl);
12581
12582         /* The memory arbiter has to be enabled in order for SRAM accesses
12583          * to succeed.  Normally on powerup the tg3 chip firmware will make
12584          * sure it is enabled, but other entities such as system netboot
12585          * code might disable it.
12586          */
12587         val = tr32(MEMARB_MODE);
12588         tw32(MEMARB_MODE, val | MEMARB_MODE_ENABLE);
12589
12590         tp->phy_id = TG3_PHY_ID_INVALID;
12591         tp->led_ctrl = LED_CTRL_MODE_PHY_1;
12592
12593         /* Assume an onboard device and WOL capable by default.  */
12594         tg3_flag_set(tp, EEPROM_WRITE_PROT);
12595         tg3_flag_set(tp, WOL_CAP);
12596
12597         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
12598                 if (!(tr32(PCIE_TRANSACTION_CFG) & PCIE_TRANS_CFG_LOM)) {
12599                         tg3_flag_clear(tp, EEPROM_WRITE_PROT);
12600                         tg3_flag_set(tp, IS_NIC);
12601                 }
12602                 val = tr32(VCPU_CFGSHDW);
12603                 if (val & VCPU_CFGSHDW_ASPM_DBNC)
12604                         tg3_flag_set(tp, ASPM_WORKAROUND);
12605                 if ((val & VCPU_CFGSHDW_WOL_ENABLE) &&
12606                     (val & VCPU_CFGSHDW_WOL_MAGPKT)) {
12607                         tg3_flag_set(tp, WOL_ENABLE);
12608                         device_set_wakeup_enable(&tp->pdev->dev, true);
12609                 }
12610                 goto done;
12611         }
12612
12613         tg3_read_mem(tp, NIC_SRAM_DATA_SIG, &val);
12614         if (val == NIC_SRAM_DATA_SIG_MAGIC) {
12615                 u32 nic_cfg, led_cfg;
12616                 u32 nic_phy_id, ver, cfg2 = 0, cfg4 = 0, eeprom_phy_id;
12617                 int eeprom_phy_serdes = 0;
12618
12619                 tg3_read_mem(tp, NIC_SRAM_DATA_CFG, &nic_cfg);
12620                 tp->nic_sram_data_cfg = nic_cfg;
12621
12622                 tg3_read_mem(tp, NIC_SRAM_DATA_VER, &ver);
12623                 ver >>= NIC_SRAM_DATA_VER_SHIFT;
12624                 if ((GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700) &&
12625                     (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5701) &&
12626                     (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5703) &&
12627                     (ver > 0) && (ver < 0x100))
12628                         tg3_read_mem(tp, NIC_SRAM_DATA_CFG_2, &cfg2);
12629
12630                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5785)
12631                         tg3_read_mem(tp, NIC_SRAM_DATA_CFG_4, &cfg4);
12632
12633                 if ((nic_cfg & NIC_SRAM_DATA_CFG_PHY_TYPE_MASK) ==
12634                     NIC_SRAM_DATA_CFG_PHY_TYPE_FIBER)
12635                         eeprom_phy_serdes = 1;
12636
12637                 tg3_read_mem(tp, NIC_SRAM_DATA_PHY_ID, &nic_phy_id);
12638                 if (nic_phy_id != 0) {
12639                         u32 id1 = nic_phy_id & NIC_SRAM_DATA_PHY_ID1_MASK;
12640                         u32 id2 = nic_phy_id & NIC_SRAM_DATA_PHY_ID2_MASK;
12641
12642                         eeprom_phy_id  = (id1 >> 16) << 10;
12643                         eeprom_phy_id |= (id2 & 0xfc00) << 16;
12644                         eeprom_phy_id |= (id2 & 0x03ff) <<  0;
12645                 } else
12646                         eeprom_phy_id = 0;
12647
12648                 tp->phy_id = eeprom_phy_id;
12649                 if (eeprom_phy_serdes) {
12650                         if (!tg3_flag(tp, 5705_PLUS))
12651                                 tp->phy_flags |= TG3_PHYFLG_PHY_SERDES;
12652                         else
12653                                 tp->phy_flags |= TG3_PHYFLG_MII_SERDES;
12654                 }
12655
12656                 if (tg3_flag(tp, 5750_PLUS))
12657                         led_cfg = cfg2 & (NIC_SRAM_DATA_CFG_LED_MODE_MASK |
12658                                     SHASTA_EXT_LED_MODE_MASK);
12659                 else
12660                         led_cfg = nic_cfg & NIC_SRAM_DATA_CFG_LED_MODE_MASK;
12661
12662                 switch (led_cfg) {
12663                 default:
12664                 case NIC_SRAM_DATA_CFG_LED_MODE_PHY_1:
12665                         tp->led_ctrl = LED_CTRL_MODE_PHY_1;
12666                         break;
12667
12668                 case NIC_SRAM_DATA_CFG_LED_MODE_PHY_2:
12669                         tp->led_ctrl = LED_CTRL_MODE_PHY_2;
12670                         break;
12671
12672                 case NIC_SRAM_DATA_CFG_LED_MODE_MAC:
12673                         tp->led_ctrl = LED_CTRL_MODE_MAC;
12674
12675                         /* Default to PHY_1_MODE if 0 (MAC_MODE) is
12676                          * read on some older 5700/5701 bootcode.
12677                          */
12678                         if (GET_ASIC_REV(tp->pci_chip_rev_id) ==
12679                             ASIC_REV_5700 ||
12680                             GET_ASIC_REV(tp->pci_chip_rev_id) ==
12681                             ASIC_REV_5701)
12682                                 tp->led_ctrl = LED_CTRL_MODE_PHY_1;
12683
12684                         break;
12685
12686                 case SHASTA_EXT_LED_SHARED:
12687                         tp->led_ctrl = LED_CTRL_MODE_SHARED;
12688                         if (tp->pci_chip_rev_id != CHIPREV_ID_5750_A0 &&
12689                             tp->pci_chip_rev_id != CHIPREV_ID_5750_A1)
12690                                 tp->led_ctrl |= (LED_CTRL_MODE_PHY_1 |
12691                                                  LED_CTRL_MODE_PHY_2);
12692                         break;
12693
12694                 case SHASTA_EXT_LED_MAC:
12695                         tp->led_ctrl = LED_CTRL_MODE_SHASTA_MAC;
12696                         break;
12697
12698                 case SHASTA_EXT_LED_COMBO:
12699                         tp->led_ctrl = LED_CTRL_MODE_COMBO;
12700                         if (tp->pci_chip_rev_id != CHIPREV_ID_5750_A0)
12701                                 tp->led_ctrl |= (LED_CTRL_MODE_PHY_1 |
12702                                                  LED_CTRL_MODE_PHY_2);
12703                         break;
12704
12705                 }
12706
12707                 if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
12708                      GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) &&
12709                     tp->pdev->subsystem_vendor == PCI_VENDOR_ID_DELL)
12710                         tp->led_ctrl = LED_CTRL_MODE_PHY_2;
12711
12712                 if (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5784_AX)
12713                         tp->led_ctrl = LED_CTRL_MODE_PHY_1;
12714
12715                 if (nic_cfg & NIC_SRAM_DATA_CFG_EEPROM_WP) {
12716                         tg3_flag_set(tp, EEPROM_WRITE_PROT);
12717                         if ((tp->pdev->subsystem_vendor ==
12718                              PCI_VENDOR_ID_ARIMA) &&
12719                             (tp->pdev->subsystem_device == 0x205a ||
12720                              tp->pdev->subsystem_device == 0x2063))
12721                                 tg3_flag_clear(tp, EEPROM_WRITE_PROT);
12722                 } else {
12723                         tg3_flag_clear(tp, EEPROM_WRITE_PROT);
12724                         tg3_flag_set(tp, IS_NIC);
12725                 }
12726
12727                 if (nic_cfg & NIC_SRAM_DATA_CFG_ASF_ENABLE) {
12728                         tg3_flag_set(tp, ENABLE_ASF);
12729                         if (tg3_flag(tp, 5750_PLUS))
12730                                 tg3_flag_set(tp, ASF_NEW_HANDSHAKE);
12731                 }
12732
12733                 if ((nic_cfg & NIC_SRAM_DATA_CFG_APE_ENABLE) &&
12734                     tg3_flag(tp, 5750_PLUS))
12735                         tg3_flag_set(tp, ENABLE_APE);
12736
12737                 if (tp->phy_flags & TG3_PHYFLG_ANY_SERDES &&
12738                     !(nic_cfg & NIC_SRAM_DATA_CFG_FIBER_WOL))
12739                         tg3_flag_clear(tp, WOL_CAP);
12740
12741                 if (tg3_flag(tp, WOL_CAP) &&
12742                     (nic_cfg & NIC_SRAM_DATA_CFG_WOL_ENABLE)) {
12743                         tg3_flag_set(tp, WOL_ENABLE);
12744                         device_set_wakeup_enable(&tp->pdev->dev, true);
12745                 }
12746
12747                 if (cfg2 & (1 << 17))
12748                         tp->phy_flags |= TG3_PHYFLG_CAPACITIVE_COUPLING;
12749
12750                 /* serdes signal pre-emphasis in register 0x590 set by */
12751                 /* bootcode if bit 18 is set */
12752                 if (cfg2 & (1 << 18))
12753                         tp->phy_flags |= TG3_PHYFLG_SERDES_PREEMPHASIS;
12754
12755                 if ((tg3_flag(tp, 57765_PLUS) ||
12756                      (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 &&
12757                       GET_CHIP_REV(tp->pci_chip_rev_id) != CHIPREV_5784_AX)) &&
12758                     (cfg2 & NIC_SRAM_DATA_CFG_2_APD_EN))
12759                         tp->phy_flags |= TG3_PHYFLG_ENABLE_APD;
12760
12761                 if (tg3_flag(tp, PCI_EXPRESS) &&
12762                     GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5785 &&
12763                     !tg3_flag(tp, 57765_PLUS)) {
12764                         u32 cfg3;
12765
12766                         tg3_read_mem(tp, NIC_SRAM_DATA_CFG_3, &cfg3);
12767                         if (cfg3 & NIC_SRAM_ASPM_DEBOUNCE)
12768                                 tg3_flag_set(tp, ASPM_WORKAROUND);
12769                 }
12770
12771                 if (cfg4 & NIC_SRAM_RGMII_INBAND_DISABLE)
12772                         tg3_flag_set(tp, RGMII_INBAND_DISABLE);
12773                 if (cfg4 & NIC_SRAM_RGMII_EXT_IBND_RX_EN)
12774                         tg3_flag_set(tp, RGMII_EXT_IBND_RX_EN);
12775                 if (cfg4 & NIC_SRAM_RGMII_EXT_IBND_TX_EN)
12776                         tg3_flag_set(tp, RGMII_EXT_IBND_TX_EN);
12777         }
12778 done:
12779         if (tg3_flag(tp, WOL_CAP))
12780                 device_set_wakeup_enable(&tp->pdev->dev,
12781                                          tg3_flag(tp, WOL_ENABLE));
12782         else
12783                 device_set_wakeup_capable(&tp->pdev->dev, false);
12784 }
12785
12786 static int __devinit tg3_issue_otp_command(struct tg3 *tp, u32 cmd)
12787 {
12788         int i;
12789         u32 val;
12790
12791         tw32(OTP_CTRL, cmd | OTP_CTRL_OTP_CMD_START);
12792         tw32(OTP_CTRL, cmd);
12793
12794         /* Wait for up to 1 ms for command to execute. */
12795         for (i = 0; i < 100; i++) {
12796                 val = tr32(OTP_STATUS);
12797                 if (val & OTP_STATUS_CMD_DONE)
12798                         break;
12799                 udelay(10);
12800         }
12801
12802         return (val & OTP_STATUS_CMD_DONE) ? 0 : -EBUSY;
12803 }
12804
12805 /* Read the gphy configuration from the OTP region of the chip.  The gphy
12806  * configuration is a 32-bit value that straddles the alignment boundary.
12807  * We do two 32-bit reads and then shift and merge the results.
12808  */
12809 static u32 __devinit tg3_read_otp_phycfg(struct tg3 *tp)
12810 {
12811         u32 bhalf_otp, thalf_otp;
12812
12813         tw32(OTP_MODE, OTP_MODE_OTP_THRU_GRC);
12814
12815         if (tg3_issue_otp_command(tp, OTP_CTRL_OTP_CMD_INIT))
12816                 return 0;
12817
12818         tw32(OTP_ADDRESS, OTP_ADDRESS_MAGIC1);
12819
12820         if (tg3_issue_otp_command(tp, OTP_CTRL_OTP_CMD_READ))
12821                 return 0;
12822
12823         thalf_otp = tr32(OTP_READ_DATA);
12824
12825         tw32(OTP_ADDRESS, OTP_ADDRESS_MAGIC2);
12826
12827         if (tg3_issue_otp_command(tp, OTP_CTRL_OTP_CMD_READ))
12828                 return 0;
12829
12830         bhalf_otp = tr32(OTP_READ_DATA);
12831
12832         return ((thalf_otp & 0x0000ffff) << 16) | (bhalf_otp >> 16);
12833 }
12834
12835 static void __devinit tg3_phy_init_link_config(struct tg3 *tp)
12836 {
12837         u32 adv = ADVERTISED_Autoneg |
12838                   ADVERTISED_Pause;
12839
12840         if (!(tp->phy_flags & TG3_PHYFLG_10_100_ONLY))
12841                 adv |= ADVERTISED_1000baseT_Half |
12842                        ADVERTISED_1000baseT_Full;
12843
12844         if (!(tp->phy_flags & TG3_PHYFLG_ANY_SERDES))
12845                 adv |= ADVERTISED_100baseT_Half |
12846                        ADVERTISED_100baseT_Full |
12847                        ADVERTISED_10baseT_Half |
12848                        ADVERTISED_10baseT_Full |
12849                        ADVERTISED_TP;
12850         else
12851                 adv |= ADVERTISED_FIBRE;
12852
12853         tp->link_config.advertising = adv;
12854         tp->link_config.speed = SPEED_INVALID;
12855         tp->link_config.duplex = DUPLEX_INVALID;
12856         tp->link_config.autoneg = AUTONEG_ENABLE;
12857         tp->link_config.active_speed = SPEED_INVALID;
12858         tp->link_config.active_duplex = DUPLEX_INVALID;
12859         tp->link_config.orig_speed = SPEED_INVALID;
12860         tp->link_config.orig_duplex = DUPLEX_INVALID;
12861         tp->link_config.orig_autoneg = AUTONEG_INVALID;
12862 }
12863
12864 static int __devinit tg3_phy_probe(struct tg3 *tp)
12865 {
12866         u32 hw_phy_id_1, hw_phy_id_2;
12867         u32 hw_phy_id, hw_phy_id_masked;
12868         int err;
12869
12870         /* flow control autonegotiation is default behavior */
12871         tg3_flag_set(tp, PAUSE_AUTONEG);
12872         tp->link_config.flowctrl = FLOW_CTRL_TX | FLOW_CTRL_RX;
12873
12874         if (tg3_flag(tp, USE_PHYLIB))
12875                 return tg3_phy_init(tp);
12876
12877         /* Reading the PHY ID register can conflict with ASF
12878          * firmware access to the PHY hardware.
12879          */
12880         err = 0;
12881         if (tg3_flag(tp, ENABLE_ASF) || tg3_flag(tp, ENABLE_APE)) {
12882                 hw_phy_id = hw_phy_id_masked = TG3_PHY_ID_INVALID;
12883         } else {
12884                 /* Now read the physical PHY_ID from the chip and verify
12885                  * that it is sane.  If it doesn't look good, we fall back
12886                  * to either the hard-coded table based PHY_ID and failing
12887                  * that the value found in the eeprom area.
12888                  */
12889                 err |= tg3_readphy(tp, MII_PHYSID1, &hw_phy_id_1);
12890                 err |= tg3_readphy(tp, MII_PHYSID2, &hw_phy_id_2);
12891
12892                 hw_phy_id  = (hw_phy_id_1 & 0xffff) << 10;
12893                 hw_phy_id |= (hw_phy_id_2 & 0xfc00) << 16;
12894                 hw_phy_id |= (hw_phy_id_2 & 0x03ff) <<  0;
12895
12896                 hw_phy_id_masked = hw_phy_id & TG3_PHY_ID_MASK;
12897         }
12898
12899         if (!err && TG3_KNOWN_PHY_ID(hw_phy_id_masked)) {
12900                 tp->phy_id = hw_phy_id;
12901                 if (hw_phy_id_masked == TG3_PHY_ID_BCM8002)
12902                         tp->phy_flags |= TG3_PHYFLG_PHY_SERDES;
12903                 else
12904                         tp->phy_flags &= ~TG3_PHYFLG_PHY_SERDES;
12905         } else {
12906                 if (tp->phy_id != TG3_PHY_ID_INVALID) {
12907                         /* Do nothing, phy ID already set up in
12908                          * tg3_get_eeprom_hw_cfg().
12909                          */
12910                 } else {
12911                         struct subsys_tbl_ent *p;
12912
12913                         /* No eeprom signature?  Try the hardcoded
12914                          * subsys device table.
12915                          */
12916                         p = tg3_lookup_by_subsys(tp);
12917                         if (!p)
12918                                 return -ENODEV;
12919
12920                         tp->phy_id = p->phy_id;
12921                         if (!tp->phy_id ||
12922                             tp->phy_id == TG3_PHY_ID_BCM8002)
12923                                 tp->phy_flags |= TG3_PHYFLG_PHY_SERDES;
12924                 }
12925         }
12926
12927         if (!(tp->phy_flags & TG3_PHYFLG_ANY_SERDES) &&
12928             ((tp->pdev->device == TG3PCI_DEVICE_TIGON3_5718 &&
12929               tp->pci_chip_rev_id != CHIPREV_ID_5717_A0) ||
12930              (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_57765 &&
12931               tp->pci_chip_rev_id != CHIPREV_ID_57765_A0)))
12932                 tp->phy_flags |= TG3_PHYFLG_EEE_CAP;
12933
12934         tg3_phy_init_link_config(tp);
12935
12936         if (!(tp->phy_flags & TG3_PHYFLG_ANY_SERDES) &&
12937             !tg3_flag(tp, ENABLE_APE) &&
12938             !tg3_flag(tp, ENABLE_ASF)) {
12939                 u32 bmsr, adv_reg, tg3_ctrl, mask;
12940
12941                 tg3_readphy(tp, MII_BMSR, &bmsr);
12942                 if (!tg3_readphy(tp, MII_BMSR, &bmsr) &&
12943                     (bmsr & BMSR_LSTATUS))
12944                         goto skip_phy_reset;
12945
12946                 err = tg3_phy_reset(tp);
12947                 if (err)
12948                         return err;
12949
12950                 adv_reg = (ADVERTISE_10HALF | ADVERTISE_10FULL |
12951                            ADVERTISE_100HALF | ADVERTISE_100FULL |
12952                            ADVERTISE_CSMA | ADVERTISE_PAUSE_CAP);
12953                 tg3_ctrl = 0;
12954                 if (!(tp->phy_flags & TG3_PHYFLG_10_100_ONLY)) {
12955                         tg3_ctrl = (MII_TG3_CTRL_ADV_1000_HALF |
12956                                     MII_TG3_CTRL_ADV_1000_FULL);
12957                         if (tp->pci_chip_rev_id == CHIPREV_ID_5701_A0 ||
12958                             tp->pci_chip_rev_id == CHIPREV_ID_5701_B0)
12959                                 tg3_ctrl |= (MII_TG3_CTRL_AS_MASTER |
12960                                              MII_TG3_CTRL_ENABLE_AS_MASTER);
12961                 }
12962
12963                 mask = (ADVERTISED_10baseT_Half | ADVERTISED_10baseT_Full |
12964                         ADVERTISED_100baseT_Half | ADVERTISED_100baseT_Full |
12965                         ADVERTISED_1000baseT_Half | ADVERTISED_1000baseT_Full);
12966                 if (!tg3_copper_is_advertising_all(tp, mask)) {
12967                         tg3_writephy(tp, MII_ADVERTISE, adv_reg);
12968
12969                         if (!(tp->phy_flags & TG3_PHYFLG_10_100_ONLY))
12970                                 tg3_writephy(tp, MII_TG3_CTRL, tg3_ctrl);
12971
12972                         tg3_writephy(tp, MII_BMCR,
12973                                      BMCR_ANENABLE | BMCR_ANRESTART);
12974                 }
12975                 tg3_phy_set_wirespeed(tp);
12976
12977                 tg3_writephy(tp, MII_ADVERTISE, adv_reg);
12978                 if (!(tp->phy_flags & TG3_PHYFLG_10_100_ONLY))
12979                         tg3_writephy(tp, MII_TG3_CTRL, tg3_ctrl);
12980         }
12981
12982 skip_phy_reset:
12983         if ((tp->phy_id & TG3_PHY_ID_MASK) == TG3_PHY_ID_BCM5401) {
12984                 err = tg3_init_5401phy_dsp(tp);
12985                 if (err)
12986                         return err;
12987
12988                 err = tg3_init_5401phy_dsp(tp);
12989         }
12990
12991         return err;
12992 }
12993
12994 static void __devinit tg3_read_vpd(struct tg3 *tp)
12995 {
12996         u8 *vpd_data;
12997         unsigned int block_end, rosize, len;
12998         int j, i = 0;
12999
13000         vpd_data = (u8 *)tg3_vpd_readblock(tp);
13001         if (!vpd_data)
13002                 goto out_no_vpd;
13003
13004         i = pci_vpd_find_tag(vpd_data, 0, TG3_NVM_VPD_LEN,
13005                              PCI_VPD_LRDT_RO_DATA);
13006         if (i < 0)
13007                 goto out_not_found;
13008
13009         rosize = pci_vpd_lrdt_size(&vpd_data[i]);
13010         block_end = i + PCI_VPD_LRDT_TAG_SIZE + rosize;
13011         i += PCI_VPD_LRDT_TAG_SIZE;
13012
13013         if (block_end > TG3_NVM_VPD_LEN)
13014                 goto out_not_found;
13015
13016         j = pci_vpd_find_info_keyword(vpd_data, i, rosize,
13017                                       PCI_VPD_RO_KEYWORD_MFR_ID);
13018         if (j > 0) {
13019                 len = pci_vpd_info_field_size(&vpd_data[j]);
13020
13021                 j += PCI_VPD_INFO_FLD_HDR_SIZE;
13022                 if (j + len > block_end || len != 4 ||
13023                     memcmp(&vpd_data[j], "1028", 4))
13024                         goto partno;
13025
13026                 j = pci_vpd_find_info_keyword(vpd_data, i, rosize,
13027                                               PCI_VPD_RO_KEYWORD_VENDOR0);
13028                 if (j < 0)
13029                         goto partno;
13030
13031                 len = pci_vpd_info_field_size(&vpd_data[j]);
13032
13033                 j += PCI_VPD_INFO_FLD_HDR_SIZE;
13034                 if (j + len > block_end)
13035                         goto partno;
13036
13037                 memcpy(tp->fw_ver, &vpd_data[j], len);
13038                 strncat(tp->fw_ver, " bc ", TG3_NVM_VPD_LEN - len - 1);
13039         }
13040
13041 partno:
13042         i = pci_vpd_find_info_keyword(vpd_data, i, rosize,
13043                                       PCI_VPD_RO_KEYWORD_PARTNO);
13044         if (i < 0)
13045                 goto out_not_found;
13046
13047         len = pci_vpd_info_field_size(&vpd_data[i]);
13048
13049         i += PCI_VPD_INFO_FLD_HDR_SIZE;
13050         if (len > TG3_BPN_SIZE ||
13051             (len + i) > TG3_NVM_VPD_LEN)
13052                 goto out_not_found;
13053
13054         memcpy(tp->board_part_number, &vpd_data[i], len);
13055
13056 out_not_found:
13057         kfree(vpd_data);
13058         if (tp->board_part_number[0])
13059                 return;
13060
13061 out_no_vpd:
13062         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5717) {
13063                 if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_5717)
13064                         strcpy(tp->board_part_number, "BCM5717");
13065                 else if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_5718)
13066                         strcpy(tp->board_part_number, "BCM5718");
13067                 else
13068                         goto nomatch;
13069         } else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_57780) {
13070                 if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_57780)
13071                         strcpy(tp->board_part_number, "BCM57780");
13072                 else if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_57760)
13073                         strcpy(tp->board_part_number, "BCM57760");
13074                 else if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_57790)
13075                         strcpy(tp->board_part_number, "BCM57790");
13076                 else if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_57788)
13077                         strcpy(tp->board_part_number, "BCM57788");
13078                 else
13079                         goto nomatch;
13080         } else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_57765) {
13081                 if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_57761)
13082                         strcpy(tp->board_part_number, "BCM57761");
13083                 else if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_57765)
13084                         strcpy(tp->board_part_number, "BCM57765");
13085                 else if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_57781)
13086                         strcpy(tp->board_part_number, "BCM57781");
13087                 else if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_57785)
13088                         strcpy(tp->board_part_number, "BCM57785");
13089                 else if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_57791)
13090                         strcpy(tp->board_part_number, "BCM57791");
13091                 else if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_57795)
13092                         strcpy(tp->board_part_number, "BCM57795");
13093                 else
13094                         goto nomatch;
13095         } else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
13096                 strcpy(tp->board_part_number, "BCM95906");
13097         } else {
13098 nomatch:
13099                 strcpy(tp->board_part_number, "none");
13100         }
13101 }
13102
13103 static int __devinit tg3_fw_img_is_valid(struct tg3 *tp, u32 offset)
13104 {
13105         u32 val;
13106
13107         if (tg3_nvram_read(tp, offset, &val) ||
13108             (val & 0xfc000000) != 0x0c000000 ||
13109             tg3_nvram_read(tp, offset + 4, &val) ||
13110             val != 0)
13111                 return 0;
13112
13113         return 1;
13114 }
13115
13116 static void __devinit tg3_read_bc_ver(struct tg3 *tp)
13117 {
13118         u32 val, offset, start, ver_offset;
13119         int i, dst_off;
13120         bool newver = false;
13121
13122         if (tg3_nvram_read(tp, 0xc, &offset) ||
13123             tg3_nvram_read(tp, 0x4, &start))
13124                 return;
13125
13126         offset = tg3_nvram_logical_addr(tp, offset);
13127
13128         if (tg3_nvram_read(tp, offset, &val))
13129                 return;
13130
13131         if ((val & 0xfc000000) == 0x0c000000) {
13132                 if (tg3_nvram_read(tp, offset + 4, &val))
13133                         return;
13134
13135                 if (val == 0)
13136                         newver = true;
13137         }
13138
13139         dst_off = strlen(tp->fw_ver);
13140
13141         if (newver) {
13142                 if (TG3_VER_SIZE - dst_off < 16 ||
13143                     tg3_nvram_read(tp, offset + 8, &ver_offset))
13144                         return;
13145
13146                 offset = offset + ver_offset - start;
13147                 for (i = 0; i < 16; i += 4) {
13148                         __be32 v;
13149                         if (tg3_nvram_read_be32(tp, offset + i, &v))
13150                                 return;
13151
13152                         memcpy(tp->fw_ver + dst_off + i, &v, sizeof(v));
13153                 }
13154         } else {
13155                 u32 major, minor;
13156
13157                 if (tg3_nvram_read(tp, TG3_NVM_PTREV_BCVER, &ver_offset))
13158                         return;
13159
13160                 major = (ver_offset & TG3_NVM_BCVER_MAJMSK) >>
13161                         TG3_NVM_BCVER_MAJSFT;
13162                 minor = ver_offset & TG3_NVM_BCVER_MINMSK;
13163                 snprintf(&tp->fw_ver[dst_off], TG3_VER_SIZE - dst_off,
13164                          "v%d.%02d", major, minor);
13165         }
13166 }
13167
13168 static void __devinit tg3_read_hwsb_ver(struct tg3 *tp)
13169 {
13170         u32 val, major, minor;
13171
13172         /* Use native endian representation */
13173         if (tg3_nvram_read(tp, TG3_NVM_HWSB_CFG1, &val))
13174                 return;
13175
13176         major = (val & TG3_NVM_HWSB_CFG1_MAJMSK) >>
13177                 TG3_NVM_HWSB_CFG1_MAJSFT;
13178         minor = (val & TG3_NVM_HWSB_CFG1_MINMSK) >>
13179                 TG3_NVM_HWSB_CFG1_MINSFT;
13180
13181         snprintf(&tp->fw_ver[0], 32, "sb v%d.%02d", major, minor);
13182 }
13183
13184 static void __devinit tg3_read_sb_ver(struct tg3 *tp, u32 val)
13185 {
13186         u32 offset, major, minor, build;
13187
13188         strncat(tp->fw_ver, "sb", TG3_VER_SIZE - strlen(tp->fw_ver) - 1);
13189
13190         if ((val & TG3_EEPROM_SB_FORMAT_MASK) != TG3_EEPROM_SB_FORMAT_1)
13191                 return;
13192
13193         switch (val & TG3_EEPROM_SB_REVISION_MASK) {
13194         case TG3_EEPROM_SB_REVISION_0:
13195                 offset = TG3_EEPROM_SB_F1R0_EDH_OFF;
13196                 break;
13197         case TG3_EEPROM_SB_REVISION_2:
13198                 offset = TG3_EEPROM_SB_F1R2_EDH_OFF;
13199                 break;
13200         case TG3_EEPROM_SB_REVISION_3:
13201                 offset = TG3_EEPROM_SB_F1R3_EDH_OFF;
13202                 break;
13203         case TG3_EEPROM_SB_REVISION_4:
13204                 offset = TG3_EEPROM_SB_F1R4_EDH_OFF;
13205                 break;
13206         case TG3_EEPROM_SB_REVISION_5:
13207                 offset = TG3_EEPROM_SB_F1R5_EDH_OFF;
13208                 break;
13209         case TG3_EEPROM_SB_REVISION_6:
13210                 offset = TG3_EEPROM_SB_F1R6_EDH_OFF;
13211                 break;
13212         default:
13213                 return;
13214         }
13215
13216         if (tg3_nvram_read(tp, offset, &val))
13217                 return;
13218
13219         build = (val & TG3_EEPROM_SB_EDH_BLD_MASK) >>
13220                 TG3_EEPROM_SB_EDH_BLD_SHFT;
13221         major = (val & TG3_EEPROM_SB_EDH_MAJ_MASK) >>
13222                 TG3_EEPROM_SB_EDH_MAJ_SHFT;
13223         minor =  val & TG3_EEPROM_SB_EDH_MIN_MASK;
13224
13225         if (minor > 99 || build > 26)
13226                 return;
13227
13228         offset = strlen(tp->fw_ver);
13229         snprintf(&tp->fw_ver[offset], TG3_VER_SIZE - offset,
13230                  " v%d.%02d", major, minor);
13231
13232         if (build > 0) {
13233                 offset = strlen(tp->fw_ver);
13234                 if (offset < TG3_VER_SIZE - 1)
13235                         tp->fw_ver[offset] = 'a' + build - 1;
13236         }
13237 }
13238
13239 static void __devinit tg3_read_mgmtfw_ver(struct tg3 *tp)
13240 {
13241         u32 val, offset, start;
13242         int i, vlen;
13243
13244         for (offset = TG3_NVM_DIR_START;
13245              offset < TG3_NVM_DIR_END;
13246              offset += TG3_NVM_DIRENT_SIZE) {
13247                 if (tg3_nvram_read(tp, offset, &val))
13248                         return;
13249
13250                 if ((val >> TG3_NVM_DIRTYPE_SHIFT) == TG3_NVM_DIRTYPE_ASFINI)
13251                         break;
13252         }
13253
13254         if (offset == TG3_NVM_DIR_END)
13255                 return;
13256
13257         if (!tg3_flag(tp, 5705_PLUS))
13258                 start = 0x08000000;
13259         else if (tg3_nvram_read(tp, offset - 4, &start))
13260                 return;
13261
13262         if (tg3_nvram_read(tp, offset + 4, &offset) ||
13263             !tg3_fw_img_is_valid(tp, offset) ||
13264             tg3_nvram_read(tp, offset + 8, &val))
13265                 return;
13266
13267         offset += val - start;
13268
13269         vlen = strlen(tp->fw_ver);
13270
13271         tp->fw_ver[vlen++] = ',';
13272         tp->fw_ver[vlen++] = ' ';
13273
13274         for (i = 0; i < 4; i++) {
13275                 __be32 v;
13276                 if (tg3_nvram_read_be32(tp, offset, &v))
13277                         return;
13278
13279                 offset += sizeof(v);
13280
13281                 if (vlen > TG3_VER_SIZE - sizeof(v)) {
13282                         memcpy(&tp->fw_ver[vlen], &v, TG3_VER_SIZE - vlen);
13283                         break;
13284                 }
13285
13286                 memcpy(&tp->fw_ver[vlen], &v, sizeof(v));
13287                 vlen += sizeof(v);
13288         }
13289 }
13290
13291 static void __devinit tg3_read_dash_ver(struct tg3 *tp)
13292 {
13293         int vlen;
13294         u32 apedata;
13295         char *fwtype;
13296
13297         if (!tg3_flag(tp, ENABLE_APE) || !tg3_flag(tp, ENABLE_ASF))
13298                 return;
13299
13300         apedata = tg3_ape_read32(tp, TG3_APE_SEG_SIG);
13301         if (apedata != APE_SEG_SIG_MAGIC)
13302                 return;
13303
13304         apedata = tg3_ape_read32(tp, TG3_APE_FW_STATUS);
13305         if (!(apedata & APE_FW_STATUS_READY))
13306                 return;
13307
13308         apedata = tg3_ape_read32(tp, TG3_APE_FW_VERSION);
13309
13310         if (tg3_ape_read32(tp, TG3_APE_FW_FEATURES) & TG3_APE_FW_FEATURE_NCSI) {
13311                 tg3_flag_set(tp, APE_HAS_NCSI);
13312                 fwtype = "NCSI";
13313         } else {
13314                 fwtype = "DASH";
13315         }
13316
13317         vlen = strlen(tp->fw_ver);
13318
13319         snprintf(&tp->fw_ver[vlen], TG3_VER_SIZE - vlen, " %s v%d.%d.%d.%d",
13320                  fwtype,
13321                  (apedata & APE_FW_VERSION_MAJMSK) >> APE_FW_VERSION_MAJSFT,
13322                  (apedata & APE_FW_VERSION_MINMSK) >> APE_FW_VERSION_MINSFT,
13323                  (apedata & APE_FW_VERSION_REVMSK) >> APE_FW_VERSION_REVSFT,
13324                  (apedata & APE_FW_VERSION_BLDMSK));
13325 }
13326
13327 static void __devinit tg3_read_fw_ver(struct tg3 *tp)
13328 {
13329         u32 val;
13330         bool vpd_vers = false;
13331
13332         if (tp->fw_ver[0] != 0)
13333                 vpd_vers = true;
13334
13335         if (tg3_flag(tp, NO_NVRAM)) {
13336                 strcat(tp->fw_ver, "sb");
13337                 return;
13338         }
13339
13340         if (tg3_nvram_read(tp, 0, &val))
13341                 return;
13342
13343         if (val == TG3_EEPROM_MAGIC)
13344                 tg3_read_bc_ver(tp);
13345         else if ((val & TG3_EEPROM_MAGIC_FW_MSK) == TG3_EEPROM_MAGIC_FW)
13346                 tg3_read_sb_ver(tp, val);
13347         else if ((val & TG3_EEPROM_MAGIC_HW_MSK) == TG3_EEPROM_MAGIC_HW)
13348                 tg3_read_hwsb_ver(tp);
13349         else
13350                 return;
13351
13352         if (!tg3_flag(tp, ENABLE_ASF) || tg3_flag(tp, ENABLE_APE) || vpd_vers)
13353                 goto done;
13354
13355         tg3_read_mgmtfw_ver(tp);
13356
13357 done:
13358         tp->fw_ver[TG3_VER_SIZE - 1] = 0;
13359 }
13360
13361 static struct pci_dev * __devinit tg3_find_peer(struct tg3 *);
13362
13363 static inline u32 tg3_rx_ret_ring_size(struct tg3 *tp)
13364 {
13365         if (tg3_flag(tp, LRG_PROD_RING_CAP))
13366                 return TG3_RX_RET_MAX_SIZE_5717;
13367         else if (tg3_flag(tp, JUMBO_CAPABLE) && !tg3_flag(tp, 5780_CLASS))
13368                 return TG3_RX_RET_MAX_SIZE_5700;
13369         else
13370                 return TG3_RX_RET_MAX_SIZE_5705;
13371 }
13372
13373 static DEFINE_PCI_DEVICE_TABLE(tg3_write_reorder_chipsets) = {
13374         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_FE_GATE_700C) },
13375         { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_8131_BRIDGE) },
13376         { PCI_DEVICE(PCI_VENDOR_ID_VIA, PCI_DEVICE_ID_VIA_8385_0) },
13377         { },
13378 };
13379
13380 static int __devinit tg3_get_invariants(struct tg3 *tp)
13381 {
13382         u32 misc_ctrl_reg;
13383         u32 pci_state_reg, grc_misc_cfg;
13384         u32 val;
13385         u16 pci_cmd;
13386         int err;
13387
13388         /* Force memory write invalidate off.  If we leave it on,
13389          * then on 5700_BX chips we have to enable a workaround.
13390          * The workaround is to set the TG3PCI_DMA_RW_CTRL boundary
13391          * to match the cacheline size.  The Broadcom driver have this
13392          * workaround but turns MWI off all the times so never uses
13393          * it.  This seems to suggest that the workaround is insufficient.
13394          */
13395         pci_read_config_word(tp->pdev, PCI_COMMAND, &pci_cmd);
13396         pci_cmd &= ~PCI_COMMAND_INVALIDATE;
13397         pci_write_config_word(tp->pdev, PCI_COMMAND, pci_cmd);
13398
13399         /* It is absolutely critical that TG3PCI_MISC_HOST_CTRL
13400          * has the register indirect write enable bit set before
13401          * we try to access any of the MMIO registers.  It is also
13402          * critical that the PCI-X hw workaround situation is decided
13403          * before that as well.
13404          */
13405         pci_read_config_dword(tp->pdev, TG3PCI_MISC_HOST_CTRL,
13406                               &misc_ctrl_reg);
13407
13408         tp->pci_chip_rev_id = (misc_ctrl_reg >>
13409                                MISC_HOST_CTRL_CHIPREV_SHIFT);
13410         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_USE_PROD_ID_REG) {
13411                 u32 prod_id_asic_rev;
13412
13413                 if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_5717 ||
13414                     tp->pdev->device == TG3PCI_DEVICE_TIGON3_5718 ||
13415                     tp->pdev->device == TG3PCI_DEVICE_TIGON3_5719 ||
13416                     tp->pdev->device == TG3PCI_DEVICE_TIGON3_5720)
13417                         pci_read_config_dword(tp->pdev,
13418                                               TG3PCI_GEN2_PRODID_ASICREV,
13419                                               &prod_id_asic_rev);
13420                 else if (tp->pdev->device == TG3PCI_DEVICE_TIGON3_57781 ||
13421                          tp->pdev->device == TG3PCI_DEVICE_TIGON3_57785 ||
13422                          tp->pdev->device == TG3PCI_DEVICE_TIGON3_57761 ||
13423                          tp->pdev->device == TG3PCI_DEVICE_TIGON3_57765 ||
13424                          tp->pdev->device == TG3PCI_DEVICE_TIGON3_57791 ||
13425                          tp->pdev->device == TG3PCI_DEVICE_TIGON3_57795)
13426                         pci_read_config_dword(tp->pdev,
13427                                               TG3PCI_GEN15_PRODID_ASICREV,
13428                                               &prod_id_asic_rev);
13429                 else
13430                         pci_read_config_dword(tp->pdev, TG3PCI_PRODID_ASICREV,
13431                                               &prod_id_asic_rev);
13432
13433                 tp->pci_chip_rev_id = prod_id_asic_rev;
13434         }
13435
13436         /* Wrong chip ID in 5752 A0. This code can be removed later
13437          * as A0 is not in production.
13438          */
13439         if (tp->pci_chip_rev_id == CHIPREV_ID_5752_A0_HW)
13440                 tp->pci_chip_rev_id = CHIPREV_ID_5752_A0;
13441
13442         /* If we have 5702/03 A1 or A2 on certain ICH chipsets,
13443          * we need to disable memory and use config. cycles
13444          * only to access all registers. The 5702/03 chips
13445          * can mistakenly decode the special cycles from the
13446          * ICH chipsets as memory write cycles, causing corruption
13447          * of register and memory space. Only certain ICH bridges
13448          * will drive special cycles with non-zero data during the
13449          * address phase which can fall within the 5703's address
13450          * range. This is not an ICH bug as the PCI spec allows
13451          * non-zero address during special cycles. However, only
13452          * these ICH bridges are known to drive non-zero addresses
13453          * during special cycles.
13454          *
13455          * Since special cycles do not cross PCI bridges, we only
13456          * enable this workaround if the 5703 is on the secondary
13457          * bus of these ICH bridges.
13458          */
13459         if ((tp->pci_chip_rev_id == CHIPREV_ID_5703_A1) ||
13460             (tp->pci_chip_rev_id == CHIPREV_ID_5703_A2)) {
13461                 static struct tg3_dev_id {
13462                         u32     vendor;
13463                         u32     device;
13464                         u32     rev;
13465                 } ich_chipsets[] = {
13466                         { PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82801AA_8,
13467                           PCI_ANY_ID },
13468                         { PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82801AB_8,
13469                           PCI_ANY_ID },
13470                         { PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82801BA_11,
13471                           0xa },
13472                         { PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82801BA_6,
13473                           PCI_ANY_ID },
13474                         { },
13475                 };
13476                 struct tg3_dev_id *pci_id = &ich_chipsets[0];
13477                 struct pci_dev *bridge = NULL;
13478
13479                 while (pci_id->vendor != 0) {
13480                         bridge = pci_get_device(pci_id->vendor, pci_id->device,
13481                                                 bridge);
13482                         if (!bridge) {
13483                                 pci_id++;
13484                                 continue;
13485                         }
13486                         if (pci_id->rev != PCI_ANY_ID) {
13487                                 if (bridge->revision > pci_id->rev)
13488                                         continue;
13489                         }
13490                         if (bridge->subordinate &&
13491                             (bridge->subordinate->number ==
13492                              tp->pdev->bus->number)) {
13493                                 tg3_flag_set(tp, ICH_WORKAROUND);
13494                                 pci_dev_put(bridge);
13495                                 break;
13496                         }
13497                 }
13498         }
13499
13500         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701)) {
13501                 static struct tg3_dev_id {
13502                         u32     vendor;
13503                         u32     device;
13504                 } bridge_chipsets[] = {
13505                         { PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_PXH_0 },
13506                         { PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_PXH_1 },
13507                         { },
13508                 };
13509                 struct tg3_dev_id *pci_id = &bridge_chipsets[0];
13510                 struct pci_dev *bridge = NULL;
13511
13512                 while (pci_id->vendor != 0) {
13513                         bridge = pci_get_device(pci_id->vendor,
13514                                                 pci_id->device,
13515                                                 bridge);
13516                         if (!bridge) {
13517                                 pci_id++;
13518                                 continue;
13519                         }
13520                         if (bridge->subordinate &&
13521                             (bridge->subordinate->number <=
13522                              tp->pdev->bus->number) &&
13523                             (bridge->subordinate->subordinate >=
13524                              tp->pdev->bus->number)) {
13525                                 tg3_flag_set(tp, 5701_DMA_BUG);
13526                                 pci_dev_put(bridge);
13527                                 break;
13528                         }
13529                 }
13530         }
13531
13532         /* The EPB bridge inside 5714, 5715, and 5780 cannot support
13533          * DMA addresses > 40-bit. This bridge may have other additional
13534          * 57xx devices behind it in some 4-port NIC designs for example.
13535          * Any tg3 device found behind the bridge will also need the 40-bit
13536          * DMA workaround.
13537          */
13538         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5780 ||
13539             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714) {
13540                 tg3_flag_set(tp, 5780_CLASS);
13541                 tg3_flag_set(tp, 40BIT_DMA_BUG);
13542                 tp->msi_cap = pci_find_capability(tp->pdev, PCI_CAP_ID_MSI);
13543         } else {
13544                 struct pci_dev *bridge = NULL;
13545
13546                 do {
13547                         bridge = pci_get_device(PCI_VENDOR_ID_SERVERWORKS,
13548                                                 PCI_DEVICE_ID_SERVERWORKS_EPB,
13549                                                 bridge);
13550                         if (bridge && bridge->subordinate &&
13551                             (bridge->subordinate->number <=
13552                              tp->pdev->bus->number) &&
13553                             (bridge->subordinate->subordinate >=
13554                              tp->pdev->bus->number)) {
13555                                 tg3_flag_set(tp, 40BIT_DMA_BUG);
13556                                 pci_dev_put(bridge);
13557                                 break;
13558                         }
13559                 } while (bridge);
13560         }
13561
13562         /* Initialize misc host control in PCI block. */
13563         tp->misc_host_ctrl |= (misc_ctrl_reg &
13564                                MISC_HOST_CTRL_CHIPREV);
13565         pci_write_config_dword(tp->pdev, TG3PCI_MISC_HOST_CTRL,
13566                                tp->misc_host_ctrl);
13567
13568         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704 ||
13569             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714 ||
13570             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5717 ||
13571             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5720)
13572                 tp->pdev_peer = tg3_find_peer(tp);
13573
13574         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5717 ||
13575             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5719 ||
13576             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5720)
13577                 tg3_flag_set(tp, 5717_PLUS);
13578
13579         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_57765 ||
13580             tg3_flag(tp, 5717_PLUS))
13581                 tg3_flag_set(tp, 57765_PLUS);
13582
13583         /* Intentionally exclude ASIC_REV_5906 */
13584         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755 ||
13585             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787 ||
13586             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 ||
13587             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761 ||
13588             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5785 ||
13589             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_57780 ||
13590             tg3_flag(tp, 57765_PLUS))
13591                 tg3_flag_set(tp, 5755_PLUS);
13592
13593         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5750 ||
13594             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5752 ||
13595             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906 ||
13596             tg3_flag(tp, 5755_PLUS) ||
13597             tg3_flag(tp, 5780_CLASS))
13598                 tg3_flag_set(tp, 5750_PLUS);
13599
13600         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) ||
13601             tg3_flag(tp, 5750_PLUS))
13602                 tg3_flag_set(tp, 5705_PLUS);
13603
13604         /* 5700 B0 chips do not support checksumming correctly due
13605          * to hardware bugs.
13606          */
13607         if (tp->pci_chip_rev_id != CHIPREV_ID_5700_B0) {
13608                 u32 features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_RXCSUM;
13609
13610                 if (tg3_flag(tp, 5755_PLUS))
13611                         features |= NETIF_F_IPV6_CSUM;
13612                 tp->dev->features |= features;
13613                 tp->dev->hw_features |= features;
13614                 tp->dev->vlan_features |= features;
13615         }
13616
13617         /* Determine TSO capabilities */
13618         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5719)
13619                 ; /* Do nothing. HW bug. */
13620         else if (tg3_flag(tp, 57765_PLUS))
13621                 tg3_flag_set(tp, HW_TSO_3);
13622         else if (tg3_flag(tp, 5755_PLUS) ||
13623                  GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906)
13624                 tg3_flag_set(tp, HW_TSO_2);
13625         else if (tg3_flag(tp, 5750_PLUS)) {
13626                 tg3_flag_set(tp, HW_TSO_1);
13627                 tg3_flag_set(tp, TSO_BUG);
13628                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5750 &&
13629                     tp->pci_chip_rev_id >= CHIPREV_ID_5750_C2)
13630                         tg3_flag_clear(tp, TSO_BUG);
13631         } else if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700 &&
13632                    GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5701 &&
13633                    tp->pci_chip_rev_id != CHIPREV_ID_5705_A0) {
13634                         tg3_flag_set(tp, TSO_BUG);
13635                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705)
13636                         tp->fw_needed = FIRMWARE_TG3TSO5;
13637                 else
13638                         tp->fw_needed = FIRMWARE_TG3TSO;
13639         }
13640
13641         tp->irq_max = 1;
13642
13643         if (tg3_flag(tp, 5750_PLUS)) {
13644                 tg3_flag_set(tp, SUPPORT_MSI);
13645                 if (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5750_AX ||
13646                     GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5750_BX ||
13647                     (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714 &&
13648                      tp->pci_chip_rev_id <= CHIPREV_ID_5714_A2 &&
13649                      tp->pdev_peer == tp->pdev))
13650                         tg3_flag_clear(tp, SUPPORT_MSI);
13651
13652                 if (tg3_flag(tp, 5755_PLUS) ||
13653                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
13654                         tg3_flag_set(tp, 1SHOT_MSI);
13655                 }
13656
13657                 if (tg3_flag(tp, 57765_PLUS)) {
13658                         tg3_flag_set(tp, SUPPORT_MSIX);
13659                         tp->irq_max = TG3_IRQ_MAX_VECS;
13660                 }
13661         }
13662
13663         /* All chips can get confused if TX buffers
13664          * straddle the 4GB address boundary.
13665          */
13666         tg3_flag_set(tp, 4G_DMA_BNDRY_BUG);
13667
13668         if (tg3_flag(tp, 5755_PLUS))
13669                 tg3_flag_set(tp, SHORT_DMA_BUG);
13670         else
13671                 tg3_flag_set(tp, 40BIT_DMA_LIMIT_BUG);
13672
13673         if (tg3_flag(tp, 5717_PLUS))
13674                 tg3_flag_set(tp, LRG_PROD_RING_CAP);
13675
13676         if (tg3_flag(tp, 57765_PLUS) &&
13677             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5719)
13678                 tg3_flag_set(tp, USE_JUMBO_BDFLAG);
13679
13680         if (!tg3_flag(tp, 5705_PLUS) ||
13681             tg3_flag(tp, 5780_CLASS) ||
13682             tg3_flag(tp, USE_JUMBO_BDFLAG))
13683                 tg3_flag_set(tp, JUMBO_CAPABLE);
13684
13685         pci_read_config_dword(tp->pdev, TG3PCI_PCISTATE,
13686                               &pci_state_reg);
13687
13688         tp->pcie_cap = pci_find_capability(tp->pdev, PCI_CAP_ID_EXP);
13689         if (tp->pcie_cap != 0) {
13690                 u16 lnkctl;
13691
13692                 tg3_flag_set(tp, PCI_EXPRESS);
13693
13694                 tp->pcie_readrq = 4096;
13695                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5719 ||
13696                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5720)
13697                         tp->pcie_readrq = 2048;
13698
13699                 pcie_set_readrq(tp->pdev, tp->pcie_readrq);
13700
13701                 pci_read_config_word(tp->pdev,
13702                                      tp->pcie_cap + PCI_EXP_LNKCTL,
13703                                      &lnkctl);
13704                 if (lnkctl & PCI_EXP_LNKCTL_CLKREQ_EN) {
13705                         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906)
13706                                 tg3_flag_clear(tp, HW_TSO_2);
13707                         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 ||
13708                             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761 ||
13709                             tp->pci_chip_rev_id == CHIPREV_ID_57780_A0 ||
13710                             tp->pci_chip_rev_id == CHIPREV_ID_57780_A1)
13711                                 tg3_flag_set(tp, CLKREQ_BUG);
13712                 } else if (tp->pci_chip_rev_id == CHIPREV_ID_5717_A0) {
13713                         tg3_flag_set(tp, L1PLLPD_EN);
13714                 }
13715         } else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5785) {
13716                 tg3_flag_set(tp, PCI_EXPRESS);
13717         } else if (!tg3_flag(tp, 5705_PLUS) ||
13718                    tg3_flag(tp, 5780_CLASS)) {
13719                 tp->pcix_cap = pci_find_capability(tp->pdev, PCI_CAP_ID_PCIX);
13720                 if (!tp->pcix_cap) {
13721                         dev_err(&tp->pdev->dev,
13722                                 "Cannot find PCI-X capability, aborting\n");
13723                         return -EIO;
13724                 }
13725
13726                 if (!(pci_state_reg & PCISTATE_CONV_PCI_MODE))
13727                         tg3_flag_set(tp, PCIX_MODE);
13728         }
13729
13730         /* If we have an AMD 762 or VIA K8T800 chipset, write
13731          * reordering to the mailbox registers done by the host
13732          * controller can cause major troubles.  We read back from
13733          * every mailbox register write to force the writes to be
13734          * posted to the chip in order.
13735          */
13736         if (pci_dev_present(tg3_write_reorder_chipsets) &&
13737             !tg3_flag(tp, PCI_EXPRESS))
13738                 tg3_flag_set(tp, MBOX_WRITE_REORDER);
13739
13740         pci_read_config_byte(tp->pdev, PCI_CACHE_LINE_SIZE,
13741                              &tp->pci_cacheline_sz);
13742         pci_read_config_byte(tp->pdev, PCI_LATENCY_TIMER,
13743                              &tp->pci_lat_timer);
13744         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 &&
13745             tp->pci_lat_timer < 64) {
13746                 tp->pci_lat_timer = 64;
13747                 pci_write_config_byte(tp->pdev, PCI_LATENCY_TIMER,
13748                                       tp->pci_lat_timer);
13749         }
13750
13751         if (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5700_BX) {
13752                 /* 5700 BX chips need to have their TX producer index
13753                  * mailboxes written twice to workaround a bug.
13754                  */
13755                 tg3_flag_set(tp, TXD_MBOX_HWBUG);
13756
13757                 /* If we are in PCI-X mode, enable register write workaround.
13758                  *
13759                  * The workaround is to use indirect register accesses
13760                  * for all chip writes not to mailbox registers.
13761                  */
13762                 if (tg3_flag(tp, PCIX_MODE)) {
13763                         u32 pm_reg;
13764
13765                         tg3_flag_set(tp, PCIX_TARGET_HWBUG);
13766
13767                         /* The chip can have it's power management PCI config
13768                          * space registers clobbered due to this bug.
13769                          * So explicitly force the chip into D0 here.
13770                          */
13771                         pci_read_config_dword(tp->pdev,
13772                                               tp->pm_cap + PCI_PM_CTRL,
13773                                               &pm_reg);
13774                         pm_reg &= ~PCI_PM_CTRL_STATE_MASK;
13775                         pm_reg |= PCI_PM_CTRL_PME_ENABLE | 0 /* D0 */;
13776                         pci_write_config_dword(tp->pdev,
13777                                                tp->pm_cap + PCI_PM_CTRL,
13778                                                pm_reg);
13779
13780                         /* Also, force SERR#/PERR# in PCI command. */
13781                         pci_read_config_word(tp->pdev, PCI_COMMAND, &pci_cmd);
13782                         pci_cmd |= PCI_COMMAND_PARITY | PCI_COMMAND_SERR;
13783                         pci_write_config_word(tp->pdev, PCI_COMMAND, pci_cmd);
13784                 }
13785         }
13786
13787         if ((pci_state_reg & PCISTATE_BUS_SPEED_HIGH) != 0)
13788                 tg3_flag_set(tp, PCI_HIGH_SPEED);
13789         if ((pci_state_reg & PCISTATE_BUS_32BIT) != 0)
13790                 tg3_flag_set(tp, PCI_32BIT);
13791
13792         /* Chip-specific fixup from Broadcom driver */
13793         if ((tp->pci_chip_rev_id == CHIPREV_ID_5704_A0) &&
13794             (!(pci_state_reg & PCISTATE_RETRY_SAME_DMA))) {
13795                 pci_state_reg |= PCISTATE_RETRY_SAME_DMA;
13796                 pci_write_config_dword(tp->pdev, TG3PCI_PCISTATE, pci_state_reg);
13797         }
13798
13799         /* Default fast path register access methods */
13800         tp->read32 = tg3_read32;
13801         tp->write32 = tg3_write32;
13802         tp->read32_mbox = tg3_read32;
13803         tp->write32_mbox = tg3_write32;
13804         tp->write32_tx_mbox = tg3_write32;
13805         tp->write32_rx_mbox = tg3_write32;
13806
13807         /* Various workaround register access methods */
13808         if (tg3_flag(tp, PCIX_TARGET_HWBUG))
13809                 tp->write32 = tg3_write_indirect_reg32;
13810         else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701 ||
13811                  (tg3_flag(tp, PCI_EXPRESS) &&
13812                   tp->pci_chip_rev_id == CHIPREV_ID_5750_A0)) {
13813                 /*
13814                  * Back to back register writes can cause problems on these
13815                  * chips, the workaround is to read back all reg writes
13816                  * except those to mailbox regs.
13817                  *
13818                  * See tg3_write_indirect_reg32().
13819                  */
13820                 tp->write32 = tg3_write_flush_reg32;
13821         }
13822
13823         if (tg3_flag(tp, TXD_MBOX_HWBUG) || tg3_flag(tp, MBOX_WRITE_REORDER)) {
13824                 tp->write32_tx_mbox = tg3_write32_tx_mbox;
13825                 if (tg3_flag(tp, MBOX_WRITE_REORDER))
13826                         tp->write32_rx_mbox = tg3_write_flush_reg32;
13827         }
13828
13829         if (tg3_flag(tp, ICH_WORKAROUND)) {
13830                 tp->read32 = tg3_read_indirect_reg32;
13831                 tp->write32 = tg3_write_indirect_reg32;
13832                 tp->read32_mbox = tg3_read_indirect_mbox;
13833                 tp->write32_mbox = tg3_write_indirect_mbox;
13834                 tp->write32_tx_mbox = tg3_write_indirect_mbox;
13835                 tp->write32_rx_mbox = tg3_write_indirect_mbox;
13836
13837                 iounmap(tp->regs);
13838                 tp->regs = NULL;
13839
13840                 pci_read_config_word(tp->pdev, PCI_COMMAND, &pci_cmd);
13841                 pci_cmd &= ~PCI_COMMAND_MEMORY;
13842                 pci_write_config_word(tp->pdev, PCI_COMMAND, pci_cmd);
13843         }
13844         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
13845                 tp->read32_mbox = tg3_read32_mbox_5906;
13846                 tp->write32_mbox = tg3_write32_mbox_5906;
13847                 tp->write32_tx_mbox = tg3_write32_mbox_5906;
13848                 tp->write32_rx_mbox = tg3_write32_mbox_5906;
13849         }
13850
13851         if (tp->write32 == tg3_write_indirect_reg32 ||
13852             (tg3_flag(tp, PCIX_MODE) &&
13853              (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
13854               GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701)))
13855                 tg3_flag_set(tp, SRAM_USE_CONFIG);
13856
13857         /* Get eeprom hw config before calling tg3_set_power_state().
13858          * In particular, the TG3_FLAG_IS_NIC flag must be
13859          * determined before calling tg3_set_power_state() so that
13860          * we know whether or not to switch out of Vaux power.
13861          * When the flag is set, it means that GPIO1 is used for eeprom
13862          * write protect and also implies that it is a LOM where GPIOs
13863          * are not used to switch power.
13864          */
13865         tg3_get_eeprom_hw_cfg(tp);
13866
13867         if (tg3_flag(tp, ENABLE_APE)) {
13868                 /* Allow reads and writes to the
13869                  * APE register and memory space.
13870                  */
13871                 pci_state_reg |= PCISTATE_ALLOW_APE_CTLSPC_WR |
13872                                  PCISTATE_ALLOW_APE_SHMEM_WR |
13873                                  PCISTATE_ALLOW_APE_PSPACE_WR;
13874                 pci_write_config_dword(tp->pdev, TG3PCI_PCISTATE,
13875                                        pci_state_reg);
13876         }
13877
13878         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 ||
13879             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761 ||
13880             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5785 ||
13881             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_57780 ||
13882             tg3_flag(tp, 57765_PLUS))
13883                 tg3_flag_set(tp, CPMU_PRESENT);
13884
13885         /* Set up tp->grc_local_ctrl before calling tg3_power_up().
13886          * GPIO1 driven high will bring 5700's external PHY out of reset.
13887          * It is also used as eeprom write protect on LOMs.
13888          */
13889         tp->grc_local_ctrl = GRC_LCLCTRL_INT_ON_ATTN | GRC_LCLCTRL_AUTO_SEEPROM;
13890         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700) ||
13891             tg3_flag(tp, EEPROM_WRITE_PROT))
13892                 tp->grc_local_ctrl |= (GRC_LCLCTRL_GPIO_OE1 |
13893                                        GRC_LCLCTRL_GPIO_OUTPUT1);
13894         /* Unused GPIO3 must be driven as output on 5752 because there
13895          * are no pull-up resistors on unused GPIO pins.
13896          */
13897         else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5752)
13898                 tp->grc_local_ctrl |= GRC_LCLCTRL_GPIO_OE3;
13899
13900         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755 ||
13901             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_57780 ||
13902             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_57765)
13903                 tp->grc_local_ctrl |= GRC_LCLCTRL_GPIO_UART_SEL;
13904
13905         if (tp->pdev->device == PCI_DEVICE_ID_TIGON3_5761 ||
13906             tp->pdev->device == TG3PCI_DEVICE_TIGON3_5761S) {
13907                 /* Turn off the debug UART. */
13908                 tp->grc_local_ctrl |= GRC_LCLCTRL_GPIO_UART_SEL;
13909                 if (tg3_flag(tp, IS_NIC))
13910                         /* Keep VMain power. */
13911                         tp->grc_local_ctrl |= GRC_LCLCTRL_GPIO_OE0 |
13912                                               GRC_LCLCTRL_GPIO_OUTPUT0;
13913         }
13914
13915         /* Force the chip into D0. */
13916         err = tg3_power_up(tp);
13917         if (err) {
13918                 dev_err(&tp->pdev->dev, "Transition to D0 failed\n");
13919                 return err;
13920         }
13921
13922         /* Derive initial jumbo mode from MTU assigned in
13923          * ether_setup() via the alloc_etherdev() call
13924          */
13925         if (tp->dev->mtu > ETH_DATA_LEN && !tg3_flag(tp, 5780_CLASS))
13926                 tg3_flag_set(tp, JUMBO_RING_ENABLE);
13927
13928         /* Determine WakeOnLan speed to use. */
13929         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
13930             tp->pci_chip_rev_id == CHIPREV_ID_5701_A0 ||
13931             tp->pci_chip_rev_id == CHIPREV_ID_5701_B0 ||
13932             tp->pci_chip_rev_id == CHIPREV_ID_5701_B2) {
13933                 tg3_flag_clear(tp, WOL_SPEED_100MB);
13934         } else {
13935                 tg3_flag_set(tp, WOL_SPEED_100MB);
13936         }
13937
13938         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906)
13939                 tp->phy_flags |= TG3_PHYFLG_IS_FET;
13940
13941         /* A few boards don't want Ethernet@WireSpeed phy feature */
13942         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700) ||
13943             ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) &&
13944              (tp->pci_chip_rev_id != CHIPREV_ID_5705_A0) &&
13945              (tp->pci_chip_rev_id != CHIPREV_ID_5705_A1)) ||
13946             (tp->phy_flags & TG3_PHYFLG_IS_FET) ||
13947             (tp->phy_flags & TG3_PHYFLG_ANY_SERDES))
13948                 tp->phy_flags |= TG3_PHYFLG_NO_ETH_WIRE_SPEED;
13949
13950         if (GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5703_AX ||
13951             GET_CHIP_REV(tp->pci_chip_rev_id) == CHIPREV_5704_AX)
13952                 tp->phy_flags |= TG3_PHYFLG_ADC_BUG;
13953         if (tp->pci_chip_rev_id == CHIPREV_ID_5704_A0)
13954                 tp->phy_flags |= TG3_PHYFLG_5704_A0_BUG;
13955
13956         if (tg3_flag(tp, 5705_PLUS) &&
13957             !(tp->phy_flags & TG3_PHYFLG_IS_FET) &&
13958             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5785 &&
13959             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_57780 &&
13960             !tg3_flag(tp, 57765_PLUS)) {
13961                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755 ||
13962                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5787 ||
13963                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 ||
13964                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761) {
13965                         if (tp->pdev->device != PCI_DEVICE_ID_TIGON3_5756 &&
13966                             tp->pdev->device != PCI_DEVICE_ID_TIGON3_5722)
13967                                 tp->phy_flags |= TG3_PHYFLG_JITTER_BUG;
13968                         if (tp->pdev->device == PCI_DEVICE_ID_TIGON3_5755M)
13969                                 tp->phy_flags |= TG3_PHYFLG_ADJUST_TRIM;
13970                 } else
13971                         tp->phy_flags |= TG3_PHYFLG_BER_BUG;
13972         }
13973
13974         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 &&
13975             GET_CHIP_REV(tp->pci_chip_rev_id) != CHIPREV_5784_AX) {
13976                 tp->phy_otp = tg3_read_otp_phycfg(tp);
13977                 if (tp->phy_otp == 0)
13978                         tp->phy_otp = TG3_OTP_DEFAULT;
13979         }
13980
13981         if (tg3_flag(tp, CPMU_PRESENT))
13982                 tp->mi_mode = MAC_MI_MODE_500KHZ_CONST;
13983         else
13984                 tp->mi_mode = MAC_MI_MODE_BASE;
13985
13986         tp->coalesce_mode = 0;
13987         if (GET_CHIP_REV(tp->pci_chip_rev_id) != CHIPREV_5700_AX &&
13988             GET_CHIP_REV(tp->pci_chip_rev_id) != CHIPREV_5700_BX)
13989                 tp->coalesce_mode |= HOSTCC_MODE_32BYTE;
13990
13991         /* Set these bits to enable statistics workaround. */
13992         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5717 ||
13993             tp->pci_chip_rev_id == CHIPREV_ID_5719_A0 ||
13994             tp->pci_chip_rev_id == CHIPREV_ID_5720_A0) {
13995                 tp->coalesce_mode |= HOSTCC_MODE_ATTN;
13996                 tp->grc_mode |= GRC_MODE_IRQ_ON_FLOW_ATTN;
13997         }
13998
13999         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5785 ||
14000             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_57780)
14001                 tg3_flag_set(tp, USE_PHYLIB);
14002
14003         err = tg3_mdio_init(tp);
14004         if (err)
14005                 return err;
14006
14007         /* Initialize data/descriptor byte/word swapping. */
14008         val = tr32(GRC_MODE);
14009         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5720)
14010                 val &= (GRC_MODE_BYTE_SWAP_B2HRX_DATA |
14011                         GRC_MODE_WORD_SWAP_B2HRX_DATA |
14012                         GRC_MODE_B2HRX_ENABLE |
14013                         GRC_MODE_HTX2B_ENABLE |
14014                         GRC_MODE_HOST_STACKUP);
14015         else
14016                 val &= GRC_MODE_HOST_STACKUP;
14017
14018         tw32(GRC_MODE, val | tp->grc_mode);
14019
14020         tg3_switch_clocks(tp);
14021
14022         /* Clear this out for sanity. */
14023         tw32(TG3PCI_MEM_WIN_BASE_ADDR, 0);
14024
14025         pci_read_config_dword(tp->pdev, TG3PCI_PCISTATE,
14026                               &pci_state_reg);
14027         if ((pci_state_reg & PCISTATE_CONV_PCI_MODE) == 0 &&
14028             !tg3_flag(tp, PCIX_TARGET_HWBUG)) {
14029                 u32 chiprevid = GET_CHIP_REV_ID(tp->misc_host_ctrl);
14030
14031                 if (chiprevid == CHIPREV_ID_5701_A0 ||
14032                     chiprevid == CHIPREV_ID_5701_B0 ||
14033                     chiprevid == CHIPREV_ID_5701_B2 ||
14034                     chiprevid == CHIPREV_ID_5701_B5) {
14035                         void __iomem *sram_base;
14036
14037                         /* Write some dummy words into the SRAM status block
14038                          * area, see if it reads back correctly.  If the return
14039                          * value is bad, force enable the PCIX workaround.
14040                          */
14041                         sram_base = tp->regs + NIC_SRAM_WIN_BASE + NIC_SRAM_STATS_BLK;
14042
14043                         writel(0x00000000, sram_base);
14044                         writel(0x00000000, sram_base + 4);
14045                         writel(0xffffffff, sram_base + 4);
14046                         if (readl(sram_base) != 0x00000000)
14047                                 tg3_flag_set(tp, PCIX_TARGET_HWBUG);
14048                 }
14049         }
14050
14051         udelay(50);
14052         tg3_nvram_init(tp);
14053
14054         grc_misc_cfg = tr32(GRC_MISC_CFG);
14055         grc_misc_cfg &= GRC_MISC_CFG_BOARD_ID_MASK;
14056
14057         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705 &&
14058             (grc_misc_cfg == GRC_MISC_CFG_BOARD_ID_5788 ||
14059              grc_misc_cfg == GRC_MISC_CFG_BOARD_ID_5788M))
14060                 tg3_flag_set(tp, IS_5788);
14061
14062         if (!tg3_flag(tp, IS_5788) &&
14063             (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700))
14064                 tg3_flag_set(tp, TAGGED_STATUS);
14065         if (tg3_flag(tp, TAGGED_STATUS)) {
14066                 tp->coalesce_mode |= (HOSTCC_MODE_CLRTICK_RXBD |
14067                                       HOSTCC_MODE_CLRTICK_TXBD);
14068
14069                 tp->misc_host_ctrl |= MISC_HOST_CTRL_TAGGED_STATUS;
14070                 pci_write_config_dword(tp->pdev, TG3PCI_MISC_HOST_CTRL,
14071                                        tp->misc_host_ctrl);
14072         }
14073
14074         /* Preserve the APE MAC_MODE bits */
14075         if (tg3_flag(tp, ENABLE_APE))
14076                 tp->mac_mode = MAC_MODE_APE_TX_EN | MAC_MODE_APE_RX_EN;
14077         else
14078                 tp->mac_mode = TG3_DEF_MAC_MODE;
14079
14080         /* these are limited to 10/100 only */
14081         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 &&
14082              (grc_misc_cfg == 0x8000 || grc_misc_cfg == 0x4000)) ||
14083             (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705 &&
14084              tp->pdev->vendor == PCI_VENDOR_ID_BROADCOM &&
14085              (tp->pdev->device == PCI_DEVICE_ID_TIGON3_5901 ||
14086               tp->pdev->device == PCI_DEVICE_ID_TIGON3_5901_2 ||
14087               tp->pdev->device == PCI_DEVICE_ID_TIGON3_5705F)) ||
14088             (tp->pdev->vendor == PCI_VENDOR_ID_BROADCOM &&
14089              (tp->pdev->device == PCI_DEVICE_ID_TIGON3_5751F ||
14090               tp->pdev->device == PCI_DEVICE_ID_TIGON3_5753F ||
14091               tp->pdev->device == PCI_DEVICE_ID_TIGON3_5787F)) ||
14092             tp->pdev->device == TG3PCI_DEVICE_TIGON3_57790 ||
14093             tp->pdev->device == TG3PCI_DEVICE_TIGON3_57791 ||
14094             tp->pdev->device == TG3PCI_DEVICE_TIGON3_57795 ||
14095             (tp->phy_flags & TG3_PHYFLG_IS_FET))
14096                 tp->phy_flags |= TG3_PHYFLG_10_100_ONLY;
14097
14098         err = tg3_phy_probe(tp);
14099         if (err) {
14100                 dev_err(&tp->pdev->dev, "phy probe failed, err %d\n", err);
14101                 /* ... but do not return immediately ... */
14102                 tg3_mdio_fini(tp);
14103         }
14104
14105         tg3_read_vpd(tp);
14106         tg3_read_fw_ver(tp);
14107
14108         if (tp->phy_flags & TG3_PHYFLG_PHY_SERDES) {
14109                 tp->phy_flags &= ~TG3_PHYFLG_USE_MI_INTERRUPT;
14110         } else {
14111                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700)
14112                         tp->phy_flags |= TG3_PHYFLG_USE_MI_INTERRUPT;
14113                 else
14114                         tp->phy_flags &= ~TG3_PHYFLG_USE_MI_INTERRUPT;
14115         }
14116
14117         /* 5700 {AX,BX} chips have a broken status block link
14118          * change bit implementation, so we must use the
14119          * status register in those cases.
14120          */
14121         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700)
14122                 tg3_flag_set(tp, USE_LINKCHG_REG);
14123         else
14124                 tg3_flag_clear(tp, USE_LINKCHG_REG);
14125
14126         /* The led_ctrl is set during tg3_phy_probe, here we might
14127          * have to force the link status polling mechanism based
14128          * upon subsystem IDs.
14129          */
14130         if (tp->pdev->subsystem_vendor == PCI_VENDOR_ID_DELL &&
14131             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701 &&
14132             !(tp->phy_flags & TG3_PHYFLG_PHY_SERDES)) {
14133                 tp->phy_flags |= TG3_PHYFLG_USE_MI_INTERRUPT;
14134                 tg3_flag_set(tp, USE_LINKCHG_REG);
14135         }
14136
14137         /* For all SERDES we poll the MAC status register. */
14138         if (tp->phy_flags & TG3_PHYFLG_PHY_SERDES)
14139                 tg3_flag_set(tp, POLL_SERDES);
14140         else
14141                 tg3_flag_clear(tp, POLL_SERDES);
14142
14143         tp->rx_offset = NET_IP_ALIGN;
14144         tp->rx_copy_thresh = TG3_RX_COPY_THRESHOLD;
14145         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701 &&
14146             tg3_flag(tp, PCIX_MODE)) {
14147                 tp->rx_offset = 0;
14148 #ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
14149                 tp->rx_copy_thresh = ~(u16)0;
14150 #endif
14151         }
14152
14153         tp->rx_std_ring_mask = TG3_RX_STD_RING_SIZE(tp) - 1;
14154         tp->rx_jmb_ring_mask = TG3_RX_JMB_RING_SIZE(tp) - 1;
14155         tp->rx_ret_ring_mask = tg3_rx_ret_ring_size(tp) - 1;
14156
14157         tp->rx_std_max_post = tp->rx_std_ring_mask + 1;
14158
14159         /* Increment the rx prod index on the rx std ring by at most
14160          * 8 for these chips to workaround hw errata.
14161          */
14162         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5750 ||
14163             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5752 ||
14164             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5755)
14165                 tp->rx_std_max_post = 8;
14166
14167         if (tg3_flag(tp, ASPM_WORKAROUND))
14168                 tp->pwrmgmt_thresh = tr32(PCIE_PWR_MGMT_THRESH) &
14169                                      PCIE_PWR_MGMT_L1_THRESH_MSK;
14170
14171         return err;
14172 }
14173
14174 #ifdef CONFIG_SPARC
14175 static int __devinit tg3_get_macaddr_sparc(struct tg3 *tp)
14176 {
14177         struct net_device *dev = tp->dev;
14178         struct pci_dev *pdev = tp->pdev;
14179         struct device_node *dp = pci_device_to_OF_node(pdev);
14180         const unsigned char *addr;
14181         int len;
14182
14183         addr = of_get_property(dp, "local-mac-address", &len);
14184         if (addr && len == 6) {
14185                 memcpy(dev->dev_addr, addr, 6);
14186                 memcpy(dev->perm_addr, dev->dev_addr, 6);
14187                 return 0;
14188         }
14189         return -ENODEV;
14190 }
14191
14192 static int __devinit tg3_get_default_macaddr_sparc(struct tg3 *tp)
14193 {
14194         struct net_device *dev = tp->dev;
14195
14196         memcpy(dev->dev_addr, idprom->id_ethaddr, 6);
14197         memcpy(dev->perm_addr, idprom->id_ethaddr, 6);
14198         return 0;
14199 }
14200 #endif
14201
14202 static int __devinit tg3_get_device_address(struct tg3 *tp)
14203 {
14204         struct net_device *dev = tp->dev;
14205         u32 hi, lo, mac_offset;
14206         int addr_ok = 0;
14207
14208 #ifdef CONFIG_SPARC
14209         if (!tg3_get_macaddr_sparc(tp))
14210                 return 0;
14211 #endif
14212
14213         mac_offset = 0x7c;
14214         if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) ||
14215             tg3_flag(tp, 5780_CLASS)) {
14216                 if (tr32(TG3PCI_DUAL_MAC_CTRL) & DUAL_MAC_CTRL_ID)
14217                         mac_offset = 0xcc;
14218                 if (tg3_nvram_lock(tp))
14219                         tw32_f(NVRAM_CMD, NVRAM_CMD_RESET);
14220                 else
14221                         tg3_nvram_unlock(tp);
14222         } else if (tg3_flag(tp, 5717_PLUS)) {
14223                 if (PCI_FUNC(tp->pdev->devfn) & 1)
14224                         mac_offset = 0xcc;
14225                 if (PCI_FUNC(tp->pdev->devfn) > 1)
14226                         mac_offset += 0x18c;
14227         } else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906)
14228                 mac_offset = 0x10;
14229
14230         /* First try to get it from MAC address mailbox. */
14231         tg3_read_mem(tp, NIC_SRAM_MAC_ADDR_HIGH_MBOX, &hi);
14232         if ((hi >> 16) == 0x484b) {
14233                 dev->dev_addr[0] = (hi >>  8) & 0xff;
14234                 dev->dev_addr[1] = (hi >>  0) & 0xff;
14235
14236                 tg3_read_mem(tp, NIC_SRAM_MAC_ADDR_LOW_MBOX, &lo);
14237                 dev->dev_addr[2] = (lo >> 24) & 0xff;
14238                 dev->dev_addr[3] = (lo >> 16) & 0xff;
14239                 dev->dev_addr[4] = (lo >>  8) & 0xff;
14240                 dev->dev_addr[5] = (lo >>  0) & 0xff;
14241
14242                 /* Some old bootcode may report a 0 MAC address in SRAM */
14243                 addr_ok = is_valid_ether_addr(&dev->dev_addr[0]);
14244         }
14245         if (!addr_ok) {
14246                 /* Next, try NVRAM. */
14247                 if (!tg3_flag(tp, NO_NVRAM) &&
14248                     !tg3_nvram_read_be32(tp, mac_offset + 0, &hi) &&
14249                     !tg3_nvram_read_be32(tp, mac_offset + 4, &lo)) {
14250                         memcpy(&dev->dev_addr[0], ((char *)&hi) + 2, 2);
14251                         memcpy(&dev->dev_addr[2], (char *)&lo, sizeof(lo));
14252                 }
14253                 /* Finally just fetch it out of the MAC control regs. */
14254                 else {
14255                         hi = tr32(MAC_ADDR_0_HIGH);
14256                         lo = tr32(MAC_ADDR_0_LOW);
14257
14258                         dev->dev_addr[5] = lo & 0xff;
14259                         dev->dev_addr[4] = (lo >> 8) & 0xff;
14260                         dev->dev_addr[3] = (lo >> 16) & 0xff;
14261                         dev->dev_addr[2] = (lo >> 24) & 0xff;
14262                         dev->dev_addr[1] = hi & 0xff;
14263                         dev->dev_addr[0] = (hi >> 8) & 0xff;
14264                 }
14265         }
14266
14267         if (!is_valid_ether_addr(&dev->dev_addr[0])) {
14268 #ifdef CONFIG_SPARC
14269                 if (!tg3_get_default_macaddr_sparc(tp))
14270                         return 0;
14271 #endif
14272                 return -EINVAL;
14273         }
14274         memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len);
14275         return 0;
14276 }
14277
14278 #define BOUNDARY_SINGLE_CACHELINE       1
14279 #define BOUNDARY_MULTI_CACHELINE        2
14280
14281 static u32 __devinit tg3_calc_dma_bndry(struct tg3 *tp, u32 val)
14282 {
14283         int cacheline_size;
14284         u8 byte;
14285         int goal;
14286
14287         pci_read_config_byte(tp->pdev, PCI_CACHE_LINE_SIZE, &byte);
14288         if (byte == 0)
14289                 cacheline_size = 1024;
14290         else
14291                 cacheline_size = (int) byte * 4;
14292
14293         /* On 5703 and later chips, the boundary bits have no
14294          * effect.
14295          */
14296         if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700 &&
14297             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5701 &&
14298             !tg3_flag(tp, PCI_EXPRESS))
14299                 goto out;
14300
14301 #if defined(CONFIG_PPC64) || defined(CONFIG_IA64) || defined(CONFIG_PARISC)
14302         goal = BOUNDARY_MULTI_CACHELINE;
14303 #else
14304 #if defined(CONFIG_SPARC64) || defined(CONFIG_ALPHA)
14305         goal = BOUNDARY_SINGLE_CACHELINE;
14306 #else
14307         goal = 0;
14308 #endif
14309 #endif
14310
14311         if (tg3_flag(tp, 57765_PLUS)) {
14312                 val = goal ? 0 : DMA_RWCTRL_DIS_CACHE_ALIGNMENT;
14313                 goto out;
14314         }
14315
14316         if (!goal)
14317                 goto out;
14318
14319         /* PCI controllers on most RISC systems tend to disconnect
14320          * when a device tries to burst across a cache-line boundary.
14321          * Therefore, letting tg3 do so just wastes PCI bandwidth.
14322          *
14323          * Unfortunately, for PCI-E there are only limited
14324          * write-side controls for this, and thus for reads
14325          * we will still get the disconnects.  We'll also waste
14326          * these PCI cycles for both read and write for chips
14327          * other than 5700 and 5701 which do not implement the
14328          * boundary bits.
14329          */
14330         if (tg3_flag(tp, PCIX_MODE) && !tg3_flag(tp, PCI_EXPRESS)) {
14331                 switch (cacheline_size) {
14332                 case 16:
14333                 case 32:
14334                 case 64:
14335                 case 128:
14336                         if (goal == BOUNDARY_SINGLE_CACHELINE) {
14337                                 val |= (DMA_RWCTRL_READ_BNDRY_128_PCIX |
14338                                         DMA_RWCTRL_WRITE_BNDRY_128_PCIX);
14339                         } else {
14340                                 val |= (DMA_RWCTRL_READ_BNDRY_384_PCIX |
14341                                         DMA_RWCTRL_WRITE_BNDRY_384_PCIX);
14342                         }
14343                         break;
14344
14345                 case 256:
14346                         val |= (DMA_RWCTRL_READ_BNDRY_256_PCIX |
14347                                 DMA_RWCTRL_WRITE_BNDRY_256_PCIX);
14348                         break;
14349
14350                 default:
14351                         val |= (DMA_RWCTRL_READ_BNDRY_384_PCIX |
14352                                 DMA_RWCTRL_WRITE_BNDRY_384_PCIX);
14353                         break;
14354                 }
14355         } else if (tg3_flag(tp, PCI_EXPRESS)) {
14356                 switch (cacheline_size) {
14357                 case 16:
14358                 case 32:
14359                 case 64:
14360                         if (goal == BOUNDARY_SINGLE_CACHELINE) {
14361                                 val &= ~DMA_RWCTRL_WRITE_BNDRY_DISAB_PCIE;
14362                                 val |= DMA_RWCTRL_WRITE_BNDRY_64_PCIE;
14363                                 break;
14364                         }
14365                         /* fallthrough */
14366                 case 128:
14367                 default:
14368                         val &= ~DMA_RWCTRL_WRITE_BNDRY_DISAB_PCIE;
14369                         val |= DMA_RWCTRL_WRITE_BNDRY_128_PCIE;
14370                         break;
14371                 }
14372         } else {
14373                 switch (cacheline_size) {
14374                 case 16:
14375                         if (goal == BOUNDARY_SINGLE_CACHELINE) {
14376                                 val |= (DMA_RWCTRL_READ_BNDRY_16 |
14377                                         DMA_RWCTRL_WRITE_BNDRY_16);
14378                                 break;
14379                         }
14380                         /* fallthrough */
14381                 case 32:
14382                         if (goal == BOUNDARY_SINGLE_CACHELINE) {
14383                                 val |= (DMA_RWCTRL_READ_BNDRY_32 |
14384                                         DMA_RWCTRL_WRITE_BNDRY_32);
14385                                 break;
14386                         }
14387                         /* fallthrough */
14388                 case 64:
14389                         if (goal == BOUNDARY_SINGLE_CACHELINE) {
14390                                 val |= (DMA_RWCTRL_READ_BNDRY_64 |
14391                                         DMA_RWCTRL_WRITE_BNDRY_64);
14392                                 break;
14393                         }
14394                         /* fallthrough */
14395                 case 128:
14396                         if (goal == BOUNDARY_SINGLE_CACHELINE) {
14397                                 val |= (DMA_RWCTRL_READ_BNDRY_128 |
14398                                         DMA_RWCTRL_WRITE_BNDRY_128);
14399                                 break;
14400                         }
14401                         /* fallthrough */
14402                 case 256:
14403                         val |= (DMA_RWCTRL_READ_BNDRY_256 |
14404                                 DMA_RWCTRL_WRITE_BNDRY_256);
14405                         break;
14406                 case 512:
14407                         val |= (DMA_RWCTRL_READ_BNDRY_512 |
14408                                 DMA_RWCTRL_WRITE_BNDRY_512);
14409                         break;
14410                 case 1024:
14411                 default:
14412                         val |= (DMA_RWCTRL_READ_BNDRY_1024 |
14413                                 DMA_RWCTRL_WRITE_BNDRY_1024);
14414                         break;
14415                 }
14416         }
14417
14418 out:
14419         return val;
14420 }
14421
14422 static int __devinit tg3_do_test_dma(struct tg3 *tp, u32 *buf, dma_addr_t buf_dma, int size, int to_device)
14423 {
14424         struct tg3_internal_buffer_desc test_desc;
14425         u32 sram_dma_descs;
14426         int i, ret;
14427
14428         sram_dma_descs = NIC_SRAM_DMA_DESC_POOL_BASE;
14429
14430         tw32(FTQ_RCVBD_COMP_FIFO_ENQDEQ, 0);
14431         tw32(FTQ_RCVDATA_COMP_FIFO_ENQDEQ, 0);
14432         tw32(RDMAC_STATUS, 0);
14433         tw32(WDMAC_STATUS, 0);
14434
14435         tw32(BUFMGR_MODE, 0);
14436         tw32(FTQ_RESET, 0);
14437
14438         test_desc.addr_hi = ((u64) buf_dma) >> 32;
14439         test_desc.addr_lo = buf_dma & 0xffffffff;
14440         test_desc.nic_mbuf = 0x00002100;
14441         test_desc.len = size;
14442
14443         /*
14444          * HP ZX1 was seeing test failures for 5701 cards running at 33Mhz
14445          * the *second* time the tg3 driver was getting loaded after an
14446          * initial scan.
14447          *
14448          * Broadcom tells me:
14449          *   ...the DMA engine is connected to the GRC block and a DMA
14450          *   reset may affect the GRC block in some unpredictable way...
14451          *   The behavior of resets to individual blocks has not been tested.
14452          *
14453          * Broadcom noted the GRC reset will also reset all sub-components.
14454          */
14455         if (to_device) {
14456                 test_desc.cqid_sqid = (13 << 8) | 2;
14457
14458                 tw32_f(RDMAC_MODE, RDMAC_MODE_ENABLE);
14459                 udelay(40);
14460         } else {
14461                 test_desc.cqid_sqid = (16 << 8) | 7;
14462
14463                 tw32_f(WDMAC_MODE, WDMAC_MODE_ENABLE);
14464                 udelay(40);
14465         }
14466         test_desc.flags = 0x00000005;
14467
14468         for (i = 0; i < (sizeof(test_desc) / sizeof(u32)); i++) {
14469                 u32 val;
14470
14471                 val = *(((u32 *)&test_desc) + i);
14472                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR,
14473                                        sram_dma_descs + (i * sizeof(u32)));
14474                 pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_DATA, val);
14475         }
14476         pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, 0);
14477
14478         if (to_device)
14479                 tw32(FTQ_DMA_HIGH_READ_FIFO_ENQDEQ, sram_dma_descs);
14480         else
14481                 tw32(FTQ_DMA_HIGH_WRITE_FIFO_ENQDEQ, sram_dma_descs);
14482
14483         ret = -ENODEV;
14484         for (i = 0; i < 40; i++) {
14485                 u32 val;
14486
14487                 if (to_device)
14488                         val = tr32(FTQ_RCVBD_COMP_FIFO_ENQDEQ);
14489                 else
14490                         val = tr32(FTQ_RCVDATA_COMP_FIFO_ENQDEQ);
14491                 if ((val & 0xffff) == sram_dma_descs) {
14492                         ret = 0;
14493                         break;
14494                 }
14495
14496                 udelay(100);
14497         }
14498
14499         return ret;
14500 }
14501
14502 #define TEST_BUFFER_SIZE        0x2000
14503
14504 static DEFINE_PCI_DEVICE_TABLE(tg3_dma_wait_state_chipsets) = {
14505         { PCI_DEVICE(PCI_VENDOR_ID_APPLE, PCI_DEVICE_ID_APPLE_UNI_N_PCI15) },
14506         { },
14507 };
14508
14509 static int __devinit tg3_test_dma(struct tg3 *tp)
14510 {
14511         dma_addr_t buf_dma;
14512         u32 *buf, saved_dma_rwctrl;
14513         int ret = 0;
14514
14515         buf = dma_alloc_coherent(&tp->pdev->dev, TEST_BUFFER_SIZE,
14516                                  &buf_dma, GFP_KERNEL);
14517         if (!buf) {
14518                 ret = -ENOMEM;
14519                 goto out_nofree;
14520         }
14521
14522         tp->dma_rwctrl = ((0x7 << DMA_RWCTRL_PCI_WRITE_CMD_SHIFT) |
14523                           (0x6 << DMA_RWCTRL_PCI_READ_CMD_SHIFT));
14524
14525         tp->dma_rwctrl = tg3_calc_dma_bndry(tp, tp->dma_rwctrl);
14526
14527         if (tg3_flag(tp, 57765_PLUS))
14528                 goto out;
14529
14530         if (tg3_flag(tp, PCI_EXPRESS)) {
14531                 /* DMA read watermark not used on PCIE */
14532                 tp->dma_rwctrl |= 0x00180000;
14533         } else if (!tg3_flag(tp, PCIX_MODE)) {
14534                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705 ||
14535                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5750)
14536                         tp->dma_rwctrl |= 0x003f0000;
14537                 else
14538                         tp->dma_rwctrl |= 0x003f000f;
14539         } else {
14540                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
14541                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) {
14542                         u32 ccval = (tr32(TG3PCI_CLOCK_CTRL) & 0x1f);
14543                         u32 read_water = 0x7;
14544
14545                         /* If the 5704 is behind the EPB bridge, we can
14546                          * do the less restrictive ONE_DMA workaround for
14547                          * better performance.
14548                          */
14549                         if (tg3_flag(tp, 40BIT_DMA_BUG) &&
14550                             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704)
14551                                 tp->dma_rwctrl |= 0x8000;
14552                         else if (ccval == 0x6 || ccval == 0x7)
14553                                 tp->dma_rwctrl |= DMA_RWCTRL_ONE_DMA;
14554
14555                         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703)
14556                                 read_water = 4;
14557                         /* Set bit 23 to enable PCIX hw bug fix */
14558                         tp->dma_rwctrl |=
14559                                 (read_water << DMA_RWCTRL_READ_WATER_SHIFT) |
14560                                 (0x3 << DMA_RWCTRL_WRITE_WATER_SHIFT) |
14561                                 (1 << 23);
14562                 } else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5780) {
14563                         /* 5780 always in PCIX mode */
14564                         tp->dma_rwctrl |= 0x00144000;
14565                 } else if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5714) {
14566                         /* 5714 always in PCIX mode */
14567                         tp->dma_rwctrl |= 0x00148000;
14568                 } else {
14569                         tp->dma_rwctrl |= 0x001b000f;
14570                 }
14571         }
14572
14573         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
14574             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704)
14575                 tp->dma_rwctrl &= 0xfffffff0;
14576
14577         if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
14578             GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) {
14579                 /* Remove this if it causes problems for some boards. */
14580                 tp->dma_rwctrl |= DMA_RWCTRL_USE_MEM_READ_MULT;
14581
14582                 /* On 5700/5701 chips, we need to set this bit.
14583                  * Otherwise the chip will issue cacheline transactions
14584                  * to streamable DMA memory with not all the byte
14585                  * enables turned on.  This is an error on several
14586                  * RISC PCI controllers, in particular sparc64.
14587                  *
14588                  * On 5703/5704 chips, this bit has been reassigned
14589                  * a different meaning.  In particular, it is used
14590                  * on those chips to enable a PCI-X workaround.
14591                  */
14592                 tp->dma_rwctrl |= DMA_RWCTRL_ASSERT_ALL_BE;
14593         }
14594
14595         tw32(TG3PCI_DMA_RW_CTRL, tp->dma_rwctrl);
14596
14597 #if 0
14598         /* Unneeded, already done by tg3_get_invariants.  */
14599         tg3_switch_clocks(tp);
14600 #endif
14601
14602         if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700 &&
14603             GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5701)
14604                 goto out;
14605
14606         /* It is best to perform DMA test with maximum write burst size
14607          * to expose the 5700/5701 write DMA bug.
14608          */
14609         saved_dma_rwctrl = tp->dma_rwctrl;
14610         tp->dma_rwctrl &= ~DMA_RWCTRL_WRITE_BNDRY_MASK;
14611         tw32(TG3PCI_DMA_RW_CTRL, tp->dma_rwctrl);
14612
14613         while (1) {
14614                 u32 *p = buf, i;
14615
14616                 for (i = 0; i < TEST_BUFFER_SIZE / sizeof(u32); i++)
14617                         p[i] = i;
14618
14619                 /* Send the buffer to the chip. */
14620                 ret = tg3_do_test_dma(tp, buf, buf_dma, TEST_BUFFER_SIZE, 1);
14621                 if (ret) {
14622                         dev_err(&tp->pdev->dev,
14623                                 "%s: Buffer write failed. err = %d\n",
14624                                 __func__, ret);
14625                         break;
14626                 }
14627
14628 #if 0
14629                 /* validate data reached card RAM correctly. */
14630                 for (i = 0; i < TEST_BUFFER_SIZE / sizeof(u32); i++) {
14631                         u32 val;
14632                         tg3_read_mem(tp, 0x2100 + (i*4), &val);
14633                         if (le32_to_cpu(val) != p[i]) {
14634                                 dev_err(&tp->pdev->dev,
14635                                         "%s: Buffer corrupted on device! "
14636                                         "(%d != %d)\n", __func__, val, i);
14637                                 /* ret = -ENODEV here? */
14638                         }
14639                         p[i] = 0;
14640                 }
14641 #endif
14642                 /* Now read it back. */
14643                 ret = tg3_do_test_dma(tp, buf, buf_dma, TEST_BUFFER_SIZE, 0);
14644                 if (ret) {
14645                         dev_err(&tp->pdev->dev, "%s: Buffer read failed. "
14646                                 "err = %d\n", __func__, ret);
14647                         break;
14648                 }
14649
14650                 /* Verify it. */
14651                 for (i = 0; i < TEST_BUFFER_SIZE / sizeof(u32); i++) {
14652                         if (p[i] == i)
14653                                 continue;
14654
14655                         if ((tp->dma_rwctrl & DMA_RWCTRL_WRITE_BNDRY_MASK) !=
14656                             DMA_RWCTRL_WRITE_BNDRY_16) {
14657                                 tp->dma_rwctrl &= ~DMA_RWCTRL_WRITE_BNDRY_MASK;
14658                                 tp->dma_rwctrl |= DMA_RWCTRL_WRITE_BNDRY_16;
14659                                 tw32(TG3PCI_DMA_RW_CTRL, tp->dma_rwctrl);
14660                                 break;
14661                         } else {
14662                                 dev_err(&tp->pdev->dev,
14663                                         "%s: Buffer corrupted on read back! "
14664                                         "(%d != %d)\n", __func__, p[i], i);
14665                                 ret = -ENODEV;
14666                                 goto out;
14667                         }
14668                 }
14669
14670                 if (i == (TEST_BUFFER_SIZE / sizeof(u32))) {
14671                         /* Success. */
14672                         ret = 0;
14673                         break;
14674                 }
14675         }
14676         if ((tp->dma_rwctrl & DMA_RWCTRL_WRITE_BNDRY_MASK) !=
14677             DMA_RWCTRL_WRITE_BNDRY_16) {
14678                 /* DMA test passed without adjusting DMA boundary,
14679                  * now look for chipsets that are known to expose the
14680                  * DMA bug without failing the test.
14681                  */
14682                 if (pci_dev_present(tg3_dma_wait_state_chipsets)) {
14683                         tp->dma_rwctrl &= ~DMA_RWCTRL_WRITE_BNDRY_MASK;
14684                         tp->dma_rwctrl |= DMA_RWCTRL_WRITE_BNDRY_16;
14685                 } else {
14686                         /* Safe to use the calculated DMA boundary. */
14687                         tp->dma_rwctrl = saved_dma_rwctrl;
14688                 }
14689
14690                 tw32(TG3PCI_DMA_RW_CTRL, tp->dma_rwctrl);
14691         }
14692
14693 out:
14694         dma_free_coherent(&tp->pdev->dev, TEST_BUFFER_SIZE, buf, buf_dma);
14695 out_nofree:
14696         return ret;
14697 }
14698
14699 static void __devinit tg3_init_bufmgr_config(struct tg3 *tp)
14700 {
14701         if (tg3_flag(tp, 57765_PLUS)) {
14702                 tp->bufmgr_config.mbuf_read_dma_low_water =
14703                         DEFAULT_MB_RDMA_LOW_WATER_5705;
14704                 tp->bufmgr_config.mbuf_mac_rx_low_water =
14705                         DEFAULT_MB_MACRX_LOW_WATER_57765;
14706                 tp->bufmgr_config.mbuf_high_water =
14707                         DEFAULT_MB_HIGH_WATER_57765;
14708
14709                 tp->bufmgr_config.mbuf_read_dma_low_water_jumbo =
14710                         DEFAULT_MB_RDMA_LOW_WATER_5705;
14711                 tp->bufmgr_config.mbuf_mac_rx_low_water_jumbo =
14712                         DEFAULT_MB_MACRX_LOW_WATER_JUMBO_57765;
14713                 tp->bufmgr_config.mbuf_high_water_jumbo =
14714                         DEFAULT_MB_HIGH_WATER_JUMBO_57765;
14715         } else if (tg3_flag(tp, 5705_PLUS)) {
14716                 tp->bufmgr_config.mbuf_read_dma_low_water =
14717                         DEFAULT_MB_RDMA_LOW_WATER_5705;
14718                 tp->bufmgr_config.mbuf_mac_rx_low_water =
14719                         DEFAULT_MB_MACRX_LOW_WATER_5705;
14720                 tp->bufmgr_config.mbuf_high_water =
14721                         DEFAULT_MB_HIGH_WATER_5705;
14722                 if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5906) {
14723                         tp->bufmgr_config.mbuf_mac_rx_low_water =
14724                                 DEFAULT_MB_MACRX_LOW_WATER_5906;
14725                         tp->bufmgr_config.mbuf_high_water =
14726                                 DEFAULT_MB_HIGH_WATER_5906;
14727                 }
14728
14729                 tp->bufmgr_config.mbuf_read_dma_low_water_jumbo =
14730                         DEFAULT_MB_RDMA_LOW_WATER_JUMBO_5780;
14731                 tp->bufmgr_config.mbuf_mac_rx_low_water_jumbo =
14732                         DEFAULT_MB_MACRX_LOW_WATER_JUMBO_5780;
14733                 tp->bufmgr_config.mbuf_high_water_jumbo =
14734                         DEFAULT_MB_HIGH_WATER_JUMBO_5780;
14735         } else {
14736                 tp->bufmgr_config.mbuf_read_dma_low_water =
14737                         DEFAULT_MB_RDMA_LOW_WATER;
14738                 tp->bufmgr_config.mbuf_mac_rx_low_water =
14739                         DEFAULT_MB_MACRX_LOW_WATER;
14740                 tp->bufmgr_config.mbuf_high_water =
14741                         DEFAULT_MB_HIGH_WATER;
14742
14743                 tp->bufmgr_config.mbuf_read_dma_low_water_jumbo =
14744                         DEFAULT_MB_RDMA_LOW_WATER_JUMBO;
14745                 tp->bufmgr_config.mbuf_mac_rx_low_water_jumbo =
14746                         DEFAULT_MB_MACRX_LOW_WATER_JUMBO;
14747                 tp->bufmgr_config.mbuf_high_water_jumbo =
14748                         DEFAULT_MB_HIGH_WATER_JUMBO;
14749         }
14750
14751         tp->bufmgr_config.dma_low_water = DEFAULT_DMA_LOW_WATER;
14752         tp->bufmgr_config.dma_high_water = DEFAULT_DMA_HIGH_WATER;
14753 }
14754
14755 static char * __devinit tg3_phy_string(struct tg3 *tp)
14756 {
14757         switch (tp->phy_id & TG3_PHY_ID_MASK) {
14758         case TG3_PHY_ID_BCM5400:        return "5400";
14759         case TG3_PHY_ID_BCM5401:        return "5401";
14760         case TG3_PHY_ID_BCM5411:        return "5411";
14761         case TG3_PHY_ID_BCM5701:        return "5701";
14762         case TG3_PHY_ID_BCM5703:        return "5703";
14763         case TG3_PHY_ID_BCM5704:        return "5704";
14764         case TG3_PHY_ID_BCM5705:        return "5705";
14765         case TG3_PHY_ID_BCM5750:        return "5750";
14766         case TG3_PHY_ID_BCM5752:        return "5752";
14767         case TG3_PHY_ID_BCM5714:        return "5714";
14768         case TG3_PHY_ID_BCM5780:        return "5780";
14769         case TG3_PHY_ID_BCM5755:        return "5755";
14770         case TG3_PHY_ID_BCM5787:        return "5787";
14771         case TG3_PHY_ID_BCM5784:        return "5784";
14772         case TG3_PHY_ID_BCM5756:        return "5722/5756";
14773         case TG3_PHY_ID_BCM5906:        return "5906";
14774         case TG3_PHY_ID_BCM5761:        return "5761";
14775         case TG3_PHY_ID_BCM5718C:       return "5718C";
14776         case TG3_PHY_ID_BCM5718S:       return "5718S";
14777         case TG3_PHY_ID_BCM57765:       return "57765";
14778         case TG3_PHY_ID_BCM5719C:       return "5719C";
14779         case TG3_PHY_ID_BCM5720C:       return "5720C";
14780         case TG3_PHY_ID_BCM8002:        return "8002/serdes";
14781         case 0:                 return "serdes";
14782         default:                return "unknown";
14783         }
14784 }
14785
14786 static char * __devinit tg3_bus_string(struct tg3 *tp, char *str)
14787 {
14788         if (tg3_flag(tp, PCI_EXPRESS)) {
14789                 strcpy(str, "PCI Express");
14790                 return str;
14791         } else if (tg3_flag(tp, PCIX_MODE)) {
14792                 u32 clock_ctrl = tr32(TG3PCI_CLOCK_CTRL) & 0x1f;
14793
14794                 strcpy(str, "PCIX:");
14795
14796                 if ((clock_ctrl == 7) ||
14797                     ((tr32(GRC_MISC_CFG) & GRC_MISC_CFG_BOARD_ID_MASK) ==
14798                      GRC_MISC_CFG_BOARD_ID_5704CIOBE))
14799                         strcat(str, "133MHz");
14800                 else if (clock_ctrl == 0)
14801                         strcat(str, "33MHz");
14802                 else if (clock_ctrl == 2)
14803                         strcat(str, "50MHz");
14804                 else if (clock_ctrl == 4)
14805                         strcat(str, "66MHz");
14806                 else if (clock_ctrl == 6)
14807                         strcat(str, "100MHz");
14808         } else {
14809                 strcpy(str, "PCI:");
14810                 if (tg3_flag(tp, PCI_HIGH_SPEED))
14811                         strcat(str, "66MHz");
14812                 else
14813                         strcat(str, "33MHz");
14814         }
14815         if (tg3_flag(tp, PCI_32BIT))
14816                 strcat(str, ":32-bit");
14817         else
14818                 strcat(str, ":64-bit");
14819         return str;
14820 }
14821
14822 static struct pci_dev * __devinit tg3_find_peer(struct tg3 *tp)
14823 {
14824         struct pci_dev *peer;
14825         unsigned int func, devnr = tp->pdev->devfn & ~7;
14826
14827         for (func = 0; func < 8; func++) {
14828                 peer = pci_get_slot(tp->pdev->bus, devnr | func);
14829                 if (peer && peer != tp->pdev)
14830                         break;
14831                 pci_dev_put(peer);
14832         }
14833         /* 5704 can be configured in single-port mode, set peer to
14834          * tp->pdev in that case.
14835          */
14836         if (!peer) {
14837                 peer = tp->pdev;
14838                 return peer;
14839         }
14840
14841         /*
14842          * We don't need to keep the refcount elevated; there's no way
14843          * to remove one half of this device without removing the other
14844          */
14845         pci_dev_put(peer);
14846
14847         return peer;
14848 }
14849
14850 static void __devinit tg3_init_coal(struct tg3 *tp)
14851 {
14852         struct ethtool_coalesce *ec = &tp->coal;
14853
14854         memset(ec, 0, sizeof(*ec));
14855         ec->cmd = ETHTOOL_GCOALESCE;
14856         ec->rx_coalesce_usecs = LOW_RXCOL_TICKS;
14857         ec->tx_coalesce_usecs = LOW_TXCOL_TICKS;
14858         ec->rx_max_coalesced_frames = LOW_RXMAX_FRAMES;
14859         ec->tx_max_coalesced_frames = LOW_TXMAX_FRAMES;
14860         ec->rx_coalesce_usecs_irq = DEFAULT_RXCOAL_TICK_INT;
14861         ec->tx_coalesce_usecs_irq = DEFAULT_TXCOAL_TICK_INT;
14862         ec->rx_max_coalesced_frames_irq = DEFAULT_RXCOAL_MAXF_INT;
14863         ec->tx_max_coalesced_frames_irq = DEFAULT_TXCOAL_MAXF_INT;
14864         ec->stats_block_coalesce_usecs = DEFAULT_STAT_COAL_TICKS;
14865
14866         if (tp->coalesce_mode & (HOSTCC_MODE_CLRTICK_RXBD |
14867                                  HOSTCC_MODE_CLRTICK_TXBD)) {
14868                 ec->rx_coalesce_usecs = LOW_RXCOL_TICKS_CLRTCKS;
14869                 ec->rx_coalesce_usecs_irq = DEFAULT_RXCOAL_TICK_INT_CLRTCKS;
14870                 ec->tx_coalesce_usecs = LOW_TXCOL_TICKS_CLRTCKS;
14871                 ec->tx_coalesce_usecs_irq = DEFAULT_TXCOAL_TICK_INT_CLRTCKS;
14872         }
14873
14874         if (tg3_flag(tp, 5705_PLUS)) {
14875                 ec->rx_coalesce_usecs_irq = 0;
14876                 ec->tx_coalesce_usecs_irq = 0;
14877                 ec->stats_block_coalesce_usecs = 0;
14878         }
14879 }
14880
14881 static const struct net_device_ops tg3_netdev_ops = {
14882         .ndo_open               = tg3_open,
14883         .ndo_stop               = tg3_close,
14884         .ndo_start_xmit         = tg3_start_xmit,
14885         .ndo_get_stats64        = tg3_get_stats64,
14886         .ndo_validate_addr      = eth_validate_addr,
14887         .ndo_set_multicast_list = tg3_set_rx_mode,
14888         .ndo_set_mac_address    = tg3_set_mac_addr,
14889         .ndo_do_ioctl           = tg3_ioctl,
14890         .ndo_tx_timeout         = tg3_tx_timeout,
14891         .ndo_change_mtu         = tg3_change_mtu,
14892         .ndo_fix_features       = tg3_fix_features,
14893         .ndo_set_features       = tg3_set_features,
14894 #ifdef CONFIG_NET_POLL_CONTROLLER
14895         .ndo_poll_controller    = tg3_poll_controller,
14896 #endif
14897 };
14898
14899 static int __devinit tg3_init_one(struct pci_dev *pdev,
14900                                   const struct pci_device_id *ent)
14901 {
14902         struct net_device *dev;
14903         struct tg3 *tp;
14904         int i, err, pm_cap;
14905         u32 sndmbx, rcvmbx, intmbx;
14906         char str[40];
14907         u64 dma_mask, persist_dma_mask;
14908         u32 hw_features = 0;
14909
14910         printk_once(KERN_INFO "%s\n", version);
14911
14912         err = pci_enable_device(pdev);
14913         if (err) {
14914                 dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n");
14915                 return err;
14916         }
14917
14918         err = pci_request_regions(pdev, DRV_MODULE_NAME);
14919         if (err) {
14920                 dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting\n");
14921                 goto err_out_disable_pdev;
14922         }
14923
14924         pci_set_master(pdev);
14925
14926         /* Find power-management capability. */
14927         pm_cap = pci_find_capability(pdev, PCI_CAP_ID_PM);
14928         if (pm_cap == 0) {
14929                 dev_err(&pdev->dev,
14930                         "Cannot find Power Management capability, aborting\n");
14931                 err = -EIO;
14932                 goto err_out_free_res;
14933         }
14934
14935         dev = alloc_etherdev_mq(sizeof(*tp), TG3_IRQ_MAX_VECS);
14936         if (!dev) {
14937                 dev_err(&pdev->dev, "Etherdev alloc failed, aborting\n");
14938                 err = -ENOMEM;
14939                 goto err_out_free_res;
14940         }
14941
14942         SET_NETDEV_DEV(dev, &pdev->dev);
14943
14944         dev->features |= NETIF_F_HW_VLAN_TX | NETIF_F_HW_VLAN_RX;
14945
14946         tp = netdev_priv(dev);
14947         tp->pdev = pdev;
14948         tp->dev = dev;
14949         tp->pm_cap = pm_cap;
14950         tp->rx_mode = TG3_DEF_RX_MODE;
14951         tp->tx_mode = TG3_DEF_TX_MODE;
14952
14953         if (tg3_debug > 0)
14954                 tp->msg_enable = tg3_debug;
14955         else
14956                 tp->msg_enable = TG3_DEF_MSG_ENABLE;
14957
14958         /* The word/byte swap controls here control register access byte
14959          * swapping.  DMA data byte swapping is controlled in the GRC_MODE
14960          * setting below.
14961          */
14962         tp->misc_host_ctrl =
14963                 MISC_HOST_CTRL_MASK_PCI_INT |
14964                 MISC_HOST_CTRL_WORD_SWAP |
14965                 MISC_HOST_CTRL_INDIR_ACCESS |
14966                 MISC_HOST_CTRL_PCISTATE_RW;
14967
14968         /* The NONFRM (non-frame) byte/word swap controls take effect
14969          * on descriptor entries, anything which isn't packet data.
14970          *
14971          * The StrongARM chips on the board (one for tx, one for rx)
14972          * are running in big-endian mode.
14973          */
14974         tp->grc_mode = (GRC_MODE_WSWAP_DATA | GRC_MODE_BSWAP_DATA |
14975                         GRC_MODE_WSWAP_NONFRM_DATA);
14976 #ifdef __BIG_ENDIAN
14977         tp->grc_mode |= GRC_MODE_BSWAP_NONFRM_DATA;
14978 #endif
14979         spin_lock_init(&tp->lock);
14980         spin_lock_init(&tp->indirect_lock);
14981         INIT_WORK(&tp->reset_task, tg3_reset_task);
14982
14983         tp->regs = pci_ioremap_bar(pdev, BAR_0);
14984         if (!tp->regs) {
14985                 dev_err(&pdev->dev, "Cannot map device registers, aborting\n");
14986                 err = -ENOMEM;
14987                 goto err_out_free_dev;
14988         }
14989
14990         tp->rx_pending = TG3_DEF_RX_RING_PENDING;
14991         tp->rx_jumbo_pending = TG3_DEF_RX_JUMBO_RING_PENDING;
14992
14993         dev->ethtool_ops = &tg3_ethtool_ops;
14994         dev->watchdog_timeo = TG3_TX_TIMEOUT;
14995         dev->netdev_ops = &tg3_netdev_ops;
14996         dev->irq = pdev->irq;
14997
14998         err = tg3_get_invariants(tp);
14999         if (err) {
15000                 dev_err(&pdev->dev,
15001                         "Problem fetching invariants of chip, aborting\n");
15002                 goto err_out_iounmap;
15003         }
15004
15005         /* The EPB bridge inside 5714, 5715, and 5780 and any
15006          * device behind the EPB cannot support DMA addresses > 40-bit.
15007          * On 64-bit systems with IOMMU, use 40-bit dma_mask.
15008          * On 64-bit systems without IOMMU, use 64-bit dma_mask and
15009          * do DMA address check in tg3_start_xmit().
15010          */
15011         if (tg3_flag(tp, IS_5788))
15012                 persist_dma_mask = dma_mask = DMA_BIT_MASK(32);
15013         else if (tg3_flag(tp, 40BIT_DMA_BUG)) {
15014                 persist_dma_mask = dma_mask = DMA_BIT_MASK(40);
15015 #ifdef CONFIG_HIGHMEM
15016                 dma_mask = DMA_BIT_MASK(64);
15017 #endif
15018         } else
15019                 persist_dma_mask = dma_mask = DMA_BIT_MASK(64);
15020
15021         /* Configure DMA attributes. */
15022         if (dma_mask > DMA_BIT_MASK(32)) {
15023                 err = pci_set_dma_mask(pdev, dma_mask);
15024                 if (!err) {
15025                         dev->features |= NETIF_F_HIGHDMA;
15026                         err = pci_set_consistent_dma_mask(pdev,
15027                                                           persist_dma_mask);
15028                         if (err < 0) {
15029                                 dev_err(&pdev->dev, "Unable to obtain 64 bit "
15030                                         "DMA for consistent allocations\n");
15031                                 goto err_out_iounmap;
15032                         }
15033                 }
15034         }
15035         if (err || dma_mask == DMA_BIT_MASK(32)) {
15036                 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
15037                 if (err) {
15038                         dev_err(&pdev->dev,
15039                                 "No usable DMA configuration, aborting\n");
15040                         goto err_out_iounmap;
15041                 }
15042         }
15043
15044         tg3_init_bufmgr_config(tp);
15045
15046         /* Selectively allow TSO based on operating conditions */
15047         if ((tg3_flag(tp, HW_TSO_1) ||
15048              tg3_flag(tp, HW_TSO_2) ||
15049              tg3_flag(tp, HW_TSO_3)) ||
15050             (tp->fw_needed && !tg3_flag(tp, ENABLE_ASF)))
15051                 tg3_flag_set(tp, TSO_CAPABLE);
15052         else {
15053                 tg3_flag_clear(tp, TSO_CAPABLE);
15054                 tg3_flag_clear(tp, TSO_BUG);
15055                 tp->fw_needed = NULL;
15056         }
15057
15058         if (tp->pci_chip_rev_id == CHIPREV_ID_5701_A0)
15059                 tp->fw_needed = FIRMWARE_TG3;
15060
15061         /* TSO is on by default on chips that support hardware TSO.
15062          * Firmware TSO on older chips gives lower performance, so it
15063          * is off by default, but can be enabled using ethtool.
15064          */
15065         if ((tg3_flag(tp, HW_TSO_1) ||
15066              tg3_flag(tp, HW_TSO_2) ||
15067              tg3_flag(tp, HW_TSO_3)) &&
15068             (dev->features & NETIF_F_IP_CSUM))
15069                 hw_features |= NETIF_F_TSO;
15070         if (tg3_flag(tp, HW_TSO_2) || tg3_flag(tp, HW_TSO_3)) {
15071                 if (dev->features & NETIF_F_IPV6_CSUM)
15072                         hw_features |= NETIF_F_TSO6;
15073                 if (tg3_flag(tp, HW_TSO_3) ||
15074                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5761 ||
15075                     (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5784 &&
15076                      GET_CHIP_REV(tp->pci_chip_rev_id) != CHIPREV_5784_AX) ||
15077                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5785 ||
15078                     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_57780)
15079                         hw_features |= NETIF_F_TSO_ECN;
15080         }
15081
15082         dev->hw_features |= hw_features;
15083         dev->features |= hw_features;
15084         dev->vlan_features |= hw_features;
15085
15086         /*
15087          * Add loopback capability only for a subset of devices that support
15088          * MAC-LOOPBACK. Eventually this need to be enhanced to allow INT-PHY
15089          * loopback for the remaining devices.
15090          */
15091         if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5780 &&
15092             !tg3_flag(tp, CPMU_PRESENT))
15093                 /* Add the loopback capability */
15094                 dev->hw_features |= NETIF_F_LOOPBACK;
15095
15096         if (tp->pci_chip_rev_id == CHIPREV_ID_5705_A1 &&
15097             !tg3_flag(tp, TSO_CAPABLE) &&
15098             !(tr32(TG3PCI_PCISTATE) & PCISTATE_BUS_SPEED_HIGH)) {
15099                 tg3_flag_set(tp, MAX_RXPEND_64);
15100                 tp->rx_pending = 63;
15101         }
15102
15103         err = tg3_get_device_address(tp);
15104         if (err) {
15105                 dev_err(&pdev->dev,
15106                         "Could not obtain valid ethernet address, aborting\n");
15107                 goto err_out_iounmap;
15108         }
15109
15110         if (tg3_flag(tp, ENABLE_APE)) {
15111                 tp->aperegs = pci_ioremap_bar(pdev, BAR_2);
15112                 if (!tp->aperegs) {
15113                         dev_err(&pdev->dev,
15114                                 "Cannot map APE registers, aborting\n");
15115                         err = -ENOMEM;
15116                         goto err_out_iounmap;
15117                 }
15118
15119                 tg3_ape_lock_init(tp);
15120
15121                 if (tg3_flag(tp, ENABLE_ASF))
15122                         tg3_read_dash_ver(tp);
15123         }
15124
15125         /*
15126          * Reset chip in case UNDI or EFI driver did not shutdown
15127          * DMA self test will enable WDMAC and we'll see (spurious)
15128          * pending DMA on the PCI bus at that point.
15129          */
15130         if ((tr32(HOSTCC_MODE) & HOSTCC_MODE_ENABLE) ||
15131             (tr32(WDMAC_MODE) & WDMAC_MODE_ENABLE)) {
15132                 tw32(MEMARB_MODE, MEMARB_MODE_ENABLE);
15133                 tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
15134         }
15135
15136         err = tg3_test_dma(tp);
15137         if (err) {
15138                 dev_err(&pdev->dev, "DMA engine test failed, aborting\n");
15139                 goto err_out_apeunmap;
15140         }
15141
15142         intmbx = MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW;
15143         rcvmbx = MAILBOX_RCVRET_CON_IDX_0 + TG3_64BIT_REG_LOW;
15144         sndmbx = MAILBOX_SNDHOST_PROD_IDX_0 + TG3_64BIT_REG_LOW;
15145         for (i = 0; i < tp->irq_max; i++) {
15146                 struct tg3_napi *tnapi = &tp->napi[i];
15147
15148                 tnapi->tp = tp;
15149                 tnapi->tx_pending = TG3_DEF_TX_RING_PENDING;
15150
15151                 tnapi->int_mbox = intmbx;
15152                 if (i < 4)
15153                         intmbx += 0x8;
15154                 else
15155                         intmbx += 0x4;
15156
15157                 tnapi->consmbox = rcvmbx;
15158                 tnapi->prodmbox = sndmbx;
15159
15160                 if (i)
15161                         tnapi->coal_now = HOSTCC_MODE_COAL_VEC1_NOW << (i - 1);
15162                 else
15163                         tnapi->coal_now = HOSTCC_MODE_NOW;
15164
15165                 if (!tg3_flag(tp, SUPPORT_MSIX))
15166                         break;
15167
15168                 /*
15169                  * If we support MSIX, we'll be using RSS.  If we're using
15170                  * RSS, the first vector only handles link interrupts and the
15171                  * remaining vectors handle rx and tx interrupts.  Reuse the
15172                  * mailbox values for the next iteration.  The values we setup
15173                  * above are still useful for the single vectored mode.
15174                  */
15175                 if (!i)
15176                         continue;
15177
15178                 rcvmbx += 0x8;
15179
15180                 if (sndmbx & 0x4)
15181                         sndmbx -= 0x4;
15182                 else
15183                         sndmbx += 0xc;
15184         }
15185
15186         tg3_init_coal(tp);
15187
15188         pci_set_drvdata(pdev, dev);
15189
15190         err = register_netdev(dev);
15191         if (err) {
15192                 dev_err(&pdev->dev, "Cannot register net device, aborting\n");
15193                 goto err_out_apeunmap;
15194         }
15195
15196         netdev_info(dev, "Tigon3 [partno(%s) rev %04x] (%s) MAC address %pM\n",
15197                     tp->board_part_number,
15198                     tp->pci_chip_rev_id,
15199                     tg3_bus_string(tp, str),
15200                     dev->dev_addr);
15201
15202         if (tp->phy_flags & TG3_PHYFLG_IS_CONNECTED) {
15203                 struct phy_device *phydev;
15204                 phydev = tp->mdio_bus->phy_map[TG3_PHY_MII_ADDR];
15205                 netdev_info(dev,
15206                             "attached PHY driver [%s] (mii_bus:phy_addr=%s)\n",
15207                             phydev->drv->name, dev_name(&phydev->dev));
15208         } else {
15209                 char *ethtype;
15210
15211                 if (tp->phy_flags & TG3_PHYFLG_10_100_ONLY)
15212                         ethtype = "10/100Base-TX";
15213                 else if (tp->phy_flags & TG3_PHYFLG_ANY_SERDES)
15214                         ethtype = "1000Base-SX";
15215                 else
15216                         ethtype = "10/100/1000Base-T";
15217
15218                 netdev_info(dev, "attached PHY is %s (%s Ethernet) "
15219                             "(WireSpeed[%d], EEE[%d])\n",
15220                             tg3_phy_string(tp), ethtype,
15221                             (tp->phy_flags & TG3_PHYFLG_NO_ETH_WIRE_SPEED) == 0,
15222                             (tp->phy_flags & TG3_PHYFLG_EEE_CAP) != 0);
15223         }
15224
15225         netdev_info(dev, "RXcsums[%d] LinkChgREG[%d] MIirq[%d] ASF[%d] TSOcap[%d]\n",
15226                     (dev->features & NETIF_F_RXCSUM) != 0,
15227                     tg3_flag(tp, USE_LINKCHG_REG) != 0,
15228                     (tp->phy_flags & TG3_PHYFLG_USE_MI_INTERRUPT) != 0,
15229                     tg3_flag(tp, ENABLE_ASF) != 0,
15230                     tg3_flag(tp, TSO_CAPABLE) != 0);
15231         netdev_info(dev, "dma_rwctrl[%08x] dma_mask[%d-bit]\n",
15232                     tp->dma_rwctrl,
15233                     pdev->dma_mask == DMA_BIT_MASK(32) ? 32 :
15234                     ((u64)pdev->dma_mask) == DMA_BIT_MASK(40) ? 40 : 64);
15235
15236         pci_save_state(pdev);
15237
15238         return 0;
15239
15240 err_out_apeunmap:
15241         if (tp->aperegs) {
15242                 iounmap(tp->aperegs);
15243                 tp->aperegs = NULL;
15244         }
15245
15246 err_out_iounmap:
15247         if (tp->regs) {
15248                 iounmap(tp->regs);
15249                 tp->regs = NULL;
15250         }
15251
15252 err_out_free_dev:
15253         free_netdev(dev);
15254
15255 err_out_free_res:
15256         pci_release_regions(pdev);
15257
15258 err_out_disable_pdev:
15259         pci_disable_device(pdev);
15260         pci_set_drvdata(pdev, NULL);
15261         return err;
15262 }
15263
15264 static void __devexit tg3_remove_one(struct pci_dev *pdev)
15265 {
15266         struct net_device *dev = pci_get_drvdata(pdev);
15267
15268         if (dev) {
15269                 struct tg3 *tp = netdev_priv(dev);
15270
15271                 if (tp->fw)
15272                         release_firmware(tp->fw);
15273
15274                 cancel_work_sync(&tp->reset_task);
15275
15276                 if (!tg3_flag(tp, USE_PHYLIB)) {
15277                         tg3_phy_fini(tp);
15278                         tg3_mdio_fini(tp);
15279                 }
15280
15281                 unregister_netdev(dev);
15282                 if (tp->aperegs) {
15283                         iounmap(tp->aperegs);
15284                         tp->aperegs = NULL;
15285                 }
15286                 if (tp->regs) {
15287                         iounmap(tp->regs);
15288                         tp->regs = NULL;
15289                 }
15290                 free_netdev(dev);
15291                 pci_release_regions(pdev);
15292                 pci_disable_device(pdev);
15293                 pci_set_drvdata(pdev, NULL);
15294         }
15295 }
15296
15297 #ifdef CONFIG_PM_SLEEP
15298 static int tg3_suspend(struct device *device)
15299 {
15300         struct pci_dev *pdev = to_pci_dev(device);
15301         struct net_device *dev = pci_get_drvdata(pdev);
15302         struct tg3 *tp = netdev_priv(dev);
15303         int err;
15304
15305         if (!netif_running(dev))
15306                 return 0;
15307
15308         flush_work_sync(&tp->reset_task);
15309         tg3_phy_stop(tp);
15310         tg3_netif_stop(tp);
15311
15312         del_timer_sync(&tp->timer);
15313
15314         tg3_full_lock(tp, 1);
15315         tg3_disable_ints(tp);
15316         tg3_full_unlock(tp);
15317
15318         netif_device_detach(dev);
15319
15320         tg3_full_lock(tp, 0);
15321         tg3_halt(tp, RESET_KIND_SHUTDOWN, 1);
15322         tg3_flag_clear(tp, INIT_COMPLETE);
15323         tg3_full_unlock(tp);
15324
15325         err = tg3_power_down_prepare(tp);
15326         if (err) {
15327                 int err2;
15328
15329                 tg3_full_lock(tp, 0);
15330
15331                 tg3_flag_set(tp, INIT_COMPLETE);
15332                 err2 = tg3_restart_hw(tp, 1);
15333                 if (err2)
15334                         goto out;
15335
15336                 tp->timer.expires = jiffies + tp->timer_offset;
15337                 add_timer(&tp->timer);
15338
15339                 netif_device_attach(dev);
15340                 tg3_netif_start(tp);
15341
15342 out:
15343                 tg3_full_unlock(tp);
15344
15345                 if (!err2)
15346                         tg3_phy_start(tp);
15347         }
15348
15349         return err;
15350 }
15351
15352 static int tg3_resume(struct device *device)
15353 {
15354         struct pci_dev *pdev = to_pci_dev(device);
15355         struct net_device *dev = pci_get_drvdata(pdev);
15356         struct tg3 *tp = netdev_priv(dev);
15357         int err;
15358
15359         if (!netif_running(dev))
15360                 return 0;
15361
15362         netif_device_attach(dev);
15363
15364         tg3_full_lock(tp, 0);
15365
15366         tg3_flag_set(tp, INIT_COMPLETE);
15367         err = tg3_restart_hw(tp, 1);
15368         if (err)
15369                 goto out;
15370
15371         tp->timer.expires = jiffies + tp->timer_offset;
15372         add_timer(&tp->timer);
15373
15374         tg3_netif_start(tp);
15375
15376 out:
15377         tg3_full_unlock(tp);
15378
15379         if (!err)
15380                 tg3_phy_start(tp);
15381
15382         return err;
15383 }
15384
15385 static SIMPLE_DEV_PM_OPS(tg3_pm_ops, tg3_suspend, tg3_resume);
15386 #define TG3_PM_OPS (&tg3_pm_ops)
15387
15388 #else
15389
15390 #define TG3_PM_OPS NULL
15391
15392 #endif /* CONFIG_PM_SLEEP */
15393
15394 /**
15395  * tg3_io_error_detected - called when PCI error is detected
15396  * @pdev: Pointer to PCI device
15397  * @state: The current pci connection state
15398  *
15399  * This function is called after a PCI bus error affecting
15400  * this device has been detected.
15401  */
15402 static pci_ers_result_t tg3_io_error_detected(struct pci_dev *pdev,
15403                                               pci_channel_state_t state)
15404 {
15405         struct net_device *netdev = pci_get_drvdata(pdev);
15406         struct tg3 *tp = netdev_priv(netdev);
15407         pci_ers_result_t err = PCI_ERS_RESULT_NEED_RESET;
15408
15409         netdev_info(netdev, "PCI I/O error detected\n");
15410
15411         rtnl_lock();
15412
15413         if (!netif_running(netdev))
15414                 goto done;
15415
15416         tg3_phy_stop(tp);
15417
15418         tg3_netif_stop(tp);
15419
15420         del_timer_sync(&tp->timer);
15421         tg3_flag_clear(tp, RESTART_TIMER);
15422
15423         /* Want to make sure that the reset task doesn't run */
15424         cancel_work_sync(&tp->reset_task);
15425         tg3_flag_clear(tp, TX_RECOVERY_PENDING);
15426         tg3_flag_clear(tp, RESTART_TIMER);
15427
15428         netif_device_detach(netdev);
15429
15430         /* Clean up software state, even if MMIO is blocked */
15431         tg3_full_lock(tp, 0);
15432         tg3_halt(tp, RESET_KIND_SHUTDOWN, 0);
15433         tg3_full_unlock(tp);
15434
15435 done:
15436         if (state == pci_channel_io_perm_failure)
15437                 err = PCI_ERS_RESULT_DISCONNECT;
15438         else
15439                 pci_disable_device(pdev);
15440
15441         rtnl_unlock();
15442
15443         return err;
15444 }
15445
15446 /**
15447  * tg3_io_slot_reset - called after the pci bus has been reset.
15448  * @pdev: Pointer to PCI device
15449  *
15450  * Restart the card from scratch, as if from a cold-boot.
15451  * At this point, the card has exprienced a hard reset,
15452  * followed by fixups by BIOS, and has its config space
15453  * set up identically to what it was at cold boot.
15454  */
15455 static pci_ers_result_t tg3_io_slot_reset(struct pci_dev *pdev)
15456 {
15457         struct net_device *netdev = pci_get_drvdata(pdev);
15458         struct tg3 *tp = netdev_priv(netdev);
15459         pci_ers_result_t rc = PCI_ERS_RESULT_DISCONNECT;
15460         int err;
15461
15462         rtnl_lock();
15463
15464         if (pci_enable_device(pdev)) {
15465                 netdev_err(netdev, "Cannot re-enable PCI device after reset.\n");
15466                 goto done;
15467         }
15468
15469         pci_set_master(pdev);
15470         pci_restore_state(pdev);
15471         pci_save_state(pdev);
15472
15473         if (!netif_running(netdev)) {
15474                 rc = PCI_ERS_RESULT_RECOVERED;
15475                 goto done;
15476         }
15477
15478         err = tg3_power_up(tp);
15479         if (err) {
15480                 netdev_err(netdev, "Failed to restore register access.\n");
15481                 goto done;
15482         }
15483
15484         rc = PCI_ERS_RESULT_RECOVERED;
15485
15486 done:
15487         rtnl_unlock();
15488
15489         return rc;
15490 }
15491
15492 /**
15493  * tg3_io_resume - called when traffic can start flowing again.
15494  * @pdev: Pointer to PCI device
15495  *
15496  * This callback is called when the error recovery driver tells
15497  * us that its OK to resume normal operation.
15498  */
15499 static void tg3_io_resume(struct pci_dev *pdev)
15500 {
15501         struct net_device *netdev = pci_get_drvdata(pdev);
15502         struct tg3 *tp = netdev_priv(netdev);
15503         int err;
15504
15505         rtnl_lock();
15506
15507         if (!netif_running(netdev))
15508                 goto done;
15509
15510         tg3_full_lock(tp, 0);
15511         tg3_flag_set(tp, INIT_COMPLETE);
15512         err = tg3_restart_hw(tp, 1);
15513         tg3_full_unlock(tp);
15514         if (err) {
15515                 netdev_err(netdev, "Cannot restart hardware after reset.\n");
15516                 goto done;
15517         }
15518
15519         netif_device_attach(netdev);
15520
15521         tp->timer.expires = jiffies + tp->timer_offset;
15522         add_timer(&tp->timer);
15523
15524         tg3_netif_start(tp);
15525
15526         tg3_phy_start(tp);
15527
15528 done:
15529         rtnl_unlock();
15530 }
15531
15532 static struct pci_error_handlers tg3_err_handler = {
15533         .error_detected = tg3_io_error_detected,
15534         .slot_reset     = tg3_io_slot_reset,
15535         .resume         = tg3_io_resume
15536 };
15537
15538 static struct pci_driver tg3_driver = {
15539         .name           = DRV_MODULE_NAME,
15540         .id_table       = tg3_pci_tbl,
15541         .probe          = tg3_init_one,
15542         .remove         = __devexit_p(tg3_remove_one),
15543         .err_handler    = &tg3_err_handler,
15544         .driver.pm      = TG3_PM_OPS,
15545 };
15546
15547 static int __init tg3_init(void)
15548 {
15549         return pci_register_driver(&tg3_driver);
15550 }
15551
15552 static void __exit tg3_cleanup(void)
15553 {
15554         pci_unregister_driver(&tg3_driver);
15555 }
15556
15557 module_init(tg3_init);
15558 module_exit(tg3_cleanup);