drivers/edac/e752x_edac.c

   1 /*
   2  * Intel e752x Memory Controller kernel module
   3  * (C) 2004 Linux Networx (http://lnxi.com)
   4  * This file may be distributed under the terms of the
   5  * GNU General Public License.
   6  *
   7  * Implement support for the e7520, E7525, e7320 and i3100 memory controllers.
   8  *
   9  * Datasheets:
  10  *      http://www.intel.in/content/www/in/en/chipsets/e7525-memory-controller-hub-datasheet.html
  11  *      ftp://download.intel.com/design/intarch/datashts/31345803.pdf
  12  *
  13  * Written by Tom Zimmerman
  14  *
  15  * Contributors:
  16  *      Thayne Harbaugh at realmsys.com (?)
  17  *      Wang Zhenyu at intel.com
  18  *      Dave Jiang at mvista.com
  19  *
  20  */
  21
  22 #include <linux/module.h>
  23 #include <linux/init.h>
  24 #include <linux/pci.h>
  25 #include <linux/pci_ids.h>
  26 #include <linux/edac.h>
  27 #include "edac_core.h"
  28
  29 #define E752X_REVISION  " Ver: 2.0.2"
  30 #define EDAC_MOD_STR    "e752x_edac"
  31
  32 static int report_non_memory_errors;
  33 static int force_function_unhide;
  34 static int sysbus_parity = -1;
  35
  36 static struct edac_pci_ctl_info *e752x_pci;
  37
  38 #define e752x_printk(level, fmt, arg...) \
  39         edac_printk(level, "e752x", fmt, ##arg)
  40
  41 #define e752x_mc_printk(mci, level, fmt, arg...) \
  42         edac_mc_chipset_printk(mci, level, "e752x", fmt, ##arg)
  43
  44 #ifndef PCI_DEVICE_ID_INTEL_7520_0
  45 #define PCI_DEVICE_ID_INTEL_7520_0      0x3590
  46 #endif                          /* PCI_DEVICE_ID_INTEL_7520_0      */
  47
  48 #ifndef PCI_DEVICE_ID_INTEL_7520_1_ERR
  49 #define PCI_DEVICE_ID_INTEL_7520_1_ERR  0x3591
  50 #endif                          /* PCI_DEVICE_ID_INTEL_7520_1_ERR  */
  51
  52 #ifndef PCI_DEVICE_ID_INTEL_7525_0
  53 #define PCI_DEVICE_ID_INTEL_7525_0      0x359E
  54 #endif                          /* PCI_DEVICE_ID_INTEL_7525_0      */
  55
  56 #ifndef PCI_DEVICE_ID_INTEL_7525_1_ERR
  57 #define PCI_DEVICE_ID_INTEL_7525_1_ERR  0x3593
  58 #endif                          /* PCI_DEVICE_ID_INTEL_7525_1_ERR  */
  59
  60 #ifndef PCI_DEVICE_ID_INTEL_7320_0
  61 #define PCI_DEVICE_ID_INTEL_7320_0      0x3592
  62 #endif                          /* PCI_DEVICE_ID_INTEL_7320_0 */
  63
  64 #ifndef PCI_DEVICE_ID_INTEL_7320_1_ERR
  65 #define PCI_DEVICE_ID_INTEL_7320_1_ERR  0x3593
  66 #endif                          /* PCI_DEVICE_ID_INTEL_7320_1_ERR */
  67
  68 #ifndef PCI_DEVICE_ID_INTEL_3100_0
  69 #define PCI_DEVICE_ID_INTEL_3100_0      0x35B0
  70 #endif                          /* PCI_DEVICE_ID_INTEL_3100_0 */
  71
  72 #ifndef PCI_DEVICE_ID_INTEL_3100_1_ERR
  73 #define PCI_DEVICE_ID_INTEL_3100_1_ERR  0x35B1
  74 #endif                          /* PCI_DEVICE_ID_INTEL_3100_1_ERR */
  75
  76 #define E752X_NR_CSROWS         8       /* number of csrows */
  77
  78 /* E752X register addresses - device 0 function 0 */
  79 #define E752X_MCHSCRB           0x52    /* Memory Scrub register (16b) */
  80                                         /*
  81                                          * 6:5     Scrub Completion Count
  82                                          * 3:2     Scrub Rate (i3100 only)
  83                                          *      01=fast 10=normal
  84                                          * 1:0     Scrub Mode enable
  85                                          *      00=off 10=on
  86                                          */
  87 #define E752X_DRB               0x60    /* DRAM row boundary register (8b) */
  88 #define E752X_DRA               0x70    /* DRAM row attribute register (8b) */
  89                                         /*
  90                                          * 31:30   Device width row 7
  91                                          *      01=x8 10=x4 11=x8 DDR2
  92                                          * 27:26   Device width row 6
  93                                          * 23:22   Device width row 5
  94                                          * 19:20   Device width row 4
  95                                          * 15:14   Device width row 3
  96                                          * 11:10   Device width row 2
  97                                          *  7:6    Device width row 1
  98                                          *  3:2    Device width row 0
  99                                          */
 100 #define E752X_DRC               0x7C    /* DRAM controller mode reg (32b) */
 101                                         /* FIXME:IS THIS RIGHT? */
 102                                         /*
 103                                          * 22    Number channels 0=1,1=2
 104                                          * 19:18 DRB Granularity 32/64MB
 105                                          */
 106 #define E752X_DRM               0x80    /* Dimm mapping register */
 107 #define E752X_DDRCSR            0x9A    /* DDR control and status reg (16b) */
 108                                         /*
 109                                          * 14:12 1 single A, 2 single B, 3 dual
 110                                          */
 111 #define E752X_TOLM              0xC4    /* DRAM top of low memory reg (16b) */
 112 #define E752X_REMAPBASE         0xC6    /* DRAM remap base address reg (16b) */
 113 #define E752X_REMAPLIMIT        0xC8    /* DRAM remap limit address reg (16b) */
 114 #define E752X_REMAPOFFSET       0xCA    /* DRAM remap limit offset reg (16b) */
 115
 116 /* E752X register addresses - device 0 function 1 */
 117 #define E752X_FERR_GLOBAL       0x40    /* Global first error register (32b) */
 118 #define E752X_NERR_GLOBAL       0x44    /* Global next error register (32b) */
 119 #define E752X_HI_FERR           0x50    /* Hub interface first error reg (8b) */
 120 #define E752X_HI_NERR           0x52    /* Hub interface next error reg (8b) */
 121 #define E752X_HI_ERRMASK        0x54    /* Hub interface error mask reg (8b) */
 122 #define E752X_HI_SMICMD         0x5A    /* Hub interface SMI command reg (8b) */
 123 #define E752X_SYSBUS_FERR       0x60    /* System buss first error reg (16b) */
 124 #define E752X_SYSBUS_NERR       0x62    /* System buss next error reg (16b) */
 125 #define E752X_SYSBUS_ERRMASK    0x64    /* System buss error mask reg (16b) */
 126 #define E752X_SYSBUS_SMICMD     0x6A    /* System buss SMI command reg (16b) */
 127 #define E752X_BUF_FERR          0x70    /* Memory buffer first error reg (8b) */
 128 #define E752X_BUF_NERR          0x72    /* Memory buffer next error reg (8b) */
 129 #define E752X_BUF_ERRMASK       0x74    /* Memory buffer error mask reg (8b) */
 130 #define E752X_BUF_SMICMD        0x7A    /* Memory buffer SMI cmd reg (8b) */
 131 #define E752X_DRAM_FERR         0x80    /* DRAM first error register (16b) */
 132 #define E752X_DRAM_NERR         0x82    /* DRAM next error register (16b) */
 133 #define E752X_DRAM_ERRMASK      0x84    /* DRAM error mask register (8b) */
 134 #define E752X_DRAM_SMICMD       0x8A    /* DRAM SMI command register (8b) */
 135 #define E752X_DRAM_RETR_ADD     0xAC    /* DRAM Retry address register (32b) */
 136 #define E752X_DRAM_SEC1_ADD     0xA0    /* DRAM first correctable memory */
 137                                         /*     error address register (32b) */
 138                                         /*
 139                                          * 31    Reserved
 140                                          * 30:2  CE address (64 byte block 34:6
 141                                          * 1     Reserved
 142                                          * 0     HiLoCS
 143                                          */
 144 #define E752X_DRAM_SEC2_ADD     0xC8    /* DRAM first correctable memory */
 145                                         /*     error address register (32b) */
 146                                         /*
 147                                          * 31    Reserved
 148                                          * 30:2  CE address (64 byte block 34:6)
 149                                          * 1     Reserved
 150                                          * 0     HiLoCS
 151                                          */
 152 #define E752X_DRAM_DED_ADD      0xA4    /* DRAM first uncorrectable memory */
 153                                         /*     error address register (32b) */
 154                                         /*
 155                                          * 31    Reserved
 156                                          * 30:2  CE address (64 byte block 34:6)
 157                                          * 1     Reserved
 158                                          * 0     HiLoCS
 159                                          */
 160 #define E752X_DRAM_SCRB_ADD     0xA8    /* DRAM 1st uncorrectable scrub mem */
 161                                         /*     error address register (32b) */
 162                                         /*
 163                                          * 31    Reserved
 164                                          * 30:2  CE address (64 byte block 34:6
 165                                          * 1     Reserved
 166                                          * 0     HiLoCS
 167                                          */
 168 #define E752X_DRAM_SEC1_SYNDROME 0xC4   /* DRAM first correctable memory */
 169                                         /*     error syndrome register (16b) */
 170 #define E752X_DRAM_SEC2_SYNDROME 0xC6   /* DRAM second correctable memory */
 171                                         /*     error syndrome register (16b) */
 172 #define E752X_DEVPRES1          0xF4    /* Device Present 1 register (8b) */
 173
 174 /* 3100 IMCH specific register addresses - device 0 function 1 */
 175 #define I3100_NSI_FERR          0x48    /* NSI first error reg (32b) */
 176 #define I3100_NSI_NERR          0x4C    /* NSI next error reg (32b) */
 177 #define I3100_NSI_SMICMD        0x54    /* NSI SMI command register (32b) */
 178 #define I3100_NSI_EMASK         0x90    /* NSI error mask register (32b) */
 179
 180 /* ICH5R register addresses - device 30 function 0 */
 181 #define ICH5R_PCI_STAT          0x06    /* PCI status register (16b) */
 182 #define ICH5R_PCI_2ND_STAT      0x1E    /* PCI status secondary reg (16b) */
 183 #define ICH5R_PCI_BRIDGE_CTL    0x3E    /* PCI bridge control register (16b) */
 184
 185 enum e752x_chips {
 186         E7520 = 0,
 187         E7525 = 1,
 188         E7320 = 2,
 189         I3100 = 3
 190 };
 191
 192 /*
 193  * Those chips Support single-rank and dual-rank memories only.
 194  *
 195  * On e752x chips, the odd rows are present only on dual-rank memories.
 196  * Dividing the rank by two will provide the dimm#
 197  *
 198  * i3100 MC has a different mapping: it supports only 4 ranks.
 199  *
 200  * The mapping is (from 1 to n):
 201  *      slot       single-ranked        double-ranked
 202  *      dimm #1 -> rank #4              NA
 203  *      dimm #2 -> rank #3              NA
 204  *      dimm #3 -> rank #2              Ranks 2 and 3
 205  *      dimm #4 -> rank $1              Ranks 1 and 4
 206  *
 207  * FIXME: The current mapping for i3100 considers that it supports up to 8
 208  *        ranks/chanel, but datasheet says that the MC supports only 4 ranks.
 209  */
 210
 211 struct e752x_pvt {
 212         struct pci_dev *bridge_ck;
 213         struct pci_dev *dev_d0f0;
 214         struct pci_dev *dev_d0f1;
 215         u32 tolm;
 216         u32 remapbase;
 217         u32 remaplimit;
 218         int mc_symmetric;
 219         u8 map[8];
 220         int map_type;
 221         const struct e752x_dev_info *dev_info;
 222 };
 223
 224 struct e752x_dev_info {
 225         u16 err_dev;
 226         u16 ctl_dev;
 227         const char *ctl_name;
 228 };
 229
 230 struct e752x_error_info {
 231         u32 ferr_global;
 232         u32 nerr_global;
 233         u32 nsi_ferr;   /* 3100 only */
 234         u32 nsi_nerr;   /* 3100 only */
 235         u8 hi_ferr;     /* all but 3100 */
 236         u8 hi_nerr;     /* all but 3100 */
 237         u16 sysbus_ferr;
 238         u16 sysbus_nerr;
 239         u8 buf_ferr;
 240         u8 buf_nerr;
 241         u16 dram_ferr;
 242         u16 dram_nerr;
 243         u32 dram_sec1_add;
 244         u32 dram_sec2_add;
 245         u16 dram_sec1_syndrome;
 246         u16 dram_sec2_syndrome;
 247         u32 dram_ded_add;
 248         u32 dram_scrb_add;
 249         u32 dram_retr_add;
 250 };
 251
 252 static const struct e752x_dev_info e752x_devs[] = {
 253         [E7520] = {
 254                 .err_dev = PCI_DEVICE_ID_INTEL_7520_1_ERR,
 255                 .ctl_dev = PCI_DEVICE_ID_INTEL_7520_0,
 256                 .ctl_name = "E7520"},
 257         [E7525] = {
 258                 .err_dev = PCI_DEVICE_ID_INTEL_7525_1_ERR,
 259                 .ctl_dev = PCI_DEVICE_ID_INTEL_7525_0,
 260                 .ctl_name = "E7525"},
 261         [E7320] = {
 262                 .err_dev = PCI_DEVICE_ID_INTEL_7320_1_ERR,
 263                 .ctl_dev = PCI_DEVICE_ID_INTEL_7320_0,
 264                 .ctl_name = "E7320"},
 265         [I3100] = {
 266                 .err_dev = PCI_DEVICE_ID_INTEL_3100_1_ERR,
 267                 .ctl_dev = PCI_DEVICE_ID_INTEL_3100_0,
 268                 .ctl_name = "3100"},
 269 };
 270
 271 /* Valid scrub rates for the e752x/3100 hardware memory scrubber. We
 272  * map the scrubbing bandwidth to a hardware register value. The 'set'
 273  * operation finds the 'matching or higher value'.  Note that scrubbing
 274  * on the e752x can only be enabled/disabled.  The 3100 supports
 275  * a normal and fast mode.
 276  */
 277
 278 #define SDRATE_EOT 0xFFFFFFFF
 279
 280 struct scrubrate {
 281         u32 bandwidth;  /* bandwidth consumed by scrubbing in bytes/sec */
 282         u16 scrubval;   /* register value for scrub rate */
 283 };
 284
 285 /* Rate below assumes same performance as i3100 using PC3200 DDR2 in
 286  * normal mode.  e752x bridges don't support choosing normal or fast mode,
 287  * so the scrubbing bandwidth value isn't all that important - scrubbing is
 288  * either on or off.
 289  */
 290 static const struct scrubrate scrubrates_e752x[] = {
 291         {0,             0x00},  /* Scrubbing Off */
 292         {500000,        0x02},  /* Scrubbing On */
 293         {SDRATE_EOT,    0x00}   /* End of Table */
 294 };
 295
 296 /* Fast mode: 2 GByte PC3200 DDR2 scrubbed in 33s = 63161283 bytes/s
 297  * Normal mode: 125 (32000 / 256) times slower than fast mode.
 298  */
 299 static const struct scrubrate scrubrates_i3100[] = {
 300         {0,             0x00},  /* Scrubbing Off */
 301         {500000,        0x0a},  /* Normal mode - 32k clocks */
 302         {62500000,      0x06},  /* Fast mode - 256 clocks */
 303         {SDRATE_EOT,    0x00}   /* End of Table */
 304 };
 305
 306 static unsigned long ctl_page_to_phys(struct mem_ctl_info *mci,
 307                                 unsigned long page)
 308 {
 309         u32 remap;
 310         struct e752x_pvt *pvt = (struct e752x_pvt *)mci->pvt_info;
 311
 312         edac_dbg(3, "\n");
 313
 314         if (page < pvt->tolm)
 315                 return page;
 316
 317         if ((page >= 0x100000) && (page < pvt->remapbase))
 318                 return page;
 319
 320         remap = (page - pvt->tolm) + pvt->remapbase;
 321
 322         if (remap < pvt->remaplimit)
 323                 return remap;
 324
 325         e752x_printk(KERN_ERR, "Invalid page %lx - out of range\n", page);
 326         return pvt->tolm - 1;
 327 }
 328
 329 static void do_process_ce(struct mem_ctl_info *mci, u16 error_one,
 330                         u32 sec1_add, u16 sec1_syndrome)
 331 {
 332         u32 page;
 333         int row;
 334         int channel;
 335         int i;
 336         struct e752x_pvt *pvt = (struct e752x_pvt *)mci->pvt_info;
 337
 338         edac_dbg(3, "\n");
 339
 340         /* convert the addr to 4k page */
 341         page = sec1_add >> (PAGE_SHIFT - 4);
 342
 343         /* FIXME - check for -1 */
 344         if (pvt->mc_symmetric) {
 345                 /* chip select are bits 14 & 13 */
 346                 row = ((page >> 1) & 3);
 347                 e752x_printk(KERN_WARNING,
 348                         "Test row %d Table %d %d %d %d %d %d %d %d\n", row,
 349                         pvt->map[0], pvt->map[1], pvt->map[2], pvt->map[3],
 350                         pvt->map[4], pvt->map[5], pvt->map[6],
 351                         pvt->map[7]);
 352
 353                 /* test for channel remapping */
 354                 for (i = 0; i < 8; i++) {
 355                         if (pvt->map[i] == row)
 356                                 break;
 357                 }
 358
 359                 e752x_printk(KERN_WARNING, "Test computed row %d\n", i);
 360
 361                 if (i < 8)
 362                         row = i;
 363                 else
 364                         e752x_mc_printk(mci, KERN_WARNING,
 365                                         "row %d not found in remap table\n",
 366                                         row);
 367         } else
 368                 row = edac_mc_find_csrow_by_page(mci, page);
 369
 370         /* 0 = channel A, 1 = channel B */
 371         channel = !(error_one & 1);
 372
 373         /* e752x mc reads 34:6 of the DRAM linear address */
 374         edac_mc_handle_error(HW_EVENT_ERR_CORRECTED, mci, 1,
 375                              page, offset_in_page(sec1_add << 4), sec1_syndrome,
 376                              row, channel, -1,
 377                              "e752x CE", "");
 378 }
 379
 380 static inline void process_ce(struct mem_ctl_info *mci, u16 error_one,
 381                         u32 sec1_add, u16 sec1_syndrome, int *error_found,
 382                         int handle_error)
 383 {
 384         *error_found = 1;
 385
 386         if (handle_error)
 387                 do_process_ce(mci, error_one, sec1_add, sec1_syndrome);
 388 }
 389
 390 static void do_process_ue(struct mem_ctl_info *mci, u16 error_one,
 391                         u32 ded_add, u32 scrb_add)
 392 {
 393         u32 error_2b, block_page;
 394         int row;
 395         struct e752x_pvt *pvt = (struct e752x_pvt *)mci->pvt_info;
 396
 397         edac_dbg(3, "\n");
 398
 399         if (error_one & 0x0202) {
 400                 error_2b = ded_add;
 401
 402                 /* convert to 4k address */
 403                 block_page = error_2b >> (PAGE_SHIFT - 4);
 404
 405                 row = pvt->mc_symmetric ?
 406                 /* chip select are bits 14 & 13 */
 407                         ((block_page >> 1) & 3) :
 408                         edac_mc_find_csrow_by_page(mci, block_page);
 409
 410                 /* e752x mc reads 34:6 of the DRAM linear address */
 411                 edac_mc_handle_error(HW_EVENT_ERR_UNCORRECTED, mci, 1,
 412                                         block_page,
 413                                         offset_in_page(error_2b << 4), 0,
 414                                          row, -1, -1,
 415                                         "e752x UE from Read", "");
 416
 417         }
 418         if (error_one & 0x0404) {
 419                 error_2b = scrb_add;
 420
 421                 /* convert to 4k address */
 422                 block_page = error_2b >> (PAGE_SHIFT - 4);
 423
 424                 row = pvt->mc_symmetric ?
 425                 /* chip select are bits 14 & 13 */
 426                         ((block_page >> 1) & 3) :
 427                         edac_mc_find_csrow_by_page(mci, block_page);
 428
 429                 /* e752x mc reads 34:6 of the DRAM linear address */
 430                 edac_mc_handle_error(HW_EVENT_ERR_UNCORRECTED, mci, 1,
 431                                         block_page,
 432                                         offset_in_page(error_2b << 4), 0,
 433                                         row, -1, -1,
 434                                         "e752x UE from Scruber", "");
 435         }
 436 }
 437
 438 static inline void process_ue(struct mem_ctl_info *mci, u16 error_one,
 439                         u32 ded_add, u32 scrb_add, int *error_found,
 440                         int handle_error)
 441 {
 442         *error_found = 1;
 443
 444         if (handle_error)
 445                 do_process_ue(mci, error_one, ded_add, scrb_add);
 446 }
 447
 448 static inline void process_ue_no_info_wr(struct mem_ctl_info *mci,
 449                                          int *error_found, int handle_error)
 450 {
 451         *error_found = 1;
 452
 453         if (!handle_error)
 454                 return;
 455
 456         edac_dbg(3, "\n");
 457         edac_mc_handle_error(HW_EVENT_ERR_UNCORRECTED, mci, 1, 0, 0, 0,
 458                              -1, -1, -1,
 459                              "e752x UE log memory write", "");
 460 }
 461
 462 static void do_process_ded_retry(struct mem_ctl_info *mci, u16 error,
 463                                  u32 retry_add)
 464 {
 465         u32 error_1b, page;
 466         int row;
 467         struct e752x_pvt *pvt = (struct e752x_pvt *)mci->pvt_info;
 468
 469         error_1b = retry_add;
 470         page = error_1b >> (PAGE_SHIFT - 4);  /* convert the addr to 4k page */
 471
 472         /* chip select are bits 14 & 13 */
 473         row = pvt->mc_symmetric ? ((page >> 1) & 3) :
 474                 edac_mc_find_csrow_by_page(mci, page);
 475
 476         e752x_mc_printk(mci, KERN_WARNING,
 477                         "CE page 0x%lx, row %d : Memory read retry\n",
 478                         (long unsigned int)page, row);
 479 }
 480
 481 static inline void process_ded_retry(struct mem_ctl_info *mci, u16 error,
 482                                 u32 retry_add, int *error_found,
 483                                 int handle_error)
 484 {
 485         *error_found = 1;
 486
 487         if (handle_error)
 488                 do_process_ded_retry(mci, error, retry_add);
 489 }
 490
 491 static inline void process_threshold_ce(struct mem_ctl_info *mci, u16 error,
 492                                         int *error_found, int handle_error)
 493 {
 494         *error_found = 1;
 495
 496         if (handle_error)
 497                 e752x_mc_printk(mci, KERN_WARNING, "Memory threshold CE\n");
 498 }
 499
 500 static char *global_message[11] = {
 501         "PCI Express C1",
 502         "PCI Express C",
 503         "PCI Express B1",
 504         "PCI Express B",
 505         "PCI Express A1",
 506         "PCI Express A",
 507         "DMA Controller",
 508         "HUB or NS Interface",
 509         "System Bus",
 510         "DRAM Controller",  /* 9th entry */
 511         "Internal Buffer"
 512 };
 513
 514 #define DRAM_ENTRY      9
 515
 516 static char *fatal_message[2] = { "Non-Fatal ", "Fatal " };
 517
 518 static void do_global_error(int fatal, u32 errors)
 519 {
 520         int i;
 521
 522         for (i = 0; i < 11; i++) {
 523                 if (errors & (1 << i)) {
 524                         /* If the error is from DRAM Controller OR
 525                          * we are to report ALL errors, then
 526                          * report the error
 527                          */
 528                         if ((i == DRAM_ENTRY) || report_non_memory_errors)
 529                                 e752x_printk(KERN_WARNING, "%sError %s\n",
 530                                         fatal_message[fatal],
 531                                         global_message[i]);
 532                 }
 533         }
 534 }
 535
 536 static inline void global_error(int fatal, u32 errors, int *error_found,
 537                                 int handle_error)
 538 {
 539         *error_found = 1;
 540
 541         if (handle_error)
 542                 do_global_error(fatal, errors);
 543 }
 544
 545 static char *hub_message[7] = {
 546         "HI Address or Command Parity", "HI Illegal Access",
 547         "HI Internal Parity", "Out of Range Access",
 548         "HI Data Parity", "Enhanced Config Access",
 549         "Hub Interface Target Abort"
 550 };
 551
 552 static void do_hub_error(int fatal, u8 errors)
 553 {
 554         int i;
 555
 556         for (i = 0; i < 7; i++) {
 557                 if (errors & (1 << i))
 558                         e752x_printk(KERN_WARNING, "%sError %s\n",
 559                                 fatal_message[fatal], hub_message[i]);
 560         }
 561 }
 562
 563 static inline void hub_error(int fatal, u8 errors, int *error_found,
 564                         int handle_error)
 565 {
 566         *error_found = 1;
 567
 568         if (handle_error)
 569                 do_hub_error(fatal, errors);
 570 }
 571
 572 #define NSI_FATAL_MASK          0x0c080081
 573 #define NSI_NON_FATAL_MASK      0x23a0ba64
 574 #define NSI_ERR_MASK            (NSI_FATAL_MASK | NSI_NON_FATAL_MASK)
 575
 576 static char *nsi_message[30] = {
 577         "NSI Link Down",        /* NSI_FERR/NSI_NERR bit 0, fatal error */
 578         "",                                             /* reserved */
 579         "NSI Parity Error",                             /* bit 2, non-fatal */
 580         "",                                             /* reserved */
 581         "",                                             /* reserved */
 582         "Correctable Error Message",                    /* bit 5, non-fatal */
 583         "Non-Fatal Error Message",                      /* bit 6, non-fatal */
 584         "Fatal Error Message",                          /* bit 7, fatal */
 585         "",                                             /* reserved */
 586         "Receiver Error",                               /* bit 9, non-fatal */
 587         "",                                             /* reserved */
 588         "Bad TLP",                                      /* bit 11, non-fatal */
 589         "Bad DLLP",                                     /* bit 12, non-fatal */
 590         "REPLAY_NUM Rollover",                          /* bit 13, non-fatal */
 591         "",                                             /* reserved */
 592         "Replay Timer Timeout",                         /* bit 15, non-fatal */
 593         "",                                             /* reserved */
 594         "",                                             /* reserved */
 595         "",                                             /* reserved */
 596         "Data Link Protocol Error",                     /* bit 19, fatal */
 597         "",                                             /* reserved */
 598         "Poisoned TLP",                                 /* bit 21, non-fatal */
 599         "",                                             /* reserved */
 600         "Completion Timeout",                           /* bit 23, non-fatal */
 601         "Completer Abort",                              /* bit 24, non-fatal */
 602         "Unexpected Completion",                        /* bit 25, non-fatal */
 603         "Receiver Overflow",                            /* bit 26, fatal */
 604         "Malformed TLP",                                /* bit 27, fatal */
 605         "",                                             /* reserved */
 606         "Unsupported Request"                           /* bit 29, non-fatal */
 607 };
 608
 609 static void do_nsi_error(int fatal, u32 errors)
 610 {
 611         int i;
 612
 613         for (i = 0; i < 30; i++) {
 614                 if (errors & (1 << i))
 615                         printk(KERN_WARNING "%sError %s\n",
 616                                fatal_message[fatal], nsi_message[i]);
 617         }
 618 }
 619
 620 static inline void nsi_error(int fatal, u32 errors, int *error_found,
 621                 int handle_error)
 622 {
 623         *error_found = 1;
 624
 625         if (handle_error)
 626                 do_nsi_error(fatal, errors);
 627 }
 628
 629 static char *membuf_message[4] = {
 630         "Internal PMWB to DRAM parity",
 631         "Internal PMWB to System Bus Parity",
 632         "Internal System Bus or IO to PMWB Parity",
 633         "Internal DRAM to PMWB Parity"
 634 };
 635
 636 static void do_membuf_error(u8 errors)
 637 {
 638         int i;
 639
 640         for (i = 0; i < 4; i++) {
 641                 if (errors & (1 << i))
 642                         e752x_printk(KERN_WARNING, "Non-Fatal Error %s\n",
 643                                 membuf_message[i]);
 644         }
 645 }
 646
 647 static inline void membuf_error(u8 errors, int *error_found, int handle_error)
 648 {
 649         *error_found = 1;
 650
 651         if (handle_error)
 652                 do_membuf_error(errors);
 653 }
 654
 655 static char *sysbus_message[10] = {
 656         "Addr or Request Parity",
 657         "Data Strobe Glitch",
 658         "Addr Strobe Glitch",
 659         "Data Parity",
 660         "Addr Above TOM",
 661         "Non DRAM Lock Error",
 662         "MCERR", "BINIT",
 663         "Memory Parity",
 664         "IO Subsystem Parity"
 665 };
 666
 667 static void do_sysbus_error(int fatal, u32 errors)
 668 {
 669         int i;
 670
 671         for (i = 0; i < 10; i++) {
 672                 if (errors & (1 << i))
 673                         e752x_printk(KERN_WARNING, "%sError System Bus %s\n",
 674                                 fatal_message[fatal], sysbus_message[i]);
 675         }
 676 }
 677
 678 static inline void sysbus_error(int fatal, u32 errors, int *error_found,
 679                                 int handle_error)
 680 {
 681         *error_found = 1;
 682
 683         if (handle_error)
 684                 do_sysbus_error(fatal, errors);
 685 }
 686
 687 static void e752x_check_hub_interface(struct e752x_error_info *info,
 688                                 int *error_found, int handle_error)
 689 {
 690         u8 stat8;
 691
 692         //pci_read_config_byte(dev,E752X_HI_FERR,&stat8);
 693
 694         stat8 = info->hi_ferr;
 695
 696         if (stat8 & 0x7f) {     /* Error, so process */
 697                 stat8 &= 0x7f;
 698
 699                 if (stat8 & 0x2b)
 700                         hub_error(1, stat8 & 0x2b, error_found, handle_error);
 701
 702                 if (stat8 & 0x54)
 703                         hub_error(0, stat8 & 0x54, error_found, handle_error);
 704         }
 705         //pci_read_config_byte(dev,E752X_HI_NERR,&stat8);
 706
 707         stat8 = info->hi_nerr;
 708
 709         if (stat8 & 0x7f) {     /* Error, so process */
 710                 stat8 &= 0x7f;
 711
 712                 if (stat8 & 0x2b)
 713                         hub_error(1, stat8 & 0x2b, error_found, handle_error);
 714
 715                 if (stat8 & 0x54)
 716                         hub_error(0, stat8 & 0x54, error_found, handle_error);
 717         }
 718 }
 719
 720 static void e752x_check_ns_interface(struct e752x_error_info *info,
 721                                 int *error_found, int handle_error)
 722 {
 723         u32 stat32;
 724
 725         stat32 = info->nsi_ferr;
 726         if (stat32 & NSI_ERR_MASK) { /* Error, so process */
 727                 if (stat32 & NSI_FATAL_MASK)    /* check for fatal errors */
 728                         nsi_error(1, stat32 & NSI_FATAL_MASK, error_found,
 729                                   handle_error);
 730                 if (stat32 & NSI_NON_FATAL_MASK) /* check for non-fatal ones */
 731                         nsi_error(0, stat32 & NSI_NON_FATAL_MASK, error_found,
 732                                   handle_error);
 733         }
 734         stat32 = info->nsi_nerr;
 735         if (stat32 & NSI_ERR_MASK) {
 736                 if (stat32 & NSI_FATAL_MASK)
 737                         nsi_error(1, stat32 & NSI_FATAL_MASK, error_found,
 738                                   handle_error);
 739                 if (stat32 & NSI_NON_FATAL_MASK)
 740                         nsi_error(0, stat32 & NSI_NON_FATAL_MASK, error_found,
 741                                   handle_error);
 742         }
 743 }
 744
 745 static void e752x_check_sysbus(struct e752x_error_info *info,
 746                         int *error_found, int handle_error)
 747 {
 748         u32 stat32, error32;
 749
 750         //pci_read_config_dword(dev,E752X_SYSBUS_FERR,&stat32);
 751         stat32 = info->sysbus_ferr + (info->sysbus_nerr << 16);
 752
 753         if (stat32 == 0)
 754                 return;         /* no errors */
 755
 756         error32 = (stat32 >> 16) & 0x3ff;
 757         stat32 = stat32 & 0x3ff;
 758
 759         if (stat32 & 0x087)
 760                 sysbus_error(1, stat32 & 0x087, error_found, handle_error);
 761
 762         if (stat32 & 0x378)
 763                 sysbus_error(0, stat32 & 0x378, error_found, handle_error);
 764
 765         if (error32 & 0x087)
 766                 sysbus_error(1, error32 & 0x087, error_found, handle_error);
 767
 768         if (error32 & 0x378)
 769                 sysbus_error(0, error32 & 0x378, error_found, handle_error);
 770 }
 771
 772 static void e752x_check_membuf(struct e752x_error_info *info,
 773                         int *error_found, int handle_error)
 774 {
 775         u8 stat8;
 776
 777         stat8 = info->buf_ferr;
 778
 779         if (stat8 & 0x0f) {     /* Error, so process */
 780                 stat8 &= 0x0f;
 781                 membuf_error(stat8, error_found, handle_error);
 782         }
 783
 784         stat8 = info->buf_nerr;
 785
 786         if (stat8 & 0x0f) {     /* Error, so process */
 787                 stat8 &= 0x0f;
 788                 membuf_error(stat8, error_found, handle_error);
 789         }
 790 }
 791
 792 static void e752x_check_dram(struct mem_ctl_info *mci,
 793                         struct e752x_error_info *info, int *error_found,
 794                         int handle_error)
 795 {
 796         u16 error_one, error_next;
 797
 798         error_one = info->dram_ferr;
 799         error_next = info->dram_nerr;
 800
 801         /* decode and report errors */
 802         if (error_one & 0x0101) /* check first error correctable */
 803                 process_ce(mci, error_one, info->dram_sec1_add,
 804                         info->dram_sec1_syndrome, error_found, handle_error);
 805
 806         if (error_next & 0x0101)        /* check next error correctable */
 807                 process_ce(mci, error_next, info->dram_sec2_add,
 808                         info->dram_sec2_syndrome, error_found, handle_error);
 809
 810         if (error_one & 0x4040)
 811                 process_ue_no_info_wr(mci, error_found, handle_error);
 812
 813         if (error_next & 0x4040)
 814                 process_ue_no_info_wr(mci, error_found, handle_error);
 815
 816         if (error_one & 0x2020)
 817                 process_ded_retry(mci, error_one, info->dram_retr_add,
 818                                 error_found, handle_error);
 819
 820         if (error_next & 0x2020)
 821                 process_ded_retry(mci, error_next, info->dram_retr_add,
 822                                 error_found, handle_error);
 823
 824         if (error_one & 0x0808)
 825                 process_threshold_ce(mci, error_one, error_found, handle_error);
 826
 827         if (error_next & 0x0808)
 828                 process_threshold_ce(mci, error_next, error_found,
 829                                 handle_error);
 830
 831         if (error_one & 0x0606)
 832                 process_ue(mci, error_one, info->dram_ded_add,
 833                         info->dram_scrb_add, error_found, handle_error);
 834
 835         if (error_next & 0x0606)
 836                 process_ue(mci, error_next, info->dram_ded_add,
 837                         info->dram_scrb_add, error_found, handle_error);
 838 }
 839
 840 static void e752x_get_error_info(struct mem_ctl_info *mci,
 841                                  struct e752x_error_info *info)
 842 {
 843         struct pci_dev *dev;
 844         struct e752x_pvt *pvt;
 845
 846         memset(info, 0, sizeof(*info));
 847         pvt = (struct e752x_pvt *)mci->pvt_info;
 848         dev = pvt->dev_d0f1;
 849         pci_read_config_dword(dev, E752X_FERR_GLOBAL, &info->ferr_global);
 850
 851         if (info->ferr_global) {
 852                 if (pvt->dev_info->err_dev == PCI_DEVICE_ID_INTEL_3100_1_ERR) {
 853                         pci_read_config_dword(dev, I3100_NSI_FERR,
 854                                              &info->nsi_ferr);
 855                         info->hi_ferr = 0;
 856                 } else {
 857                         pci_read_config_byte(dev, E752X_HI_FERR,
 858                                              &info->hi_ferr);
 859                         info->nsi_ferr = 0;
 860                 }
 861                 pci_read_config_word(dev, E752X_SYSBUS_FERR,
 862                                 &info->sysbus_ferr);
 863                 pci_read_config_byte(dev, E752X_BUF_FERR, &info->buf_ferr);
 864                 pci_read_config_word(dev, E752X_DRAM_FERR, &info->dram_ferr);
 865                 pci_read_config_dword(dev, E752X_DRAM_SEC1_ADD,
 866                                 &info->dram_sec1_add);
 867                 pci_read_config_word(dev, E752X_DRAM_SEC1_SYNDROME,
 868                                 &info->dram_sec1_syndrome);
 869                 pci_read_config_dword(dev, E752X_DRAM_DED_ADD,
 870                                 &info->dram_ded_add);
 871                 pci_read_config_dword(dev, E752X_DRAM_SCRB_ADD,
 872                                 &info->dram_scrb_add);
 873                 pci_read_config_dword(dev, E752X_DRAM_RETR_ADD,
 874                                 &info->dram_retr_add);
 875
 876                 /* ignore the reserved bits just in case */
 877                 if (info->hi_ferr & 0x7f)
 878                         pci_write_config_byte(dev, E752X_HI_FERR,
 879                                         info->hi_ferr);
 880
 881                 if (info->nsi_ferr & NSI_ERR_MASK)
 882                         pci_write_config_dword(dev, I3100_NSI_FERR,
 883                                         info->nsi_ferr);
 884
 885                 if (info->sysbus_ferr)
 886                         pci_write_config_word(dev, E752X_SYSBUS_FERR,
 887                                         info->sysbus_ferr);
 888
 889                 if (info->buf_ferr & 0x0f)
 890                         pci_write_config_byte(dev, E752X_BUF_FERR,
 891                                         info->buf_ferr);
 892
 893                 if (info->dram_ferr)
 894                         pci_write_bits16(pvt->bridge_ck, E752X_DRAM_FERR,
 895                                          info->dram_ferr, info->dram_ferr);
 896
 897                 pci_write_config_dword(dev, E752X_FERR_GLOBAL,
 898                                 info->ferr_global);
 899         }
 900
 901         pci_read_config_dword(dev, E752X_NERR_GLOBAL, &info->nerr_global);
 902
 903         if (info->nerr_global) {
 904                 if (pvt->dev_info->err_dev == PCI_DEVICE_ID_INTEL_3100_1_ERR) {
 905                         pci_read_config_dword(dev, I3100_NSI_NERR,
 906                                              &info->nsi_nerr);
 907                         info->hi_nerr = 0;
 908                 } else {
 909                         pci_read_config_byte(dev, E752X_HI_NERR,
 910                                              &info->hi_nerr);
 911                         info->nsi_nerr = 0;
 912                 }
 913                 pci_read_config_word(dev, E752X_SYSBUS_NERR,
 914                                 &info->sysbus_nerr);
 915                 pci_read_config_byte(dev, E752X_BUF_NERR, &info->buf_nerr);
 916                 pci_read_config_word(dev, E752X_DRAM_NERR, &info->dram_nerr);
 917                 pci_read_config_dword(dev, E752X_DRAM_SEC2_ADD,
 918                                 &info->dram_sec2_add);
 919                 pci_read_config_word(dev, E752X_DRAM_SEC2_SYNDROME,
 920                                 &info->dram_sec2_syndrome);
 921
 922                 if (info->hi_nerr & 0x7f)
 923                         pci_write_config_byte(dev, E752X_HI_NERR,
 924                                         info->hi_nerr);
 925
 926                 if (info->nsi_nerr & NSI_ERR_MASK)
 927                         pci_write_config_dword(dev, I3100_NSI_NERR,
 928                                         info->nsi_nerr);
 929
 930                 if (info->sysbus_nerr)
 931                         pci_write_config_word(dev, E752X_SYSBUS_NERR,
 932                                         info->sysbus_nerr);
 933
 934                 if (info->buf_nerr & 0x0f)
 935                         pci_write_config_byte(dev, E752X_BUF_NERR,
 936                                         info->buf_nerr);
 937
 938                 if (info->dram_nerr)
 939                         pci_write_bits16(pvt->bridge_ck, E752X_DRAM_NERR,
 940                                          info->dram_nerr, info->dram_nerr);
 941
 942                 pci_write_config_dword(dev, E752X_NERR_GLOBAL,
 943                                 info->nerr_global);
 944         }
 945 }
 946
 947 static int e752x_process_error_info(struct mem_ctl_info *mci,
 948                                 struct e752x_error_info *info,
 949                                 int handle_errors)
 950 {
 951         u32 error32, stat32;
 952         int error_found;
 953
 954         error_found = 0;
 955         error32 = (info->ferr_global >> 18) & 0x3ff;
 956         stat32 = (info->ferr_global >> 4) & 0x7ff;
 957
 958         if (error32)
 959                 global_error(1, error32, &error_found, handle_errors);
 960
 961         if (stat32)
 962                 global_error(0, stat32, &error_found, handle_errors);
 963
 964         error32 = (info->nerr_global >> 18) & 0x3ff;
 965         stat32 = (info->nerr_global >> 4) & 0x7ff;
 966
 967         if (error32)
 968                 global_error(1, error32, &error_found, handle_errors);
 969
 970         if (stat32)
 971                 global_error(0, stat32, &error_found, handle_errors);
 972
 973         e752x_check_hub_interface(info, &error_found, handle_errors);
 974         e752x_check_ns_interface(info, &error_found, handle_errors);
 975         e752x_check_sysbus(info, &error_found, handle_errors);
 976         e752x_check_membuf(info, &error_found, handle_errors);
 977         e752x_check_dram(mci, info, &error_found, handle_errors);
 978         return error_found;
 979 }
 980
 981 static void e752x_check(struct mem_ctl_info *mci)
 982 {
 983         struct e752x_error_info info;
 984
 985         edac_dbg(3, "\n");
 986         e752x_get_error_info(mci, &info);
 987         e752x_process_error_info(mci, &info, 1);
 988 }
 989
 990 /* Program byte/sec bandwidth scrub rate to hardware */
 991 static int set_sdram_scrub_rate(struct mem_ctl_info *mci, u32 new_bw)
 992 {
 993         const struct scrubrate *scrubrates;
 994         struct e752x_pvt *pvt = (struct e752x_pvt *) mci->pvt_info;
 995         struct pci_dev *pdev = pvt->dev_d0f0;
 996         int i;
 997
 998         if (pvt->dev_info->ctl_dev == PCI_DEVICE_ID_INTEL_3100_0)
 999                 scrubrates = scrubrates_i3100;
1000         else
1001                 scrubrates = scrubrates_e752x;
1002
1003         /* Translate the desired scrub rate to a e752x/3100 register value.
1004          * Search for the bandwidth that is equal or greater than the
1005          * desired rate and program the cooresponding register value.
1006          */
1007         for (i = 0; scrubrates[i].bandwidth != SDRATE_EOT; i++)
1008                 if (scrubrates[i].bandwidth >= new_bw)
1009                         break;
1010
1011         if (scrubrates[i].bandwidth == SDRATE_EOT)
1012                 return -1;
1013
1014         pci_write_config_word(pdev, E752X_MCHSCRB, scrubrates[i].scrubval);
1015
1016         return scrubrates[i].bandwidth;
1017 }
1018
1019 /* Convert current scrub rate value into byte/sec bandwidth */
1020 static int get_sdram_scrub_rate(struct mem_ctl_info *mci)
1021 {
1022         const struct scrubrate *scrubrates;
1023         struct e752x_pvt *pvt = (struct e752x_pvt *) mci->pvt_info;
1024         struct pci_dev *pdev = pvt->dev_d0f0;
1025         u16 scrubval;
1026         int i;
1027
1028         if (pvt->dev_info->ctl_dev == PCI_DEVICE_ID_INTEL_3100_0)
1029                 scrubrates = scrubrates_i3100;
1030         else
1031                 scrubrates = scrubrates_e752x;
1032
1033         /* Find the bandwidth matching the memory scrubber configuration */
1034         pci_read_config_word(pdev, E752X_MCHSCRB, &scrubval);
1035         scrubval = scrubval & 0x0f;
1036
1037         for (i = 0; scrubrates[i].bandwidth != SDRATE_EOT; i++)
1038                 if (scrubrates[i].scrubval == scrubval)
1039                         break;
1040
1041         if (scrubrates[i].bandwidth == SDRATE_EOT) {
1042                 e752x_printk(KERN_WARNING,
1043                         "Invalid sdram scrub control value: 0x%x\n", scrubval);
1044                 return -1;
1045         }
1046         return scrubrates[i].bandwidth;
1047
1048 }
1049
1050 /* Return 1 if dual channel mode is active.  Else return 0. */
1051 static inline int dual_channel_active(u16 ddrcsr)
1052 {
1053         return (((ddrcsr >> 12) & 3) == 3);
1054 }
1055
1056 /* Remap csrow index numbers if map_type is "reverse"
1057  */
1058 static inline int remap_csrow_index(struct mem_ctl_info *mci, int index)
1059 {
1060         struct e752x_pvt *pvt = mci->pvt_info;
1061
1062         if (!pvt->map_type)
1063                 return (7 - index);
1064
1065         return (index);
1066 }
1067
1068 static void e752x_init_csrows(struct mem_ctl_info *mci, struct pci_dev *pdev,
1069                         u16 ddrcsr)
1070 {
1071         struct csrow_info *csrow;
1072         enum edac_type edac_mode;
1073         unsigned long last_cumul_size;
1074         int index, mem_dev, drc_chan;
1075         int drc_drbg;           /* DRB granularity 0=64mb, 1=128mb */
1076         int drc_ddim;           /* DRAM Data Integrity Mode 0=none, 2=edac */
1077         u8 value;
1078         u32 dra, drc, cumul_size, i, nr_pages;
1079
1080         dra = 0;
1081         for (index = 0; index < 4; index++) {
1082                 u8 dra_reg;
1083                 pci_read_config_byte(pdev, E752X_DRA + index, &dra_reg);
1084                 dra |= dra_reg << (index * 8);
1085         }
1086         pci_read_config_dword(pdev, E752X_DRC, &drc);
1087         drc_chan = dual_channel_active(ddrcsr) ? 1 : 0;
1088         drc_drbg = drc_chan + 1;        /* 128 in dual mode, 64 in single */
1089         drc_ddim = (drc >> 20) & 0x3;
1090
1091         /* The dram row boundary (DRB) reg values are boundary address for
1092          * each DRAM row with a granularity of 64 or 128MB (single/dual
1093          * channel operation).  DRB regs are cumulative; therefore DRB7 will
1094          * contain the total memory contained in all eight rows.
1095          */
1096         for (last_cumul_size = index = 0; index < mci->nr_csrows; index++) {
1097                 /* mem_dev 0=x8, 1=x4 */
1098                 mem_dev = (dra >> (index * 4 + 2)) & 0x3;
1099                 csrow = mci->csrows[remap_csrow_index(mci, index)];
1100
1101                 mem_dev = (mem_dev == 2);
1102                 pci_read_config_byte(pdev, E752X_DRB + index, &value);
1103                 /* convert a 128 or 64 MiB DRB to a page size. */
1104                 cumul_size = value << (25 + drc_drbg - PAGE_SHIFT);
1105                 edac_dbg(3, "(%d) cumul_size 0x%x\n", index, cumul_size);
1106                 if (cumul_size == last_cumul_size)
1107                         continue;       /* not populated */
1108
1109                 csrow->first_page = last_cumul_size;
1110                 csrow->last_page = cumul_size - 1;
1111                 nr_pages = cumul_size - last_cumul_size;
1112                 last_cumul_size = cumul_size;
1113
1114                 /*
1115                 * if single channel or x8 devices then SECDED
1116                 * if dual channel and x4 then S4ECD4ED
1117                 */
1118                 if (drc_ddim) {
1119                         if (drc_chan && mem_dev) {
1120                                 edac_mode = EDAC_S4ECD4ED;
1121                                 mci->edac_cap |= EDAC_FLAG_S4ECD4ED;
1122                         } else {
1123                                 edac_mode = EDAC_SECDED;
1124                                 mci->edac_cap |= EDAC_FLAG_SECDED;
1125                         }
1126                 } else
1127                         edac_mode = EDAC_NONE;
1128                 for (i = 0; i < csrow->nr_channels; i++) {
1129                         struct dimm_info *dimm = csrow->channels[i]->dimm;
1130
1131                         edac_dbg(3, "Initializing rank at (%i,%i)\n", index, i);
1132                         dimm->nr_pages = nr_pages / csrow->nr_channels;
1133                         dimm->grain = 1 << 12;  /* 4KiB - resolution of CELOG */
1134                         dimm->mtype = MEM_RDDR; /* only one type supported */
1135                         dimm->dtype = mem_dev ? DEV_X4 : DEV_X8;
1136                         dimm->edac_mode = edac_mode;
1137                 }
1138         }
1139 }
1140
1141 static void e752x_init_mem_map_table(struct pci_dev *pdev,
1142                                 struct e752x_pvt *pvt)
1143 {
1144         int index;
1145         u8 value, last, row;
1146
1147         last = 0;
1148         row = 0;
1149
1150         for (index = 0; index < 8; index += 2) {
1151                 pci_read_config_byte(pdev, E752X_DRB + index, &value);
1152                 /* test if there is a dimm in this slot */
1153                 if (value == last) {
1154                         /* no dimm in the slot, so flag it as empty */
1155                         pvt->map[index] = 0xff;
1156                         pvt->map[index + 1] = 0xff;
1157                 } else {        /* there is a dimm in the slot */
1158                         pvt->map[index] = row;
1159                         row++;
1160                         last = value;
1161                         /* test the next value to see if the dimm is double
1162                          * sided
1163                          */
1164                         pci_read_config_byte(pdev, E752X_DRB + index + 1,
1165                                         &value);
1166
1167                         /* the dimm is single sided, so flag as empty */
1168                         /* this is a double sided dimm to save the next row #*/
1169                         pvt->map[index + 1] = (value == last) ? 0xff :  row;
1170                         row++;
1171                         last = value;
1172                 }
1173         }
1174 }
1175
1176 /* Return 0 on success or 1 on failure. */
1177 static int e752x_get_devs(struct pci_dev *pdev, int dev_idx,
1178                         struct e752x_pvt *pvt)
1179 {
1180         struct pci_dev *dev;
1181
1182         pvt->bridge_ck = pci_get_device(PCI_VENDOR_ID_INTEL,
1183                                 pvt->dev_info->err_dev, pvt->bridge_ck);
1184
1185         if (pvt->bridge_ck == NULL)
1186                 pvt->bridge_ck = pci_scan_single_device(pdev->bus,
1187                                                         PCI_DEVFN(0, 1));
1188
1189         if (pvt->bridge_ck == NULL) {
1190                 e752x_printk(KERN_ERR, "error reporting device not found:"
1191                         "vendor %x device 0x%x (broken BIOS?)\n",
1192                         PCI_VENDOR_ID_INTEL, e752x_devs[dev_idx].err_dev);
1193                 return 1;
1194         }
1195
1196         dev = pci_get_device(PCI_VENDOR_ID_INTEL,
1197                                 e752x_devs[dev_idx].ctl_dev,
1198                                 NULL);
1199
1200         if (dev == NULL)
1201                 goto fail;
1202
1203         pvt->dev_d0f0 = dev;
1204         pvt->dev_d0f1 = pci_dev_get(pvt->bridge_ck);
1205
1206         return 0;
1207
1208 fail:
1209         pci_dev_put(pvt->bridge_ck);
1210         return 1;
1211 }
1212
1213 /* Setup system bus parity mask register.
1214  * Sysbus parity supported on:
1215  * e7320/e7520/e7525 + Xeon
1216  */
1217 static void e752x_init_sysbus_parity_mask(struct e752x_pvt *pvt)
1218 {
1219         char *cpu_id = cpu_data(0).x86_model_id;
1220         struct pci_dev *dev = pvt->dev_d0f1;
1221         int enable = 1;
1222
1223         /* Allow module parameter override, else see if CPU supports parity */
1224         if (sysbus_parity != -1) {
1225                 enable = sysbus_parity;
1226         } else if (cpu_id[0] && !strstr(cpu_id, "Xeon")) {
1227                 e752x_printk(KERN_INFO, "System Bus Parity not "
1228                              "supported by CPU, disabling\n");
1229                 enable = 0;
1230         }
1231
1232         if (enable)
1233                 pci_write_config_word(dev, E752X_SYSBUS_ERRMASK, 0x0000);
1234         else
1235                 pci_write_config_word(dev, E752X_SYSBUS_ERRMASK, 0x0309);
1236 }
1237
1238 static void e752x_init_error_reporting_regs(struct e752x_pvt *pvt)
1239 {
1240         struct pci_dev *dev;
1241
1242         dev = pvt->dev_d0f1;
1243         /* Turn off error disable & SMI in case the BIOS turned it on */
1244         if (pvt->dev_info->err_dev == PCI_DEVICE_ID_INTEL_3100_1_ERR) {
1245                 pci_write_config_dword(dev, I3100_NSI_EMASK, 0);
1246                 pci_write_config_dword(dev, I3100_NSI_SMICMD, 0);
1247         } else {
1248                 pci_write_config_byte(dev, E752X_HI_ERRMASK, 0x00);
1249                 pci_write_config_byte(dev, E752X_HI_SMICMD, 0x00);
1250         }
1251
1252         e752x_init_sysbus_parity_mask(pvt);
1253
1254         pci_write_config_word(dev, E752X_SYSBUS_SMICMD, 0x00);
1255         pci_write_config_byte(dev, E752X_BUF_ERRMASK, 0x00);
1256         pci_write_config_byte(dev, E752X_BUF_SMICMD, 0x00);
1257         pci_write_config_byte(dev, E752X_DRAM_ERRMASK, 0x00);
1258         pci_write_config_byte(dev, E752X_DRAM_SMICMD, 0x00);
1259 }
1260
1261 static int e752x_probe1(struct pci_dev *pdev, int dev_idx)
1262 {
1263         u16 pci_data;
1264         u8 stat8;
1265         struct mem_ctl_info *mci;
1266         struct edac_mc_layer layers[2];
1267         struct e752x_pvt *pvt;
1268         u16 ddrcsr;
1269         int drc_chan;           /* Number of channels 0=1chan,1=2chan */
1270         struct e752x_error_info discard;
1271
1272         edac_dbg(0, "mci\n");
1273         edac_dbg(0, "Starting Probe1\n");
1274
1275         /* check to see if device 0 function 1 is enabled; if it isn't, we
1276          * assume the BIOS has reserved it for a reason and is expecting
1277          * exclusive access, we take care not to violate that assumption and
1278          * fail the probe. */
1279         pci_read_config_byte(pdev, E752X_DEVPRES1, &stat8);
1280         if (!force_function_unhide && !(stat8 & (1 << 5))) {
1281                 printk(KERN_INFO "Contact your BIOS vendor to see if the "
1282                         "E752x error registers can be safely un-hidden\n");
1283                 return -ENODEV;
1284         }
1285         stat8 |= (1 << 5);
1286         pci_write_config_byte(pdev, E752X_DEVPRES1, stat8);
1287
1288         pci_read_config_word(pdev, E752X_DDRCSR, &ddrcsr);
1289         /* FIXME: should check >>12 or 0xf, true for all? */
1290         /* Dual channel = 1, Single channel = 0 */
1291         drc_chan = dual_channel_active(ddrcsr);
1292
1293         layers[0].type = EDAC_MC_LAYER_CHIP_SELECT;
1294         layers[0].size = E752X_NR_CSROWS;
1295         layers[0].is_virt_csrow = true;
1296         layers[1].type = EDAC_MC_LAYER_CHANNEL;
1297         layers[1].size = drc_chan + 1;
1298         layers[1].is_virt_csrow = false;
1299         mci = edac_mc_alloc(0, ARRAY_SIZE(layers), layers, sizeof(*pvt));
1300         if (mci == NULL)
1301                 return -ENOMEM;
1302
1303         edac_dbg(3, "init mci\n");
1304         mci->mtype_cap = MEM_FLAG_RDDR;
1305         /* 3100 IMCH supports SECDEC only */
1306         mci->edac_ctl_cap = (dev_idx == I3100) ? EDAC_FLAG_SECDED :
1307                 (EDAC_FLAG_NONE | EDAC_FLAG_SECDED | EDAC_FLAG_S4ECD4ED);
1308         /* FIXME - what if different memory types are in different csrows? */
1309         mci->mod_name = EDAC_MOD_STR;
1310         mci->mod_ver = E752X_REVISION;
1311         mci->pdev = &pdev->dev;
1312
1313         edac_dbg(3, "init pvt\n");
1314         pvt = (struct e752x_pvt *)mci->pvt_info;
1315         pvt->dev_info = &e752x_devs[dev_idx];
1316         pvt->mc_symmetric = ((ddrcsr & 0x10) != 0);
1317
1318         if (e752x_get_devs(pdev, dev_idx, pvt)) {
1319                 edac_mc_free(mci);
1320                 return -ENODEV;
1321         }
1322
1323         edac_dbg(3, "more mci init\n");
1324         mci->ctl_name = pvt->dev_info->ctl_name;
1325         mci->dev_name = pci_name(pdev);
1326         mci->edac_check = e752x_check;
1327         mci->ctl_page_to_phys = ctl_page_to_phys;
1328         mci->set_sdram_scrub_rate = set_sdram_scrub_rate;
1329         mci->get_sdram_scrub_rate = get_sdram_scrub_rate;
1330
1331         /* set the map type.  1 = normal, 0 = reversed
1332          * Must be set before e752x_init_csrows in case csrow mapping
1333          * is reversed.
1334          */
1335         pci_read_config_byte(pdev, E752X_DRM, &stat8);
1336         pvt->map_type = ((stat8 & 0x0f) > ((stat8 >> 4) & 0x0f));
1337
1338         e752x_init_csrows(mci, pdev, ddrcsr);
1339         e752x_init_mem_map_table(pdev, pvt);
1340
1341         if (dev_idx == I3100)
1342                 mci->edac_cap = EDAC_FLAG_SECDED; /* the only mode supported */
1343         else
1344                 mci->edac_cap |= EDAC_FLAG_NONE;
1345         edac_dbg(3, "tolm, remapbase, remaplimit\n");
1346
1347         /* load the top of low memory, remap base, and remap limit vars */
1348         pci_read_config_word(pdev, E752X_TOLM, &pci_data);
1349         pvt->tolm = ((u32) pci_data) << 4;
1350         pci_read_config_word(pdev, E752X_REMAPBASE, &pci_data);
1351         pvt->remapbase = ((u32) pci_data) << 14;
1352         pci_read_config_word(pdev, E752X_REMAPLIMIT, &pci_data);
1353         pvt->remaplimit = ((u32) pci_data) << 14;
1354         e752x_printk(KERN_INFO,
1355                         "tolm = %x, remapbase = %x, remaplimit = %x\n",
1356                         pvt->tolm, pvt->remapbase, pvt->remaplimit);
1357
1358         /* Here we assume that we will never see multiple instances of this
1359          * type of memory controller.  The ID is therefore hardcoded to 0.
1360          */
1361         if (edac_mc_add_mc(mci)) {
1362                 edac_dbg(3, "failed edac_mc_add_mc()\n");
1363                 goto fail;
1364         }
1365
1366         e752x_init_error_reporting_regs(pvt);
1367         e752x_get_error_info(mci, &discard);    /* clear other MCH errors */
1368
1369         /* allocating generic PCI control info */
1370         e752x_pci = edac_pci_create_generic_ctl(&pdev->dev, EDAC_MOD_STR);
1371         if (!e752x_pci) {
1372                 printk(KERN_WARNING
1373                         "%s(): Unable to create PCI control\n", __func__);
1374                 printk(KERN_WARNING
1375                         "%s(): PCI error report via EDAC not setup\n",
1376                         __func__);
1377         }
1378
1379         /* get this far and it's successful */
1380         edac_dbg(3, "success\n");
1381         return 0;
1382
1383 fail:
1384         pci_dev_put(pvt->dev_d0f0);
1385         pci_dev_put(pvt->dev_d0f1);
1386         pci_dev_put(pvt->bridge_ck);
1387         edac_mc_free(mci);
1388
1389         return -ENODEV;
1390 }
1391
1392 /* returns count (>= 0), or negative on error */
1393 static int e752x_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
1394 {
1395         edac_dbg(0, "\n");
1396
1397         /* wake up and enable device */
1398         if (pci_enable_device(pdev) < 0)
1399                 return -EIO;
1400
1401         return e752x_probe1(pdev, ent->driver_data);
1402 }
1403
1404 static void e752x_remove_one(struct pci_dev *pdev)
1405 {
1406         struct mem_ctl_info *mci;
1407         struct e752x_pvt *pvt;
1408
1409         edac_dbg(0, "\n");
1410
1411         if (e752x_pci)
1412                 edac_pci_release_generic_ctl(e752x_pci);
1413
1414         if ((mci = edac_mc_del_mc(&pdev->dev)) == NULL)
1415                 return;
1416
1417         pvt = (struct e752x_pvt *)mci->pvt_info;
1418         pci_dev_put(pvt->dev_d0f0);
1419         pci_dev_put(pvt->dev_d0f1);
1420         pci_dev_put(pvt->bridge_ck);
1421         edac_mc_free(mci);
1422 }
1423
1424 static DEFINE_PCI_DEVICE_TABLE(e752x_pci_tbl) = {
1425         {
1426          PCI_VEND_DEV(INTEL, 7520_0), PCI_ANY_ID, PCI_ANY_ID, 0, 0,
1427          E7520},
1428         {
1429          PCI_VEND_DEV(INTEL, 7525_0), PCI_ANY_ID, PCI_ANY_ID, 0, 0,
1430          E7525},
1431         {
1432          PCI_VEND_DEV(INTEL, 7320_0), PCI_ANY_ID, PCI_ANY_ID, 0, 0,
1433          E7320},
1434         {
1435          PCI_VEND_DEV(INTEL, 3100_0), PCI_ANY_ID, PCI_ANY_ID, 0, 0,
1436          I3100},
1437         {
1438          0,
1439          }                      /* 0 terminated list. */
1440 };
1441
1442 MODULE_DEVICE_TABLE(pci, e752x_pci_tbl);
1443
1444 static struct pci_driver e752x_driver = {
1445         .name = EDAC_MOD_STR,
1446         .probe = e752x_init_one,
1447         .remove = e752x_remove_one,
1448         .id_table = e752x_pci_tbl,
1449 };
1450
1451 static int __init e752x_init(void)
1452 {
1453         int pci_rc;
1454
1455         edac_dbg(3, "\n");
1456
1457        /* Ensure that the OPSTATE is set correctly for POLL or NMI */
1458        opstate_init();
1459
1460         pci_rc = pci_register_driver(&e752x_driver);
1461         return (pci_rc < 0) ? pci_rc : 0;
1462 }
1463
1464 static void __exit e752x_exit(void)
1465 {
1466         edac_dbg(3, "\n");
1467         pci_unregister_driver(&e752x_driver);
1468 }
1469
1470 module_init(e752x_init);
1471 module_exit(e752x_exit);
1472
1473 MODULE_LICENSE("GPL");
1474 MODULE_AUTHOR("Linux Networx (http://lnxi.com) Tom Zimmerman\n");
1475 MODULE_DESCRIPTION("MC support for Intel e752x/3100 memory controllers");
1476
1477 module_param(force_function_unhide, int, 0444);
1478 MODULE_PARM_DESC(force_function_unhide, "if BIOS sets Dev0:Fun1 up as hidden:"
1479                  " 1=force unhide and hope BIOS doesn't fight driver for "
1480                 "Dev0:Fun1 access");
1481
1482 module_param(edac_op_state, int, 0444);
1483 MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI");
1484
1485 module_param(sysbus_parity, int, 0444);
1486 MODULE_PARM_DESC(sysbus_parity, "0=disable system bus parity checking,"
1487                 " 1=enable system bus parity checking, default=auto-detect");
1488 module_param(report_non_memory_errors, int, 0644);
1489 MODULE_PARM_DESC(report_non_memory_errors, "0=disable non-memory error "
1490                 "reporting, 1=enable non-memory error reporting");