drivers/edac/i7core_edac.c

   1 /* Intel i7 core/Nehalem Memory Controller kernel module
   2  *
   3  * This driver supports the memory controllers found on the Intel
   4  * processor families i7core, i7core 7xx/8xx, i5core, Xeon 35xx,
   5  * Xeon 55xx and Xeon 56xx also known as Nehalem, Nehalem-EP, Lynnfield
   6  * and Westmere-EP.
   7  *
   8  * This file may be distributed under the terms of the
   9  * GNU General Public License version 2 only.
  10  *
  11  * Copyright (c) 2009-2010 by:
  12  *       Mauro Carvalho Chehab <mchehab@redhat.com>
  13  *
  14  * Red Hat Inc. http://www.redhat.com
  15  *
  16  * Forked and adapted from the i5400_edac driver
  17  *
  18  * Based on the following public Intel datasheets:
  19  * Intel Core i7 Processor Extreme Edition and Intel Core i7 Processor
  20  * Datasheet, Volume 2:
  21  *      http://download.intel.com/design/processor/datashts/320835.pdf
  22  * Intel Xeon Processor 5500 Series Datasheet Volume 2
  23  *      http://www.intel.com/Assets/PDF/datasheet/321322.pdf
  24  * also available at:
  25  *      http://www.arrownac.com/manufacturers/intel/s/nehalem/5500-datasheet-v2.pdf
  26  */
  27
  28 #include <linux/module.h>
  29 #include <linux/init.h>
  30 #include <linux/pci.h>
  31 #include <linux/pci_ids.h>
  32 #include <linux/slab.h>
  33 #include <linux/delay.h>
  34 #include <linux/dmi.h>
  35 #include <linux/edac.h>
  36 #include <linux/mmzone.h>
  37 #include <linux/smp.h>
  38 #include <asm/mce.h>
  39 #include <asm/processor.h>
  40 #include <asm/div64.h>
  41
  42 #include "edac_core.h"
  43
  44 /* Static vars */
  45 static LIST_HEAD(i7core_edac_list);
  46 static DEFINE_MUTEX(i7core_edac_lock);
  47 static int probed;
  48
  49 static int use_pci_fixup;
  50 module_param(use_pci_fixup, int, 0444);
  51 MODULE_PARM_DESC(use_pci_fixup, "Enable PCI fixup to seek for hidden devices");
  52 /*
  53  * This is used for Nehalem-EP and Nehalem-EX devices, where the non-core
  54  * registers start at bus 255, and are not reported by BIOS.
  55  * We currently find devices with only 2 sockets. In order to support more QPI
  56  * Quick Path Interconnect, just increment this number.
  57  */
  58 #define MAX_SOCKET_BUSES        2
  59
  60
  61 /*
  62  * Alter this version for the module when modifications are made
  63  */
  64 #define I7CORE_REVISION    " Ver: 1.0.0"
  65 #define EDAC_MOD_STR      "i7core_edac"
  66
  67 /*
  68  * Debug macros
  69  */
  70 #define i7core_printk(level, fmt, arg...)                       \
  71         edac_printk(level, "i7core", fmt, ##arg)
  72
  73 #define i7core_mc_printk(mci, level, fmt, arg...)               \
  74         edac_mc_chipset_printk(mci, level, "i7core", fmt, ##arg)
  75
  76 /*
  77  * i7core Memory Controller Registers
  78  */
  79
  80         /* OFFSETS for Device 0 Function 0 */
  81
  82 #define MC_CFG_CONTROL  0x90
  83   #define MC_CFG_UNLOCK         0x02
  84   #define MC_CFG_LOCK           0x00
  85
  86         /* OFFSETS for Device 3 Function 0 */
  87
  88 #define MC_CONTROL      0x48
  89 #define MC_STATUS       0x4c
  90 #define MC_MAX_DOD      0x64
  91
  92 /*
  93  * OFFSETS for Device 3 Function 4, as inicated on Xeon 5500 datasheet:
  94  * http://www.arrownac.com/manufacturers/intel/s/nehalem/5500-datasheet-v2.pdf
  95  */
  96
  97 #define MC_TEST_ERR_RCV1        0x60
  98   #define DIMM2_COR_ERR(r)                      ((r) & 0x7fff)
  99
 100 #define MC_TEST_ERR_RCV0        0x64
 101   #define DIMM1_COR_ERR(r)                      (((r) >> 16) & 0x7fff)
 102   #define DIMM0_COR_ERR(r)                      ((r) & 0x7fff)
 103
 104 /* OFFSETS for Device 3 Function 2, as inicated on Xeon 5500 datasheet */
 105 #define MC_SSRCONTROL           0x48
 106   #define SSR_MODE_DISABLE      0x00
 107   #define SSR_MODE_ENABLE       0x01
 108   #define SSR_MODE_MASK         0x03
 109
 110 #define MC_SCRUB_CONTROL        0x4c
 111   #define STARTSCRUB            (1 << 24)
 112   #define SCRUBINTERVAL_MASK    0xffffff
 113
 114 #define MC_COR_ECC_CNT_0        0x80
 115 #define MC_COR_ECC_CNT_1        0x84
 116 #define MC_COR_ECC_CNT_2        0x88
 117 #define MC_COR_ECC_CNT_3        0x8c
 118 #define MC_COR_ECC_CNT_4        0x90
 119 #define MC_COR_ECC_CNT_5        0x94
 120
 121 #define DIMM_TOP_COR_ERR(r)                     (((r) >> 16) & 0x7fff)
 122 #define DIMM_BOT_COR_ERR(r)                     ((r) & 0x7fff)
 123
 124
 125         /* OFFSETS for Devices 4,5 and 6 Function 0 */
 126
 127 #define MC_CHANNEL_DIMM_INIT_PARAMS 0x58
 128   #define THREE_DIMMS_PRESENT           (1 << 24)
 129   #define SINGLE_QUAD_RANK_PRESENT      (1 << 23)
 130   #define QUAD_RANK_PRESENT             (1 << 22)
 131   #define REGISTERED_DIMM               (1 << 15)
 132
 133 #define MC_CHANNEL_MAPPER       0x60
 134   #define RDLCH(r, ch)          ((((r) >> (3 + (ch * 6))) & 0x07) - 1)
 135   #define WRLCH(r, ch)          ((((r) >> (ch * 6)) & 0x07) - 1)
 136
 137 #define MC_CHANNEL_RANK_PRESENT 0x7c
 138   #define RANK_PRESENT_MASK             0xffff
 139
 140 #define MC_CHANNEL_ADDR_MATCH   0xf0
 141 #define MC_CHANNEL_ERROR_MASK   0xf8
 142 #define MC_CHANNEL_ERROR_INJECT 0xfc
 143   #define INJECT_ADDR_PARITY    0x10
 144   #define INJECT_ECC            0x08
 145   #define MASK_CACHELINE        0x06
 146   #define MASK_FULL_CACHELINE   0x06
 147   #define MASK_MSB32_CACHELINE  0x04
 148   #define MASK_LSB32_CACHELINE  0x02
 149   #define NO_MASK_CACHELINE     0x00
 150   #define REPEAT_EN             0x01
 151
 152         /* OFFSETS for Devices 4,5 and 6 Function 1 */
 153
 154 #define MC_DOD_CH_DIMM0         0x48
 155 #define MC_DOD_CH_DIMM1         0x4c
 156 #define MC_DOD_CH_DIMM2         0x50
 157   #define RANKOFFSET_MASK       ((1 << 12) | (1 << 11) | (1 << 10))
 158   #define RANKOFFSET(x)         ((x & RANKOFFSET_MASK) >> 10)
 159   #define DIMM_PRESENT_MASK     (1 << 9)
 160   #define DIMM_PRESENT(x)       (((x) & DIMM_PRESENT_MASK) >> 9)
 161   #define MC_DOD_NUMBANK_MASK           ((1 << 8) | (1 << 7))
 162   #define MC_DOD_NUMBANK(x)             (((x) & MC_DOD_NUMBANK_MASK) >> 7)
 163   #define MC_DOD_NUMRANK_MASK           ((1 << 6) | (1 << 5))
 164   #define MC_DOD_NUMRANK(x)             (((x) & MC_DOD_NUMRANK_MASK) >> 5)
 165   #define MC_DOD_NUMROW_MASK            ((1 << 4) | (1 << 3) | (1 << 2))
 166   #define MC_DOD_NUMROW(x)              (((x) & MC_DOD_NUMROW_MASK) >> 2)
 167   #define MC_DOD_NUMCOL_MASK            3
 168   #define MC_DOD_NUMCOL(x)              ((x) & MC_DOD_NUMCOL_MASK)
 169
 170 #define MC_RANK_PRESENT         0x7c
 171
 172 #define MC_SAG_CH_0     0x80
 173 #define MC_SAG_CH_1     0x84
 174 #define MC_SAG_CH_2     0x88
 175 #define MC_SAG_CH_3     0x8c
 176 #define MC_SAG_CH_4     0x90
 177 #define MC_SAG_CH_5     0x94
 178 #define MC_SAG_CH_6     0x98
 179 #define MC_SAG_CH_7     0x9c
 180
 181 #define MC_RIR_LIMIT_CH_0       0x40
 182 #define MC_RIR_LIMIT_CH_1       0x44
 183 #define MC_RIR_LIMIT_CH_2       0x48
 184 #define MC_RIR_LIMIT_CH_3       0x4C
 185 #define MC_RIR_LIMIT_CH_4       0x50
 186 #define MC_RIR_LIMIT_CH_5       0x54
 187 #define MC_RIR_LIMIT_CH_6       0x58
 188 #define MC_RIR_LIMIT_CH_7       0x5C
 189 #define MC_RIR_LIMIT_MASK       ((1 << 10) - 1)
 190
 191 #define MC_RIR_WAY_CH           0x80
 192   #define MC_RIR_WAY_OFFSET_MASK        (((1 << 14) - 1) & ~0x7)
 193   #define MC_RIR_WAY_RANK_MASK          0x7
 194
 195 /*
 196  * i7core structs
 197  */
 198
 199 #define NUM_CHANS 3
 200 #define MAX_DIMMS 3             /* Max DIMMS per channel */
 201 #define MAX_MCR_FUNC  4
 202 #define MAX_CHAN_FUNC 3
 203
 204 struct i7core_info {
 205         u32     mc_control;
 206         u32     mc_status;
 207         u32     max_dod;
 208         u32     ch_map;
 209 };
 210
 211
 212 struct i7core_inject {
 213         int     enable;
 214
 215         u32     section;
 216         u32     type;
 217         u32     eccmask;
 218
 219         /* Error address mask */
 220         int channel, dimm, rank, bank, page, col;
 221 };
 222
 223 struct i7core_channel {
 224         bool            is_3dimms_present;
 225         bool            is_single_4rank;
 226         bool            has_4rank;
 227         u32             dimms;
 228 };
 229
 230 struct pci_id_descr {
 231         int                     dev;
 232         int                     func;
 233         int                     dev_id;
 234         int                     optional;
 235 };
 236
 237 struct pci_id_table {
 238         const struct pci_id_descr       *descr;
 239         int                             n_devs;
 240 };
 241
 242 struct i7core_dev {
 243         struct list_head        list;
 244         u8                      socket;
 245         struct pci_dev          **pdev;
 246         int                     n_devs;
 247         struct mem_ctl_info     *mci;
 248 };
 249
 250 struct i7core_pvt {
 251         struct device addrmatch_dev, chancounts_dev;
 252
 253         struct pci_dev  *pci_noncore;
 254         struct pci_dev  *pci_mcr[MAX_MCR_FUNC + 1];
 255         struct pci_dev  *pci_ch[NUM_CHANS][MAX_CHAN_FUNC + 1];
 256
 257         struct i7core_dev *i7core_dev;
 258
 259         struct i7core_info      info;
 260         struct i7core_inject    inject;
 261         struct i7core_channel   channel[NUM_CHANS];
 262
 263         int             ce_count_available;
 264
 265                         /* ECC corrected errors counts per udimm */
 266         unsigned long   udimm_ce_count[MAX_DIMMS];
 267         int             udimm_last_ce_count[MAX_DIMMS];
 268                         /* ECC corrected errors counts per rdimm */
 269         unsigned long   rdimm_ce_count[NUM_CHANS][MAX_DIMMS];
 270         int             rdimm_last_ce_count[NUM_CHANS][MAX_DIMMS];
 271
 272         bool            is_registered, enable_scrub;
 273
 274         /* Fifo double buffers */
 275         struct mce              mce_entry[MCE_LOG_LEN];
 276         struct mce              mce_outentry[MCE_LOG_LEN];
 277
 278         /* Fifo in/out counters */
 279         unsigned                mce_in, mce_out;
 280
 281         /* Count indicator to show errors not got */
 282         unsigned                mce_overrun;
 283
 284         /* DCLK Frequency used for computing scrub rate */
 285         int                     dclk_freq;
 286
 287         /* Struct to control EDAC polling */
 288         struct edac_pci_ctl_info *i7core_pci;
 289 };
 290
 291 #define PCI_DESCR(device, function, device_id)  \
 292         .dev = (device),                        \
 293         .func = (function),                     \
 294         .dev_id = (device_id)
 295
 296 static const struct pci_id_descr pci_dev_descr_i7core_nehalem[] = {
 297                 /* Memory controller */
 298         { PCI_DESCR(3, 0, PCI_DEVICE_ID_INTEL_I7_MCR)     },
 299         { PCI_DESCR(3, 1, PCI_DEVICE_ID_INTEL_I7_MC_TAD)  },
 300                         /* Exists only for RDIMM */
 301         { PCI_DESCR(3, 2, PCI_DEVICE_ID_INTEL_I7_MC_RAS), .optional = 1  },
 302         { PCI_DESCR(3, 4, PCI_DEVICE_ID_INTEL_I7_MC_TEST) },
 303
 304                 /* Channel 0 */
 305         { PCI_DESCR(4, 0, PCI_DEVICE_ID_INTEL_I7_MC_CH0_CTRL) },
 306         { PCI_DESCR(4, 1, PCI_DEVICE_ID_INTEL_I7_MC_CH0_ADDR) },
 307         { PCI_DESCR(4, 2, PCI_DEVICE_ID_INTEL_I7_MC_CH0_RANK) },
 308         { PCI_DESCR(4, 3, PCI_DEVICE_ID_INTEL_I7_MC_CH0_TC)   },
 309
 310                 /* Channel 1 */
 311         { PCI_DESCR(5, 0, PCI_DEVICE_ID_INTEL_I7_MC_CH1_CTRL) },
 312         { PCI_DESCR(5, 1, PCI_DEVICE_ID_INTEL_I7_MC_CH1_ADDR) },
 313         { PCI_DESCR(5, 2, PCI_DEVICE_ID_INTEL_I7_MC_CH1_RANK) },
 314         { PCI_DESCR(5, 3, PCI_DEVICE_ID_INTEL_I7_MC_CH1_TC)   },
 315
 316                 /* Channel 2 */
 317         { PCI_DESCR(6, 0, PCI_DEVICE_ID_INTEL_I7_MC_CH2_CTRL) },
 318         { PCI_DESCR(6, 1, PCI_DEVICE_ID_INTEL_I7_MC_CH2_ADDR) },
 319         { PCI_DESCR(6, 2, PCI_DEVICE_ID_INTEL_I7_MC_CH2_RANK) },
 320         { PCI_DESCR(6, 3, PCI_DEVICE_ID_INTEL_I7_MC_CH2_TC)   },
 321
 322                 /* Generic Non-core registers */
 323         /*
 324          * This is the PCI device on i7core and on Xeon 35xx (8086:2c41)
 325          * On Xeon 55xx, however, it has a different id (8086:2c40). So,
 326          * the probing code needs to test for the other address in case of
 327          * failure of this one
 328          */
 329         { PCI_DESCR(0, 0, PCI_DEVICE_ID_INTEL_I7_NONCORE)  },
 330
 331 };
 332
 333 static const struct pci_id_descr pci_dev_descr_lynnfield[] = {
 334         { PCI_DESCR( 3, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MCR)         },
 335         { PCI_DESCR( 3, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_TAD)      },
 336         { PCI_DESCR( 3, 4, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_TEST)     },
 337
 338         { PCI_DESCR( 4, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_CTRL) },
 339         { PCI_DESCR( 4, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_ADDR) },
 340         { PCI_DESCR( 4, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_RANK) },
 341         { PCI_DESCR( 4, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_TC)   },
 342
 343         { PCI_DESCR( 5, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_CTRL) },
 344         { PCI_DESCR( 5, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_ADDR) },
 345         { PCI_DESCR( 5, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_RANK) },
 346         { PCI_DESCR( 5, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_TC)   },
 347
 348         /*
 349          * This is the PCI device has an alternate address on some
 350          * processors like Core i7 860
 351          */
 352         { PCI_DESCR( 0, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE)     },
 353 };
 354
 355 static const struct pci_id_descr pci_dev_descr_i7core_westmere[] = {
 356                 /* Memory controller */
 357         { PCI_DESCR(3, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MCR_REV2)     },
 358         { PCI_DESCR(3, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_TAD_REV2)  },
 359                         /* Exists only for RDIMM */
 360         { PCI_DESCR(3, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_RAS_REV2), .optional = 1  },
 361         { PCI_DESCR(3, 4, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_TEST_REV2) },
 362
 363                 /* Channel 0 */
 364         { PCI_DESCR(4, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_CTRL_REV2) },
 365         { PCI_DESCR(4, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_ADDR_REV2) },
 366         { PCI_DESCR(4, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_RANK_REV2) },
 367         { PCI_DESCR(4, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_TC_REV2)   },
 368
 369                 /* Channel 1 */
 370         { PCI_DESCR(5, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_CTRL_REV2) },
 371         { PCI_DESCR(5, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_ADDR_REV2) },
 372         { PCI_DESCR(5, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_RANK_REV2) },
 373         { PCI_DESCR(5, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_TC_REV2)   },
 374
 375                 /* Channel 2 */
 376         { PCI_DESCR(6, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH2_CTRL_REV2) },
 377         { PCI_DESCR(6, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH2_ADDR_REV2) },
 378         { PCI_DESCR(6, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH2_RANK_REV2) },
 379         { PCI_DESCR(6, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH2_TC_REV2)   },
 380
 381                 /* Generic Non-core registers */
 382         { PCI_DESCR(0, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE_REV2)  },
 383
 384 };
 385
 386 #define PCI_ID_TABLE_ENTRY(A) { .descr=A, .n_devs = ARRAY_SIZE(A) }
 387 static const struct pci_id_table pci_dev_table[] = {
 388         PCI_ID_TABLE_ENTRY(pci_dev_descr_i7core_nehalem),
 389         PCI_ID_TABLE_ENTRY(pci_dev_descr_lynnfield),
 390         PCI_ID_TABLE_ENTRY(pci_dev_descr_i7core_westmere),
 391         {0,}                    /* 0 terminated list. */
 392 };
 393
 394 /*
 395  *      pci_device_id   table for which devices we are looking for
 396  */
 397 static DEFINE_PCI_DEVICE_TABLE(i7core_pci_tbl) = {
 398         {PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_X58_HUB_MGMT)},
 399         {PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_LYNNFIELD_QPI_LINK0)},
 400         {0,}                    /* 0 terminated list. */
 401 };
 402
 403 /****************************************************************************
 404                         Anciliary status routines
 405  ****************************************************************************/
 406
 407         /* MC_CONTROL bits */
 408 #define CH_ACTIVE(pvt, ch)      ((pvt)->info.mc_control & (1 << (8 + ch)))
 409 #define ECCx8(pvt)              ((pvt)->info.mc_control & (1 << 1))
 410
 411         /* MC_STATUS bits */
 412 #define ECC_ENABLED(pvt)        ((pvt)->info.mc_status & (1 << 4))
 413 #define CH_DISABLED(pvt, ch)    ((pvt)->info.mc_status & (1 << ch))
 414
 415         /* MC_MAX_DOD read functions */
 416 static inline int numdimms(u32 dimms)
 417 {
 418         return (dimms & 0x3) + 1;
 419 }
 420
 421 static inline int numrank(u32 rank)
 422 {
 423         static int ranks[4] = { 1, 2, 4, -EINVAL };
 424
 425         return ranks[rank & 0x3];
 426 }
 427
 428 static inline int numbank(u32 bank)
 429 {
 430         static int banks[4] = { 4, 8, 16, -EINVAL };
 431
 432         return banks[bank & 0x3];
 433 }
 434
 435 static inline int numrow(u32 row)
 436 {
 437         static int rows[8] = {
 438                 1 << 12, 1 << 13, 1 << 14, 1 << 15,
 439                 1 << 16, -EINVAL, -EINVAL, -EINVAL,
 440         };
 441
 442         return rows[row & 0x7];
 443 }
 444
 445 static inline int numcol(u32 col)
 446 {
 447         static int cols[8] = {
 448                 1 << 10, 1 << 11, 1 << 12, -EINVAL,
 449         };
 450         return cols[col & 0x3];
 451 }
 452
 453 static struct i7core_dev *get_i7core_dev(u8 socket)
 454 {
 455         struct i7core_dev *i7core_dev;
 456
 457         list_for_each_entry(i7core_dev, &i7core_edac_list, list) {
 458                 if (i7core_dev->socket == socket)
 459                         return i7core_dev;
 460         }
 461
 462         return NULL;
 463 }
 464
 465 static struct i7core_dev *alloc_i7core_dev(u8 socket,
 466                                            const struct pci_id_table *table)
 467 {
 468         struct i7core_dev *i7core_dev;
 469
 470         i7core_dev = kzalloc(sizeof(*i7core_dev), GFP_KERNEL);
 471         if (!i7core_dev)
 472                 return NULL;
 473
 474         i7core_dev->pdev = kzalloc(sizeof(*i7core_dev->pdev) * table->n_devs,
 475                                    GFP_KERNEL);
 476         if (!i7core_dev->pdev) {
 477                 kfree(i7core_dev);
 478                 return NULL;
 479         }
 480
 481         i7core_dev->socket = socket;
 482         i7core_dev->n_devs = table->n_devs;
 483         list_add_tail(&i7core_dev->list, &i7core_edac_list);
 484
 485         return i7core_dev;
 486 }
 487
 488 static void free_i7core_dev(struct i7core_dev *i7core_dev)
 489 {
 490         list_del(&i7core_dev->list);
 491         kfree(i7core_dev->pdev);
 492         kfree(i7core_dev);
 493 }
 494
 495 /****************************************************************************
 496                         Memory check routines
 497  ****************************************************************************/
 498
 499 static int get_dimm_config(struct mem_ctl_info *mci)
 500 {
 501         struct i7core_pvt *pvt = mci->pvt_info;
 502         struct pci_dev *pdev;
 503         int i, j;
 504         enum edac_type mode;
 505         enum mem_type mtype;
 506         struct dimm_info *dimm;
 507
 508         /* Get data from the MC register, function 0 */
 509         pdev = pvt->pci_mcr[0];
 510         if (!pdev)
 511                 return -ENODEV;
 512
 513         /* Device 3 function 0 reads */
 514         pci_read_config_dword(pdev, MC_CONTROL, &pvt->info.mc_control);
 515         pci_read_config_dword(pdev, MC_STATUS, &pvt->info.mc_status);
 516         pci_read_config_dword(pdev, MC_MAX_DOD, &pvt->info.max_dod);
 517         pci_read_config_dword(pdev, MC_CHANNEL_MAPPER, &pvt->info.ch_map);
 518
 519         debugf0("QPI %d control=0x%08x status=0x%08x dod=0x%08x map=0x%08x\n",
 520                 pvt->i7core_dev->socket, pvt->info.mc_control, pvt->info.mc_status,
 521                 pvt->info.max_dod, pvt->info.ch_map);
 522
 523         if (ECC_ENABLED(pvt)) {
 524                 debugf0("ECC enabled with x%d SDCC\n", ECCx8(pvt) ? 8 : 4);
 525                 if (ECCx8(pvt))
 526                         mode = EDAC_S8ECD8ED;
 527                 else
 528                         mode = EDAC_S4ECD4ED;
 529         } else {
 530                 debugf0("ECC disabled\n");
 531                 mode = EDAC_NONE;
 532         }
 533
 534         /* FIXME: need to handle the error codes */
 535         debugf0("DOD Max limits: DIMMS: %d, %d-ranked, %d-banked "
 536                 "x%x x 0x%x\n",
 537                 numdimms(pvt->info.max_dod),
 538                 numrank(pvt->info.max_dod >> 2),
 539                 numbank(pvt->info.max_dod >> 4),
 540                 numrow(pvt->info.max_dod >> 6),
 541                 numcol(pvt->info.max_dod >> 9));
 542
 543         for (i = 0; i < NUM_CHANS; i++) {
 544                 u32 data, dimm_dod[3], value[8];
 545
 546                 if (!pvt->pci_ch[i][0])
 547                         continue;
 548
 549                 if (!CH_ACTIVE(pvt, i)) {
 550                         debugf0("Channel %i is not active\n", i);
 551                         continue;
 552                 }
 553                 if (CH_DISABLED(pvt, i)) {
 554                         debugf0("Channel %i is disabled\n", i);
 555                         continue;
 556                 }
 557
 558                 /* Devices 4-6 function 0 */
 559                 pci_read_config_dword(pvt->pci_ch[i][0],
 560                                 MC_CHANNEL_DIMM_INIT_PARAMS, &data);
 561
 562
 563                 if (data & THREE_DIMMS_PRESENT)
 564                         pvt->channel[i].is_3dimms_present = true;
 565
 566                 if (data & SINGLE_QUAD_RANK_PRESENT)
 567                         pvt->channel[i].is_single_4rank = true;
 568
 569                 if (data & QUAD_RANK_PRESENT)
 570                         pvt->channel[i].has_4rank = true;
 571
 572                 if (data & REGISTERED_DIMM)
 573                         mtype = MEM_RDDR3;
 574                 else
 575                         mtype = MEM_DDR3;
 576
 577                 /* Devices 4-6 function 1 */
 578                 pci_read_config_dword(pvt->pci_ch[i][1],
 579                                 MC_DOD_CH_DIMM0, &dimm_dod[0]);
 580                 pci_read_config_dword(pvt->pci_ch[i][1],
 581                                 MC_DOD_CH_DIMM1, &dimm_dod[1]);
 582                 pci_read_config_dword(pvt->pci_ch[i][1],
 583                                 MC_DOD_CH_DIMM2, &dimm_dod[2]);
 584
 585                 debugf0("Ch%d phy rd%d, wr%d (0x%08x): "
 586                         "%s%s%s%cDIMMs\n",
 587                         i,
 588                         RDLCH(pvt->info.ch_map, i), WRLCH(pvt->info.ch_map, i),
 589                         data,
 590                         pvt->channel[i].is_3dimms_present ? "3DIMMS " : "",
 591                         pvt->channel[i].is_3dimms_present ? "SINGLE_4R " : "",
 592                         pvt->channel[i].has_4rank ? "HAS_4R " : "",
 593                         (data & REGISTERED_DIMM) ? 'R' : 'U');
 594
 595                 for (j = 0; j < 3; j++) {
 596                         u32 banks, ranks, rows, cols;
 597                         u32 size, npages;
 598
 599                         if (!DIMM_PRESENT(dimm_dod[j]))
 600                                 continue;
 601
 602                         dimm = EDAC_DIMM_PTR(mci->layers, mci->dimms, mci->n_layers,
 603                                        i, j, 0);
 604                         banks = numbank(MC_DOD_NUMBANK(dimm_dod[j]));
 605                         ranks = numrank(MC_DOD_NUMRANK(dimm_dod[j]));
 606                         rows = numrow(MC_DOD_NUMROW(dimm_dod[j]));
 607                         cols = numcol(MC_DOD_NUMCOL(dimm_dod[j]));
 608
 609                         /* DDR3 has 8 I/O banks */
 610                         size = (rows * cols * banks * ranks) >> (20 - 3);
 611
 612                         debugf0("\tdimm %d %d Mb offset: %x, "
 613                                 "bank: %d, rank: %d, row: %#x, col: %#x\n",
 614                                 j, size,
 615                                 RANKOFFSET(dimm_dod[j]),
 616                                 banks, ranks, rows, cols);
 617
 618                         npages = MiB_TO_PAGES(size);
 619
 620                         dimm->nr_pages = npages;
 621
 622                         switch (banks) {
 623                         case 4:
 624                                 dimm->dtype = DEV_X4;
 625                                 break;
 626                         case 8:
 627                                 dimm->dtype = DEV_X8;
 628                                 break;
 629                         case 16:
 630                                 dimm->dtype = DEV_X16;
 631                                 break;
 632                         default:
 633                                 dimm->dtype = DEV_UNKNOWN;
 634                         }
 635
 636                         snprintf(dimm->label, sizeof(dimm->label),
 637                                  "CPU#%uChannel#%u_DIMM#%u",
 638                                  pvt->i7core_dev->socket, i, j);
 639                         dimm->grain = 8;
 640                         dimm->edac_mode = mode;
 641                         dimm->mtype = mtype;
 642                 }
 643
 644                 pci_read_config_dword(pdev, MC_SAG_CH_0, &value[0]);
 645                 pci_read_config_dword(pdev, MC_SAG_CH_1, &value[1]);
 646                 pci_read_config_dword(pdev, MC_SAG_CH_2, &value[2]);
 647                 pci_read_config_dword(pdev, MC_SAG_CH_3, &value[3]);
 648                 pci_read_config_dword(pdev, MC_SAG_CH_4, &value[4]);
 649                 pci_read_config_dword(pdev, MC_SAG_CH_5, &value[5]);
 650                 pci_read_config_dword(pdev, MC_SAG_CH_6, &value[6]);
 651                 pci_read_config_dword(pdev, MC_SAG_CH_7, &value[7]);
 652                 debugf1("\t[%i] DIVBY3\tREMOVED\tOFFSET\n", i);
 653                 for (j = 0; j < 8; j++)
 654                         debugf1("\t\t%#x\t%#x\t%#x\n",
 655                                 (value[j] >> 27) & 0x1,
 656                                 (value[j] >> 24) & 0x7,
 657                                 (value[j] & ((1 << 24) - 1)));
 658         }
 659
 660         return 0;
 661 }
 662
 663 /****************************************************************************
 664                         Error insertion routines
 665  ****************************************************************************/
 666
 667 #define to_mci(k) container_of(k, struct mem_ctl_info, dev)
 668
 669 /* The i7core has independent error injection features per channel.
 670    However, to have a simpler code, we don't allow enabling error injection
 671    on more than one channel.
 672    Also, since a change at an inject parameter will be applied only at enable,
 673    we're disabling error injection on all write calls to the sysfs nodes that
 674    controls the error code injection.
 675  */
 676 static int disable_inject(const struct mem_ctl_info *mci)
 677 {
 678         struct i7core_pvt *pvt = mci->pvt_info;
 679
 680         pvt->inject.enable = 0;
 681
 682         if (!pvt->pci_ch[pvt->inject.channel][0])
 683                 return -ENODEV;
 684
 685         pci_write_config_dword(pvt->pci_ch[pvt->inject.channel][0],
 686                                 MC_CHANNEL_ERROR_INJECT, 0);
 687
 688         return 0;
 689 }
 690
 691 /*
 692  * i7core inject inject.section
 693  *
 694  *      accept and store error injection inject.section value
 695  *      bit 0 - refers to the lower 32-byte half cacheline
 696  *      bit 1 - refers to the upper 32-byte half cacheline
 697  */
 698 static ssize_t i7core_inject_section_store(struct device *dev,
 699                                            struct device_attribute *mattr,
 700                                            const char *data, size_t count)
 701 {
 702         struct mem_ctl_info *mci = to_mci(dev);
 703         struct i7core_pvt *pvt = mci->pvt_info;
 704         unsigned long value;
 705         int rc;
 706
 707         if (pvt->inject.enable)
 708                 disable_inject(mci);
 709
 710         rc = strict_strtoul(data, 10, &value);
 711         if ((rc < 0) || (value > 3))
 712                 return -EIO;
 713
 714         pvt->inject.section = (u32) value;
 715         return count;
 716 }
 717
 718 static ssize_t i7core_inject_section_show(struct device *dev,
 719                                           struct device_attribute *mattr,
 720                                           char *data)
 721 {
 722         struct mem_ctl_info *mci = to_mci(dev);
 723         struct i7core_pvt *pvt = mci->pvt_info;
 724         return sprintf(data, "0x%08x\n", pvt->inject.section);
 725 }
 726
 727 /*
 728  * i7core inject.type
 729  *
 730  *      accept and store error injection inject.section value
 731  *      bit 0 - repeat enable - Enable error repetition
 732  *      bit 1 - inject ECC error
 733  *      bit 2 - inject parity error
 734  */
 735 static ssize_t i7core_inject_type_store(struct device *dev,
 736                                         struct device_attribute *mattr,
 737                                         const char *data, size_t count)
 738 {
 739         struct mem_ctl_info *mci = to_mci(dev);
 740 struct i7core_pvt *pvt = mci->pvt_info;
 741         unsigned long value;
 742         int rc;
 743
 744         if (pvt->inject.enable)
 745                 disable_inject(mci);
 746
 747         rc = strict_strtoul(data, 10, &value);
 748         if ((rc < 0) || (value > 7))
 749                 return -EIO;
 750
 751         pvt->inject.type = (u32) value;
 752         return count;
 753 }
 754
 755 static ssize_t i7core_inject_type_show(struct device *dev,
 756                                        struct device_attribute *mattr,
 757                                        char *data)
 758 {
 759         struct mem_ctl_info *mci = to_mci(dev);
 760         struct i7core_pvt *pvt = mci->pvt_info;
 761
 762         return sprintf(data, "0x%08x\n", pvt->inject.type);
 763 }
 764
 765 /*
 766  * i7core_inject_inject.eccmask_store
 767  *
 768  * The type of error (UE/CE) will depend on the inject.eccmask value:
 769  *   Any bits set to a 1 will flip the corresponding ECC bit
 770  *   Correctable errors can be injected by flipping 1 bit or the bits within
 771  *   a symbol pair (2 consecutive aligned 8-bit pairs - i.e. 7:0 and 15:8 or
 772  *   23:16 and 31:24). Flipping bits in two symbol pairs will cause an
 773  *   uncorrectable error to be injected.
 774  */
 775 static ssize_t i7core_inject_eccmask_store(struct device *dev,
 776                                            struct device_attribute *mattr,
 777                                            const char *data, size_t count)
 778 {
 779         struct mem_ctl_info *mci = to_mci(dev);
 780         struct i7core_pvt *pvt = mci->pvt_info;
 781         unsigned long value;
 782         int rc;
 783
 784         if (pvt->inject.enable)
 785                 disable_inject(mci);
 786
 787         rc = strict_strtoul(data, 10, &value);
 788         if (rc < 0)
 789                 return -EIO;
 790
 791         pvt->inject.eccmask = (u32) value;
 792         return count;
 793 }
 794
 795 static ssize_t i7core_inject_eccmask_show(struct device *dev,
 796                                           struct device_attribute *mattr,
 797                                           char *data)
 798 {
 799         struct mem_ctl_info *mci = to_mci(dev);
 800         struct i7core_pvt *pvt = mci->pvt_info;
 801
 802         return sprintf(data, "0x%08x\n", pvt->inject.eccmask);
 803 }
 804
 805 /*
 806  * i7core_addrmatch
 807  *
 808  * The type of error (UE/CE) will depend on the inject.eccmask value:
 809  *   Any bits set to a 1 will flip the corresponding ECC bit
 810  *   Correctable errors can be injected by flipping 1 bit or the bits within
 811  *   a symbol pair (2 consecutive aligned 8-bit pairs - i.e. 7:0 and 15:8 or
 812  *   23:16 and 31:24). Flipping bits in two symbol pairs will cause an
 813  *   uncorrectable error to be injected.
 814  */
 815
 816 #define DECLARE_ADDR_MATCH(param, limit)                        \
 817 static ssize_t i7core_inject_store_##param(                     \
 818         struct device *dev,                                     \
 819         struct device_attribute *mattr,                         \
 820         const char *data, size_t count)                         \
 821 {                                                               \
 822         struct mem_ctl_info *mci = to_mci(dev);                 \
 823         struct i7core_pvt *pvt;                                 \
 824         long value;                                             \
 825         int rc;                                                 \
 826                                                                 \
 827         debugf1("%s()\n", __func__);                            \
 828         pvt = mci->pvt_info;                                    \
 829                                                                 \
 830         if (pvt->inject.enable)                                 \
 831                 disable_inject(mci);                            \
 832                                                                 \
 833         if (!strcasecmp(data, "any") || !strcasecmp(data, "any\n"))\
 834                 value = -1;                                     \
 835         else {                                                  \
 836                 rc = strict_strtoul(data, 10, &value);          \
 837                 if ((rc < 0) || (value >= limit))               \
 838                         return -EIO;                            \
 839         }                                                       \
 840                                                                 \
 841         pvt->inject.param = value;                              \
 842                                                                 \
 843         return count;                                           \
 844 }                                                               \
 845                                                                 \
 846 static ssize_t i7core_inject_show_##param(                      \
 847         struct device *dev,                                     \
 848         struct device_attribute *mattr,                         \
 849         char *data)                                             \
 850 {                                                               \
 851         struct mem_ctl_info *mci = to_mci(dev);                 \
 852         struct i7core_pvt *pvt;                                 \
 853                                                                 \
 854         pvt = mci->pvt_info;                                    \
 855         debugf1("%s() pvt=%p\n", __func__, pvt);                \
 856         if (pvt->inject.param < 0)                              \
 857                 return sprintf(data, "any\n");                  \
 858         else                                                    \
 859                 return sprintf(data, "%d\n", pvt->inject.param);\
 860 }
 861
 862 #define ATTR_ADDR_MATCH(param)                                  \
 863         static DEVICE_ATTR(param, S_IRUGO | S_IWUSR,            \
 864                     i7core_inject_show_##param,                 \
 865                     i7core_inject_store_##param)
 866
 867 DECLARE_ADDR_MATCH(channel, 3);
 868 DECLARE_ADDR_MATCH(dimm, 3);
 869 DECLARE_ADDR_MATCH(rank, 4);
 870 DECLARE_ADDR_MATCH(bank, 32);
 871 DECLARE_ADDR_MATCH(page, 0x10000);
 872 DECLARE_ADDR_MATCH(col, 0x4000);
 873
 874 ATTR_ADDR_MATCH(channel);
 875 ATTR_ADDR_MATCH(dimm);
 876 ATTR_ADDR_MATCH(rank);
 877 ATTR_ADDR_MATCH(bank);
 878 ATTR_ADDR_MATCH(page);
 879 ATTR_ADDR_MATCH(col);
 880
 881 static int write_and_test(struct pci_dev *dev, const int where, const u32 val)
 882 {
 883         u32 read;
 884         int count;
 885
 886         debugf0("setting pci %02x:%02x.%x reg=%02x value=%08x\n",
 887                 dev->bus->number, PCI_SLOT(dev->devfn), PCI_FUNC(dev->devfn),
 888                 where, val);
 889
 890         for (count = 0; count < 10; count++) {
 891                 if (count)
 892                         msleep(100);
 893                 pci_write_config_dword(dev, where, val);
 894                 pci_read_config_dword(dev, where, &read);
 895
 896                 if (read == val)
 897                         return 0;
 898         }
 899
 900         i7core_printk(KERN_ERR, "Error during set pci %02x:%02x.%x reg=%02x "
 901                 "write=%08x. Read=%08x\n",
 902                 dev->bus->number, PCI_SLOT(dev->devfn), PCI_FUNC(dev->devfn),
 903                 where, val, read);
 904
 905         return -EINVAL;
 906 }
 907
 908 /*
 909  * This routine prepares the Memory Controller for error injection.
 910  * The error will be injected when some process tries to write to the
 911  * memory that matches the given criteria.
 912  * The criteria can be set in terms of a mask where dimm, rank, bank, page
 913  * and col can be specified.
 914  * A -1 value for any of the mask items will make the MCU to ignore
 915  * that matching criteria for error injection.
 916  *
 917  * It should be noticed that the error will only happen after a write operation
 918  * on a memory that matches the condition. if REPEAT_EN is not enabled at
 919  * inject mask, then it will produce just one error. Otherwise, it will repeat
 920  * until the injectmask would be cleaned.
 921  *
 922  * FIXME: This routine assumes that MAXNUMDIMMS value of MC_MAX_DOD
 923  *    is reliable enough to check if the MC is using the
 924  *    three channels. However, this is not clear at the datasheet.
 925  */
 926 static ssize_t i7core_inject_enable_store(struct device *dev,
 927                                           struct device_attribute *mattr,
 928                                           const char *data, size_t count)
 929 {
 930         struct mem_ctl_info *mci = to_mci(dev);
 931         struct i7core_pvt *pvt = mci->pvt_info;
 932         u32 injectmask;
 933         u64 mask = 0;
 934         int  rc;
 935         long enable;
 936
 937         if (!pvt->pci_ch[pvt->inject.channel][0])
 938                 return 0;
 939
 940         rc = strict_strtoul(data, 10, &enable);
 941         if ((rc < 0))
 942                 return 0;
 943
 944         if (enable) {
 945                 pvt->inject.enable = 1;
 946         } else {
 947                 disable_inject(mci);
 948                 return count;
 949         }
 950
 951         /* Sets pvt->inject.dimm mask */
 952         if (pvt->inject.dimm < 0)
 953                 mask |= 1LL << 41;
 954         else {
 955                 if (pvt->channel[pvt->inject.channel].dimms > 2)
 956                         mask |= (pvt->inject.dimm & 0x3LL) << 35;
 957                 else
 958                         mask |= (pvt->inject.dimm & 0x1LL) << 36;
 959         }
 960
 961         /* Sets pvt->inject.rank mask */
 962         if (pvt->inject.rank < 0)
 963                 mask |= 1LL << 40;
 964         else {
 965                 if (pvt->channel[pvt->inject.channel].dimms > 2)
 966                         mask |= (pvt->inject.rank & 0x1LL) << 34;
 967                 else
 968                         mask |= (pvt->inject.rank & 0x3LL) << 34;
 969         }
 970
 971         /* Sets pvt->inject.bank mask */
 972         if (pvt->inject.bank < 0)
 973                 mask |= 1LL << 39;
 974         else
 975                 mask |= (pvt->inject.bank & 0x15LL) << 30;
 976
 977         /* Sets pvt->inject.page mask */
 978         if (pvt->inject.page < 0)
 979                 mask |= 1LL << 38;
 980         else
 981                 mask |= (pvt->inject.page & 0xffff) << 14;
 982
 983         /* Sets pvt->inject.column mask */
 984         if (pvt->inject.col < 0)
 985                 mask |= 1LL << 37;
 986         else
 987                 mask |= (pvt->inject.col & 0x3fff);
 988
 989         /*
 990          * bit    0: REPEAT_EN
 991          * bits 1-2: MASK_HALF_CACHELINE
 992          * bit    3: INJECT_ECC
 993          * bit    4: INJECT_ADDR_PARITY
 994          */
 995
 996         injectmask = (pvt->inject.type & 1) |
 997                      (pvt->inject.section & 0x3) << 1 |
 998                      (pvt->inject.type & 0x6) << (3 - 1);
 999
1000         /* Unlock writes to registers - this register is write only */
1001         pci_write_config_dword(pvt->pci_noncore,
1002                                MC_CFG_CONTROL, 0x2);
1003
1004         write_and_test(pvt->pci_ch[pvt->inject.channel][0],
1005                                MC_CHANNEL_ADDR_MATCH, mask);
1006         write_and_test(pvt->pci_ch[pvt->inject.channel][0],
1007                                MC_CHANNEL_ADDR_MATCH + 4, mask >> 32L);
1008
1009         write_and_test(pvt->pci_ch[pvt->inject.channel][0],
1010                                MC_CHANNEL_ERROR_MASK, pvt->inject.eccmask);
1011
1012         write_and_test(pvt->pci_ch[pvt->inject.channel][0],
1013                                MC_CHANNEL_ERROR_INJECT, injectmask);
1014
1015         /*
1016          * This is something undocumented, based on my tests
1017          * Without writing 8 to this register, errors aren't injected. Not sure
1018          * why.
1019          */
1020         pci_write_config_dword(pvt->pci_noncore,
1021                                MC_CFG_CONTROL, 8);
1022
1023         debugf0("Error inject addr match 0x%016llx, ecc 0x%08x,"
1024                 " inject 0x%08x\n",
1025                 mask, pvt->inject.eccmask, injectmask);
1026
1027
1028         return count;
1029 }
1030
1031 static ssize_t i7core_inject_enable_show(struct device *dev,
1032                                          struct device_attribute *mattr,
1033                                          char *data)
1034 {
1035         struct mem_ctl_info *mci = to_mci(dev);
1036         struct i7core_pvt *pvt = mci->pvt_info;
1037         u32 injectmask;
1038
1039         if (!pvt->pci_ch[pvt->inject.channel][0])
1040                 return 0;
1041
1042         pci_read_config_dword(pvt->pci_ch[pvt->inject.channel][0],
1043                                MC_CHANNEL_ERROR_INJECT, &injectmask);
1044
1045         debugf0("Inject error read: 0x%018x\n", injectmask);
1046
1047         if (injectmask & 0x0c)
1048                 pvt->inject.enable = 1;
1049
1050         return sprintf(data, "%d\n", pvt->inject.enable);
1051 }
1052
1053 #define DECLARE_COUNTER(param)                                  \
1054 static ssize_t i7core_show_counter_##param(                     \
1055         struct device *dev,                                     \
1056         struct device_attribute *mattr,                         \
1057         char *data)                                             \
1058 {                                                               \
1059         struct mem_ctl_info *mci = to_mci(dev);                 \
1060         struct i7core_pvt *pvt = mci->pvt_info;                 \
1061                                                                 \
1062         debugf1("%s()\n", __func__);                            \
1063         if (!pvt->ce_count_available || (pvt->is_registered))   \
1064                 return sprintf(data, "data unavailable\n");     \
1065         return sprintf(data, "%lu\n",                           \
1066                         pvt->udimm_ce_count[param]);            \
1067 }
1068
1069 #define ATTR_COUNTER(param)                                     \
1070         static DEVICE_ATTR(udimm##param, S_IRUGO | S_IWUSR,     \
1071                     i7core_show_counter_##param,                \
1072                     NULL)
1073
1074 DECLARE_COUNTER(0);
1075 DECLARE_COUNTER(1);
1076 DECLARE_COUNTER(2);
1077
1078 ATTR_COUNTER(0);
1079 ATTR_COUNTER(1);
1080 ATTR_COUNTER(2);
1081
1082 /*
1083  * inject_addrmatch device sysfs struct
1084  */
1085
1086 static struct attribute *i7core_addrmatch_attrs[] = {
1087         &dev_attr_channel.attr,
1088         &dev_attr_dimm.attr,
1089         &dev_attr_rank.attr,
1090         &dev_attr_bank.attr,
1091         &dev_attr_page.attr,
1092         &dev_attr_col.attr,
1093         NULL
1094 };
1095
1096 static struct attribute_group addrmatch_grp = {
1097         .attrs  = i7core_addrmatch_attrs,
1098 };
1099
1100 static const struct attribute_group *addrmatch_groups[] = {
1101         &addrmatch_grp,
1102         NULL
1103 };
1104
1105 static void addrmatch_release(struct device *device)
1106 {
1107         debugf1("Releasing device %s\n", dev_name(device));
1108 }
1109
1110 static struct device_type addrmatch_type = {
1111         .groups         = addrmatch_groups,
1112         .release        = addrmatch_release,
1113 };
1114
1115 /*
1116  * all_channel_counts sysfs struct
1117  */
1118
1119 static struct attribute *i7core_udimm_counters_attrs[] = {
1120         &dev_attr_udimm0.attr,
1121         &dev_attr_udimm1.attr,
1122         &dev_attr_udimm2.attr,
1123         NULL
1124 };
1125
1126 static struct attribute_group all_channel_counts_grp = {
1127         .attrs  = i7core_udimm_counters_attrs,
1128 };
1129
1130 static const struct attribute_group *all_channel_counts_groups[] = {
1131         &all_channel_counts_grp,
1132         NULL
1133 };
1134
1135 static void all_channel_counts_release(struct device *device)
1136 {
1137         debugf1("Releasing device %s\n", dev_name(device));
1138 }
1139
1140 static struct device_type all_channel_counts_type = {
1141         .groups         = all_channel_counts_groups,
1142         .release        = all_channel_counts_release,
1143 };
1144
1145 /*
1146  * inject sysfs attributes
1147  */
1148
1149 static DEVICE_ATTR(inject_section, S_IRUGO | S_IWUSR,
1150                    i7core_inject_section_show, i7core_inject_section_store);
1151
1152 static DEVICE_ATTR(inject_type, S_IRUGO | S_IWUSR,
1153                    i7core_inject_type_show, i7core_inject_type_store);
1154
1155
1156 static DEVICE_ATTR(inject_eccmask, S_IRUGO | S_IWUSR,
1157                    i7core_inject_eccmask_show, i7core_inject_eccmask_store);
1158
1159 static DEVICE_ATTR(inject_enable, S_IRUGO | S_IWUSR,
1160                    i7core_inject_enable_show, i7core_inject_enable_store);
1161
1162 static int i7core_create_sysfs_devices(struct mem_ctl_info *mci)
1163 {
1164         struct i7core_pvt *pvt = mci->pvt_info;
1165         int rc;
1166
1167         rc = device_create_file(&mci->dev, &dev_attr_inject_section);
1168         if (rc < 0)
1169                 return rc;
1170         rc = device_create_file(&mci->dev, &dev_attr_inject_type);
1171         if (rc < 0)
1172                 return rc;
1173         rc = device_create_file(&mci->dev, &dev_attr_inject_eccmask);
1174         if (rc < 0)
1175                 return rc;
1176         rc = device_create_file(&mci->dev, &dev_attr_inject_enable);
1177         if (rc < 0)
1178                 return rc;
1179
1180         pvt->addrmatch_dev.type = &addrmatch_type;
1181         pvt->addrmatch_dev.bus = mci->dev.bus;
1182         device_initialize(&pvt->addrmatch_dev);
1183         pvt->addrmatch_dev.parent = &mci->dev;
1184         dev_set_name(&pvt->addrmatch_dev, "inject_addrmatch");
1185         dev_set_drvdata(&pvt->addrmatch_dev, mci);
1186
1187         debugf1("%s(): creating %s\n", __func__,
1188                 dev_name(&pvt->addrmatch_dev));
1189
1190         rc = device_add(&pvt->addrmatch_dev);
1191         if (rc < 0)
1192                 return rc;
1193
1194         if (!pvt->is_registered) {
1195                 pvt->chancounts_dev.type = &all_channel_counts_type;
1196                 pvt->chancounts_dev.bus = mci->dev.bus;
1197                 device_initialize(&pvt->chancounts_dev);
1198                 pvt->chancounts_dev.parent = &mci->dev;
1199                 dev_set_name(&pvt->chancounts_dev, "all_channel_counts");
1200                 dev_set_drvdata(&pvt->chancounts_dev, mci);
1201
1202                 debugf1("%s(): creating %s\n", __func__,
1203                         dev_name(&pvt->chancounts_dev));
1204
1205                 rc = device_add(&pvt->chancounts_dev);
1206                 if (rc < 0)
1207                         return rc;
1208         }
1209         return 0;
1210 }
1211
1212 static void i7core_delete_sysfs_devices(struct mem_ctl_info *mci)
1213 {
1214         struct i7core_pvt *pvt = mci->pvt_info;
1215
1216         debugf1("\n");
1217
1218         device_remove_file(&mci->dev, &dev_attr_inject_section);
1219         device_remove_file(&mci->dev, &dev_attr_inject_type);
1220         device_remove_file(&mci->dev, &dev_attr_inject_eccmask);
1221         device_remove_file(&mci->dev, &dev_attr_inject_enable);
1222
1223         if (!pvt->is_registered) {
1224                 put_device(&pvt->chancounts_dev);
1225                 device_del(&pvt->chancounts_dev);
1226         }
1227         put_device(&pvt->addrmatch_dev);
1228         device_del(&pvt->addrmatch_dev);
1229 }
1230
1231 /****************************************************************************
1232         Device initialization routines: put/get, init/exit
1233  ****************************************************************************/
1234
1235 /*
1236  *      i7core_put_all_devices  'put' all the devices that we have
1237  *                              reserved via 'get'
1238  */
1239 static void i7core_put_devices(struct i7core_dev *i7core_dev)
1240 {
1241         int i;
1242
1243         debugf0(__FILE__ ": %s()\n", __func__);
1244         for (i = 0; i < i7core_dev->n_devs; i++) {
1245                 struct pci_dev *pdev = i7core_dev->pdev[i];
1246                 if (!pdev)
1247                         continue;
1248                 debugf0("Removing dev %02x:%02x.%d\n",
1249                         pdev->bus->number,
1250                         PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
1251                 pci_dev_put(pdev);
1252         }
1253 }
1254
1255 static void i7core_put_all_devices(void)
1256 {
1257         struct i7core_dev *i7core_dev, *tmp;
1258
1259         list_for_each_entry_safe(i7core_dev, tmp, &i7core_edac_list, list) {
1260                 i7core_put_devices(i7core_dev);
1261                 free_i7core_dev(i7core_dev);
1262         }
1263 }
1264
1265 static void __init i7core_xeon_pci_fixup(const struct pci_id_table *table)
1266 {
1267         struct pci_dev *pdev = NULL;
1268         int i;
1269
1270         /*
1271          * On Xeon 55xx, the Intel Quick Path Arch Generic Non-core pci buses
1272          * aren't announced by acpi. So, we need to use a legacy scan probing
1273          * to detect them
1274          */
1275         while (table && table->descr) {
1276                 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, table->descr[0].dev_id, NULL);
1277                 if (unlikely(!pdev)) {
1278                         for (i = 0; i < MAX_SOCKET_BUSES; i++)
1279                                 pcibios_scan_specific_bus(255-i);
1280                 }
1281                 pci_dev_put(pdev);
1282                 table++;
1283         }
1284 }
1285
1286 static unsigned i7core_pci_lastbus(void)
1287 {
1288         int last_bus = 0, bus;
1289         struct pci_bus *b = NULL;
1290
1291         while ((b = pci_find_next_bus(b)) != NULL) {
1292                 bus = b->number;
1293                 debugf0("Found bus %d\n", bus);
1294                 if (bus > last_bus)
1295                         last_bus = bus;
1296         }
1297
1298         debugf0("Last bus %d\n", last_bus);
1299
1300         return last_bus;
1301 }
1302
1303 /*
1304  *      i7core_get_all_devices  Find and perform 'get' operation on the MCH's
1305  *                      device/functions we want to reference for this driver
1306  *
1307  *                      Need to 'get' device 16 func 1 and func 2
1308  */
1309 static int i7core_get_onedevice(struct pci_dev **prev,
1310                                 const struct pci_id_table *table,
1311                                 const unsigned devno,
1312                                 const unsigned last_bus)
1313 {
1314         struct i7core_dev *i7core_dev;
1315         const struct pci_id_descr *dev_descr = &table->descr[devno];
1316
1317         struct pci_dev *pdev = NULL;
1318         u8 bus = 0;
1319         u8 socket = 0;
1320
1321         pdev = pci_get_device(PCI_VENDOR_ID_INTEL,
1322                               dev_descr->dev_id, *prev);
1323
1324         /*
1325          * On Xeon 55xx, the Intel Quckpath Arch Generic Non-core regs
1326          * is at addr 8086:2c40, instead of 8086:2c41. So, we need
1327          * to probe for the alternate address in case of failure
1328          */
1329         if (dev_descr->dev_id == PCI_DEVICE_ID_INTEL_I7_NONCORE && !pdev)
1330                 pdev = pci_get_device(PCI_VENDOR_ID_INTEL,
1331                                       PCI_DEVICE_ID_INTEL_I7_NONCORE_ALT, *prev);
1332
1333         if (dev_descr->dev_id == PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE && !pdev)
1334                 pdev = pci_get_device(PCI_VENDOR_ID_INTEL,
1335                                       PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE_ALT,
1336                                       *prev);
1337
1338         if (!pdev) {
1339                 if (*prev) {
1340                         *prev = pdev;
1341                         return 0;
1342                 }
1343
1344                 if (dev_descr->optional)
1345                         return 0;
1346
1347                 if (devno == 0)
1348                         return -ENODEV;
1349
1350                 i7core_printk(KERN_INFO,
1351                         "Device not found: dev %02x.%d PCI ID %04x:%04x\n",
1352                         dev_descr->dev, dev_descr->func,
1353                         PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
1354
1355                 /* End of list, leave */
1356                 return -ENODEV;
1357         }
1358         bus = pdev->bus->number;
1359
1360         socket = last_bus - bus;
1361
1362         i7core_dev = get_i7core_dev(socket);
1363         if (!i7core_dev) {
1364                 i7core_dev = alloc_i7core_dev(socket, table);
1365                 if (!i7core_dev) {
1366                         pci_dev_put(pdev);
1367                         return -ENOMEM;
1368                 }
1369         }
1370
1371         if (i7core_dev->pdev[devno]) {
1372                 i7core_printk(KERN_ERR,
1373                         "Duplicated device for "
1374                         "dev %02x:%02x.%d PCI ID %04x:%04x\n",
1375                         bus, dev_descr->dev, dev_descr->func,
1376                         PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
1377                 pci_dev_put(pdev);
1378                 return -ENODEV;
1379         }
1380
1381         i7core_dev->pdev[devno] = pdev;
1382
1383         /* Sanity check */
1384         if (unlikely(PCI_SLOT(pdev->devfn) != dev_descr->dev ||
1385                         PCI_FUNC(pdev->devfn) != dev_descr->func)) {
1386                 i7core_printk(KERN_ERR,
1387                         "Device PCI ID %04x:%04x "
1388                         "has dev %02x:%02x.%d instead of dev %02x:%02x.%d\n",
1389                         PCI_VENDOR_ID_INTEL, dev_descr->dev_id,
1390                         bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn),
1391                         bus, dev_descr->dev, dev_descr->func);
1392                 return -ENODEV;
1393         }
1394
1395         /* Be sure that the device is enabled */
1396         if (unlikely(pci_enable_device(pdev) < 0)) {
1397                 i7core_printk(KERN_ERR,
1398                         "Couldn't enable "
1399                         "dev %02x:%02x.%d PCI ID %04x:%04x\n",
1400                         bus, dev_descr->dev, dev_descr->func,
1401                         PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
1402                 return -ENODEV;
1403         }
1404
1405         debugf0("Detected socket %d dev %02x:%02x.%d PCI ID %04x:%04x\n",
1406                 socket, bus, dev_descr->dev,
1407                 dev_descr->func,
1408                 PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
1409
1410         /*
1411          * As stated on drivers/pci/search.c, the reference count for
1412          * @from is always decremented if it is not %NULL. So, as we need
1413          * to get all devices up to null, we need to do a get for the device
1414          */
1415         pci_dev_get(pdev);
1416
1417         *prev = pdev;
1418
1419         return 0;
1420 }
1421
1422 static int i7core_get_all_devices(void)
1423 {
1424         int i, rc, last_bus;
1425         struct pci_dev *pdev = NULL;
1426         const struct pci_id_table *table = pci_dev_table;
1427
1428         last_bus = i7core_pci_lastbus();
1429
1430         while (table && table->descr) {
1431                 for (i = 0; i < table->n_devs; i++) {
1432                         pdev = NULL;
1433                         do {
1434                                 rc = i7core_get_onedevice(&pdev, table, i,
1435                                                           last_bus);
1436                                 if (rc < 0) {
1437                                         if (i == 0) {
1438                                                 i = table->n_devs;
1439                                                 break;
1440                                         }
1441                                         i7core_put_all_devices();
1442                                         return -ENODEV;
1443                                 }
1444                         } while (pdev);
1445                 }
1446                 table++;
1447         }
1448
1449         return 0;
1450 }
1451
1452 static int mci_bind_devs(struct mem_ctl_info *mci,
1453                          struct i7core_dev *i7core_dev)
1454 {
1455         struct i7core_pvt *pvt = mci->pvt_info;
1456         struct pci_dev *pdev;
1457         int i, func, slot;
1458         char *family;
1459
1460         pvt->is_registered = false;
1461         pvt->enable_scrub  = false;
1462         for (i = 0; i < i7core_dev->n_devs; i++) {
1463                 pdev = i7core_dev->pdev[i];
1464                 if (!pdev)
1465                         continue;
1466
1467                 func = PCI_FUNC(pdev->devfn);
1468                 slot = PCI_SLOT(pdev->devfn);
1469                 if (slot == 3) {
1470                         if (unlikely(func > MAX_MCR_FUNC))
1471                                 goto error;
1472                         pvt->pci_mcr[func] = pdev;
1473                 } else if (likely(slot >= 4 && slot < 4 + NUM_CHANS)) {
1474                         if (unlikely(func > MAX_CHAN_FUNC))
1475                                 goto error;
1476                         pvt->pci_ch[slot - 4][func] = pdev;
1477                 } else if (!slot && !func) {
1478                         pvt->pci_noncore = pdev;
1479
1480                         /* Detect the processor family */
1481                         switch (pdev->device) {
1482                         case PCI_DEVICE_ID_INTEL_I7_NONCORE:
1483                                 family = "Xeon 35xx/ i7core";
1484                                 pvt->enable_scrub = false;
1485                                 break;
1486                         case PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE_ALT:
1487                                 family = "i7-800/i5-700";
1488                                 pvt->enable_scrub = false;
1489                                 break;
1490                         case PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE:
1491                                 family = "Xeon 34xx";
1492                                 pvt->enable_scrub = false;
1493                                 break;
1494                         case PCI_DEVICE_ID_INTEL_I7_NONCORE_ALT:
1495                                 family = "Xeon 55xx";
1496                                 pvt->enable_scrub = true;
1497                                 break;
1498                         case PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE_REV2:
1499                                 family = "Xeon 56xx / i7-900";
1500                                 pvt->enable_scrub = true;
1501                                 break;
1502                         default:
1503                                 family = "unknown";
1504                                 pvt->enable_scrub = false;
1505                         }
1506                         debugf0("Detected a processor type %s\n", family);
1507                 } else
1508                         goto error;
1509
1510                 debugf0("Associated fn %d.%d, dev = %p, socket %d\n",
1511                         PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn),
1512                         pdev, i7core_dev->socket);
1513
1514                 if (PCI_SLOT(pdev->devfn) == 3 &&
1515                         PCI_FUNC(pdev->devfn) == 2)
1516                         pvt->is_registered = true;
1517         }
1518
1519         return 0;
1520
1521 error:
1522         i7core_printk(KERN_ERR, "Device %d, function %d "
1523                       "is out of the expected range\n",
1524                       slot, func);
1525         return -EINVAL;
1526 }
1527
1528 /****************************************************************************
1529                         Error check routines
1530  ****************************************************************************/
1531 static void i7core_rdimm_update_errcount(struct mem_ctl_info *mci,
1532                                       const int chan,
1533                                       const int dimm,
1534                                       const int add)
1535 {
1536         int i;
1537
1538         for (i = 0; i < add; i++) {
1539                 edac_mc_handle_error(HW_EVENT_ERR_CORRECTED, mci, 0, 0, 0,
1540                                      chan, dimm, -1, "error", "", NULL);
1541         }
1542 }
1543
1544 static void i7core_rdimm_update_ce_count(struct mem_ctl_info *mci,
1545                                          const int chan,
1546                                          const int new0,
1547                                          const int new1,
1548                                          const int new2)
1549 {
1550         struct i7core_pvt *pvt = mci->pvt_info;
1551         int add0 = 0, add1 = 0, add2 = 0;
1552         /* Updates CE counters if it is not the first time here */
1553         if (pvt->ce_count_available) {
1554                 /* Updates CE counters */
1555
1556                 add2 = new2 - pvt->rdimm_last_ce_count[chan][2];
1557                 add1 = new1 - pvt->rdimm_last_ce_count[chan][1];
1558                 add0 = new0 - pvt->rdimm_last_ce_count[chan][0];
1559
1560                 if (add2 < 0)
1561                         add2 += 0x7fff;
1562                 pvt->rdimm_ce_count[chan][2] += add2;
1563
1564                 if (add1 < 0)
1565                         add1 += 0x7fff;
1566                 pvt->rdimm_ce_count[chan][1] += add1;
1567
1568                 if (add0 < 0)
1569                         add0 += 0x7fff;
1570                 pvt->rdimm_ce_count[chan][0] += add0;
1571         } else
1572                 pvt->ce_count_available = 1;
1573
1574         /* Store the new values */
1575         pvt->rdimm_last_ce_count[chan][2] = new2;
1576         pvt->rdimm_last_ce_count[chan][1] = new1;
1577         pvt->rdimm_last_ce_count[chan][0] = new0;
1578
1579         /*updated the edac core */
1580         if (add0 != 0)
1581                 i7core_rdimm_update_errcount(mci, chan, 0, add0);
1582         if (add1 != 0)
1583                 i7core_rdimm_update_errcount(mci, chan, 1, add1);
1584         if (add2 != 0)
1585                 i7core_rdimm_update_errcount(mci, chan, 2, add2);
1586
1587 }
1588
1589 static void i7core_rdimm_check_mc_ecc_err(struct mem_ctl_info *mci)
1590 {
1591         struct i7core_pvt *pvt = mci->pvt_info;
1592         u32 rcv[3][2];
1593         int i, new0, new1, new2;
1594
1595         /*Read DEV 3: FUN 2:  MC_COR_ECC_CNT regs directly*/
1596         pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_0,
1597                                                                 &rcv[0][0]);
1598         pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_1,
1599                                                                 &rcv[0][1]);
1600         pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_2,
1601                                                                 &rcv[1][0]);
1602         pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_3,
1603                                                                 &rcv[1][1]);
1604         pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_4,
1605                                                                 &rcv[2][0]);
1606         pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_5,
1607                                                                 &rcv[2][1]);
1608         for (i = 0 ; i < 3; i++) {
1609                 debugf3("MC_COR_ECC_CNT%d = 0x%x; MC_COR_ECC_CNT%d = 0x%x\n",
1610                         (i * 2), rcv[i][0], (i * 2) + 1, rcv[i][1]);
1611                 /*if the channel has 3 dimms*/
1612                 if (pvt->channel[i].dimms > 2) {
1613                         new0 = DIMM_BOT_COR_ERR(rcv[i][0]);
1614                         new1 = DIMM_TOP_COR_ERR(rcv[i][0]);
1615                         new2 = DIMM_BOT_COR_ERR(rcv[i][1]);
1616                 } else {
1617                         new0 = DIMM_TOP_COR_ERR(rcv[i][0]) +
1618                                         DIMM_BOT_COR_ERR(rcv[i][0]);
1619                         new1 = DIMM_TOP_COR_ERR(rcv[i][1]) +
1620                                         DIMM_BOT_COR_ERR(rcv[i][1]);
1621                         new2 = 0;
1622                 }
1623
1624                 i7core_rdimm_update_ce_count(mci, i, new0, new1, new2);
1625         }
1626 }
1627
1628 /* This function is based on the device 3 function 4 registers as described on:
1629  * Intel Xeon Processor 5500 Series Datasheet Volume 2
1630  *      http://www.intel.com/Assets/PDF/datasheet/321322.pdf
1631  * also available at:
1632  *      http://www.arrownac.com/manufacturers/intel/s/nehalem/5500-datasheet-v2.pdf
1633  */
1634 static void i7core_udimm_check_mc_ecc_err(struct mem_ctl_info *mci)
1635 {
1636         struct i7core_pvt *pvt = mci->pvt_info;
1637         u32 rcv1, rcv0;
1638         int new0, new1, new2;
1639
1640         if (!pvt->pci_mcr[4]) {
1641                 debugf0("%s MCR registers not found\n", __func__);
1642                 return;
1643         }
1644
1645         /* Corrected test errors */
1646         pci_read_config_dword(pvt->pci_mcr[4], MC_TEST_ERR_RCV1, &rcv1);
1647         pci_read_config_dword(pvt->pci_mcr[4], MC_TEST_ERR_RCV0, &rcv0);
1648
1649         /* Store the new values */
1650         new2 = DIMM2_COR_ERR(rcv1);
1651         new1 = DIMM1_COR_ERR(rcv0);
1652         new0 = DIMM0_COR_ERR(rcv0);
1653
1654         /* Updates CE counters if it is not the first time here */
1655         if (pvt->ce_count_available) {
1656                 /* Updates CE counters */
1657                 int add0, add1, add2;
1658
1659                 add2 = new2 - pvt->udimm_last_ce_count[2];
1660                 add1 = new1 - pvt->udimm_last_ce_count[1];
1661                 add0 = new0 - pvt->udimm_last_ce_count[0];
1662
1663                 if (add2 < 0)
1664                         add2 += 0x7fff;
1665                 pvt->udimm_ce_count[2] += add2;
1666
1667                 if (add1 < 0)
1668                         add1 += 0x7fff;
1669                 pvt->udimm_ce_count[1] += add1;
1670
1671                 if (add0 < 0)
1672                         add0 += 0x7fff;
1673                 pvt->udimm_ce_count[0] += add0;
1674
1675                 if (add0 | add1 | add2)
1676                         i7core_printk(KERN_ERR, "New Corrected error(s): "
1677                                       "dimm0: +%d, dimm1: +%d, dimm2 +%d\n",
1678                                       add0, add1, add2);
1679         } else
1680                 pvt->ce_count_available = 1;
1681
1682         /* Store the new values */
1683         pvt->udimm_last_ce_count[2] = new2;
1684         pvt->udimm_last_ce_count[1] = new1;
1685         pvt->udimm_last_ce_count[0] = new0;
1686 }
1687
1688 /*
1689  * According with tables E-11 and E-12 of chapter E.3.3 of Intel 64 and IA-32
1690  * Architectures Software Developer’s Manual Volume 3B.
1691  * Nehalem are defined as family 0x06, model 0x1a
1692  *
1693  * The MCA registers used here are the following ones:
1694  *     struct mce field MCA Register
1695  *     m->status        MSR_IA32_MC8_STATUS
1696  *     m->addr          MSR_IA32_MC8_ADDR
1697  *     m->misc          MSR_IA32_MC8_MISC
1698  * In the case of Nehalem, the error information is masked at .status and .misc
1699  * fields
1700  */
1701 static void i7core_mce_output_error(struct mem_ctl_info *mci,
1702                                     const struct mce *m)
1703 {
1704         struct i7core_pvt *pvt = mci->pvt_info;
1705         char *type, *optype, *err, msg[80];
1706         enum hw_event_mc_err_type tp_event;
1707         unsigned long error = m->status & 0x1ff0000l;
1708         bool uncorrected_error = m->mcgstatus & 1ll << 61;
1709         bool ripv = m->mcgstatus & 1;
1710         u32 optypenum = (m->status >> 4) & 0x07;
1711         u32 core_err_cnt = (m->status >> 38) & 0x7fff;
1712         u32 dimm = (m->misc >> 16) & 0x3;
1713         u32 channel = (m->misc >> 18) & 0x3;
1714         u32 syndrome = m->misc >> 32;
1715         u32 errnum = find_first_bit(&error, 32);
1716
1717         if (uncorrected_error) {
1718                 if (ripv) {
1719                         type = "FATAL";
1720                         tp_event = HW_EVENT_ERR_FATAL;
1721                 } else {
1722                         type = "NON_FATAL";
1723                         tp_event = HW_EVENT_ERR_UNCORRECTED;
1724                 }
1725         } else {
1726                 type = "CORRECTED";
1727                 tp_event = HW_EVENT_ERR_CORRECTED;
1728         }
1729
1730         switch (optypenum) {
1731         case 0:
1732                 optype = "generic undef request";
1733                 break;
1734         case 1:
1735                 optype = "read error";
1736                 break;
1737         case 2:
1738                 optype = "write error";
1739                 break;
1740         case 3:
1741                 optype = "addr/cmd error";
1742                 break;
1743         case 4:
1744                 optype = "scrubbing error";
1745                 break;
1746         default:
1747                 optype = "reserved";
1748                 break;
1749         }
1750
1751         switch (errnum) {
1752         case 16:
1753                 err = "read ECC error";
1754                 break;
1755         case 17:
1756                 err = "RAS ECC error";
1757                 break;
1758         case 18:
1759                 err = "write parity error";
1760                 break;
1761         case 19:
1762                 err = "redundacy loss";
1763                 break;
1764         case 20:
1765                 err = "reserved";
1766                 break;
1767         case 21:
1768                 err = "memory range error";
1769                 break;
1770         case 22:
1771                 err = "RTID out of range";
1772                 break;
1773         case 23:
1774                 err = "address parity error";
1775                 break;
1776         case 24:
1777                 err = "byte enable parity error";
1778                 break;
1779         default:
1780                 err = "unknown";
1781         }
1782
1783         snprintf(msg, sizeof(msg), "count=%d %s", core_err_cnt, optype);
1784
1785         /*
1786          * Call the helper to output message
1787          * FIXME: what to do if core_err_cnt > 1? Currently, it generates
1788          * only one event
1789          */
1790         if (uncorrected_error || !pvt->is_registered)
1791                 edac_mc_handle_error(tp_event, mci,
1792                                      m->addr >> PAGE_SHIFT,
1793                                      m->addr & ~PAGE_MASK,
1794                                      syndrome,
1795                                      channel, dimm, -1,
1796                                      err, msg, m);
1797 }
1798
1799 /*
1800  *      i7core_check_error      Retrieve and process errors reported by the
1801  *                              hardware. Called by the Core module.
1802  */
1803 static void i7core_check_error(struct mem_ctl_info *mci)
1804 {
1805         struct i7core_pvt *pvt = mci->pvt_info;
1806         int i;
1807         unsigned count = 0;
1808         struct mce *m;
1809
1810         /*
1811          * MCE first step: Copy all mce errors into a temporary buffer
1812          * We use a double buffering here, to reduce the risk of
1813          * losing an error.
1814          */
1815         smp_rmb();
1816         count = (pvt->mce_out + MCE_LOG_LEN - pvt->mce_in)
1817                 % MCE_LOG_LEN;
1818         if (!count)
1819                 goto check_ce_error;
1820
1821         m = pvt->mce_outentry;
1822         if (pvt->mce_in + count > MCE_LOG_LEN) {
1823                 unsigned l = MCE_LOG_LEN - pvt->mce_in;
1824
1825                 memcpy(m, &pvt->mce_entry[pvt->mce_in], sizeof(*m) * l);
1826                 smp_wmb();
1827                 pvt->mce_in = 0;
1828                 count -= l;
1829                 m += l;
1830         }
1831         memcpy(m, &pvt->mce_entry[pvt->mce_in], sizeof(*m) * count);
1832         smp_wmb();
1833         pvt->mce_in += count;
1834
1835         smp_rmb();
1836         if (pvt->mce_overrun) {
1837                 i7core_printk(KERN_ERR, "Lost %d memory errors\n",
1838                               pvt->mce_overrun);
1839                 smp_wmb();
1840                 pvt->mce_overrun = 0;
1841         }
1842
1843         /*
1844          * MCE second step: parse errors and display
1845          */
1846         for (i = 0; i < count; i++)
1847                 i7core_mce_output_error(mci, &pvt->mce_outentry[i]);
1848
1849         /*
1850          * Now, let's increment CE error counts
1851          */
1852 check_ce_error:
1853         if (!pvt->is_registered)
1854                 i7core_udimm_check_mc_ecc_err(mci);
1855         else
1856                 i7core_rdimm_check_mc_ecc_err(mci);
1857 }
1858
1859 /*
1860  * i7core_mce_check_error       Replicates mcelog routine to get errors
1861  *                              This routine simply queues mcelog errors, and
1862  *                              return. The error itself should be handled later
1863  *                              by i7core_check_error.
1864  * WARNING: As this routine should be called at NMI time, extra care should
1865  * be taken to avoid deadlocks, and to be as fast as possible.
1866  */
1867 static int i7core_mce_check_error(struct notifier_block *nb, unsigned long val,
1868                                   void *data)
1869 {
1870         struct mce *mce = (struct mce *)data;
1871         struct i7core_dev *i7_dev;
1872         struct mem_ctl_info *mci;
1873         struct i7core_pvt *pvt;
1874
1875         i7_dev = get_i7core_dev(mce->socketid);
1876         if (!i7_dev)
1877                 return NOTIFY_BAD;
1878
1879         mci = i7_dev->mci;
1880         pvt = mci->pvt_info;
1881
1882         /*
1883          * Just let mcelog handle it if the error is
1884          * outside the memory controller
1885          */
1886         if (((mce->status & 0xffff) >> 7) != 1)
1887                 return NOTIFY_DONE;
1888
1889         /* Bank 8 registers are the only ones that we know how to handle */
1890         if (mce->bank != 8)
1891                 return NOTIFY_DONE;
1892
1893 #ifdef CONFIG_SMP
1894         /* Only handle if it is the right mc controller */
1895         if (mce->socketid != pvt->i7core_dev->socket)
1896                 return NOTIFY_DONE;
1897 #endif
1898
1899         smp_rmb();
1900         if ((pvt->mce_out + 1) % MCE_LOG_LEN == pvt->mce_in) {
1901                 smp_wmb();
1902                 pvt->mce_overrun++;
1903                 return NOTIFY_DONE;
1904         }
1905
1906         /* Copy memory error at the ringbuffer */
1907         memcpy(&pvt->mce_entry[pvt->mce_out], mce, sizeof(*mce));
1908         smp_wmb();
1909         pvt->mce_out = (pvt->mce_out + 1) % MCE_LOG_LEN;
1910
1911         /* Handle fatal errors immediately */
1912         if (mce->mcgstatus & 1)
1913                 i7core_check_error(mci);
1914
1915         /* Advise mcelog that the errors were handled */
1916         return NOTIFY_STOP;
1917 }
1918
1919 static struct notifier_block i7_mce_dec = {
1920         .notifier_call  = i7core_mce_check_error,
1921 };
1922
1923 struct memdev_dmi_entry {
1924         u8 type;
1925         u8 length;
1926         u16 handle;
1927         u16 phys_mem_array_handle;
1928         u16 mem_err_info_handle;
1929         u16 total_width;
1930         u16 data_width;
1931         u16 size;
1932         u8 form;
1933         u8 device_set;
1934         u8 device_locator;
1935         u8 bank_locator;
1936         u8 memory_type;
1937         u16 type_detail;
1938         u16 speed;
1939         u8 manufacturer;
1940         u8 serial_number;
1941         u8 asset_tag;
1942         u8 part_number;
1943         u8 attributes;
1944         u32 extended_size;
1945         u16 conf_mem_clk_speed;
1946 } __attribute__((__packed__));
1947
1948
1949 /*
1950  * Decode the DRAM Clock Frequency, be paranoid, make sure that all
1951  * memory devices show the same speed, and if they don't then consider
1952  * all speeds to be invalid.
1953  */
1954 static void decode_dclk(const struct dmi_header *dh, void *_dclk_freq)
1955 {
1956         int *dclk_freq = _dclk_freq;
1957         u16 dmi_mem_clk_speed;
1958
1959         if (*dclk_freq == -1)
1960                 return;
1961
1962         if (dh->type == DMI_ENTRY_MEM_DEVICE) {
1963                 struct memdev_dmi_entry *memdev_dmi_entry =
1964                         (struct memdev_dmi_entry *)dh;
1965                 unsigned long conf_mem_clk_speed_offset =
1966                         (unsigned long)&memdev_dmi_entry->conf_mem_clk_speed -
1967                         (unsigned long)&memdev_dmi_entry->type;
1968                 unsigned long speed_offset =
1969                         (unsigned long)&memdev_dmi_entry->speed -
1970                         (unsigned long)&memdev_dmi_entry->type;
1971
1972                 /* Check that a DIMM is present */
1973                 if (memdev_dmi_entry->size == 0)
1974                         return;
1975
1976                 /*
1977                  * Pick the configured speed if it's available, otherwise
1978                  * pick the DIMM speed, or we don't have a speed.
1979                  */
1980                 if (memdev_dmi_entry->length > conf_mem_clk_speed_offset) {
1981                         dmi_mem_clk_speed =
1982                                 memdev_dmi_entry->conf_mem_clk_speed;
1983                 } else if (memdev_dmi_entry->length > speed_offset) {
1984                         dmi_mem_clk_speed = memdev_dmi_entry->speed;
1985                 } else {
1986                         *dclk_freq = -1;
1987                         return;
1988                 }
1989
1990                 if (*dclk_freq == 0) {
1991                         /* First pass, speed was 0 */
1992                         if (dmi_mem_clk_speed > 0) {
1993                                 /* Set speed if a valid speed is read */
1994                                 *dclk_freq = dmi_mem_clk_speed;
1995                         } else {
1996                                 /* Otherwise we don't have a valid speed */
1997                                 *dclk_freq = -1;
1998                         }
1999                 } else if (*dclk_freq > 0 &&
2000                            *dclk_freq != dmi_mem_clk_speed) {
2001                         /*
2002                          * If we have a speed, check that all DIMMS are the same
2003                          * speed, otherwise set the speed as invalid.
2004                          */
2005                         *dclk_freq = -1;
2006                 }
2007         }
2008 }
2009
2010 /*
2011  * The default DCLK frequency is used as a fallback if we
2012  * fail to find anything reliable in the DMI. The value
2013  * is taken straight from the datasheet.
2014  */
2015 #define DEFAULT_DCLK_FREQ 800
2016
2017 static int get_dclk_freq(void)
2018 {
2019         int dclk_freq = 0;
2020
2021         dmi_walk(decode_dclk, (void *)&dclk_freq);
2022
2023         if (dclk_freq < 1)
2024                 return DEFAULT_DCLK_FREQ;
2025
2026         return dclk_freq;
2027 }
2028
2029 /*
2030  * set_sdram_scrub_rate         This routine sets byte/sec bandwidth scrub rate
2031  *                              to hardware according to SCRUBINTERVAL formula
2032  *                              found in datasheet.
2033  */
2034 static int set_sdram_scrub_rate(struct mem_ctl_info *mci, u32 new_bw)
2035 {
2036         struct i7core_pvt *pvt = mci->pvt_info;
2037         struct pci_dev *pdev;
2038         u32 dw_scrub;
2039         u32 dw_ssr;
2040
2041         /* Get data from the MC register, function 2 */
2042         pdev = pvt->pci_mcr[2];
2043         if (!pdev)
2044                 return -ENODEV;
2045
2046         pci_read_config_dword(pdev, MC_SCRUB_CONTROL, &dw_scrub);
2047
2048         if (new_bw == 0) {
2049                 /* Prepare to disable petrol scrub */
2050                 dw_scrub &= ~STARTSCRUB;
2051                 /* Stop the patrol scrub engine */
2052                 write_and_test(pdev, MC_SCRUB_CONTROL,
2053                                dw_scrub & ~SCRUBINTERVAL_MASK);
2054
2055                 /* Get current status of scrub rate and set bit to disable */
2056                 pci_read_config_dword(pdev, MC_SSRCONTROL, &dw_ssr);
2057                 dw_ssr &= ~SSR_MODE_MASK;
2058                 dw_ssr |= SSR_MODE_DISABLE;
2059         } else {
2060                 const int cache_line_size = 64;
2061                 const u32 freq_dclk_mhz = pvt->dclk_freq;
2062                 unsigned long long scrub_interval;
2063                 /*
2064                  * Translate the desired scrub rate to a register value and
2065                  * program the corresponding register value.
2066                  */
2067                 scrub_interval = (unsigned long long)freq_dclk_mhz *
2068                         cache_line_size * 1000000;
2069                 do_div(scrub_interval, new_bw);
2070
2071                 if (!scrub_interval || scrub_interval > SCRUBINTERVAL_MASK)
2072                         return -EINVAL;
2073
2074                 dw_scrub = SCRUBINTERVAL_MASK & scrub_interval;
2075
2076                 /* Start the patrol scrub engine */
2077                 pci_write_config_dword(pdev, MC_SCRUB_CONTROL,
2078                                        STARTSCRUB | dw_scrub);
2079
2080                 /* Get current status of scrub rate and set bit to enable */
2081                 pci_read_config_dword(pdev, MC_SSRCONTROL, &dw_ssr);
2082                 dw_ssr &= ~SSR_MODE_MASK;
2083                 dw_ssr |= SSR_MODE_ENABLE;
2084         }
2085         /* Disable or enable scrubbing */
2086         pci_write_config_dword(pdev, MC_SSRCONTROL, dw_ssr);
2087
2088         return new_bw;
2089 }
2090
2091 /*
2092  * get_sdram_scrub_rate         This routine convert current scrub rate value
2093  *                              into byte/sec bandwidth accourding to
2094  *                              SCRUBINTERVAL formula found in datasheet.
2095  */
2096 static int get_sdram_scrub_rate(struct mem_ctl_info *mci)
2097 {
2098         struct i7core_pvt *pvt = mci->pvt_info;
2099         struct pci_dev *pdev;
2100         const u32 cache_line_size = 64;
2101         const u32 freq_dclk_mhz = pvt->dclk_freq;
2102         unsigned long long scrub_rate;
2103         u32 scrubval;
2104
2105         /* Get data from the MC register, function 2 */
2106         pdev = pvt->pci_mcr[2];
2107         if (!pdev)
2108                 return -ENODEV;
2109
2110         /* Get current scrub control data */
2111         pci_read_config_dword(pdev, MC_SCRUB_CONTROL, &scrubval);
2112
2113         /* Mask highest 8-bits to 0 */
2114         scrubval &=  SCRUBINTERVAL_MASK;
2115         if (!scrubval)
2116                 return 0;
2117
2118         /* Calculate scrub rate value into byte/sec bandwidth */
2119         scrub_rate =  (unsigned long long)freq_dclk_mhz *
2120                 1000000 * cache_line_size;
2121         do_div(scrub_rate, scrubval);
2122         return (int)scrub_rate;
2123 }
2124
2125 static void enable_sdram_scrub_setting(struct mem_ctl_info *mci)
2126 {
2127         struct i7core_pvt *pvt = mci->pvt_info;
2128         u32 pci_lock;
2129
2130         /* Unlock writes to pci registers */
2131         pci_read_config_dword(pvt->pci_noncore, MC_CFG_CONTROL, &pci_lock);
2132         pci_lock &= ~0x3;
2133         pci_write_config_dword(pvt->pci_noncore, MC_CFG_CONTROL,
2134                                pci_lock | MC_CFG_UNLOCK);
2135
2136         mci->set_sdram_scrub_rate = set_sdram_scrub_rate;
2137         mci->get_sdram_scrub_rate = get_sdram_scrub_rate;
2138 }
2139
2140 static void disable_sdram_scrub_setting(struct mem_ctl_info *mci)
2141 {
2142         struct i7core_pvt *pvt = mci->pvt_info;
2143         u32 pci_lock;
2144
2145         /* Lock writes to pci registers */
2146         pci_read_config_dword(pvt->pci_noncore, MC_CFG_CONTROL, &pci_lock);
2147         pci_lock &= ~0x3;
2148         pci_write_config_dword(pvt->pci_noncore, MC_CFG_CONTROL,
2149                                pci_lock | MC_CFG_LOCK);
2150 }
2151
2152 static void i7core_pci_ctl_create(struct i7core_pvt *pvt)
2153 {
2154         pvt->i7core_pci = edac_pci_create_generic_ctl(
2155                                                 &pvt->i7core_dev->pdev[0]->dev,
2156                                                 EDAC_MOD_STR);
2157         if (unlikely(!pvt->i7core_pci))
2158                 i7core_printk(KERN_WARNING,
2159                               "Unable to setup PCI error report via EDAC\n");
2160 }
2161
2162 static void i7core_pci_ctl_release(struct i7core_pvt *pvt)
2163 {
2164         if (likely(pvt->i7core_pci))
2165                 edac_pci_release_generic_ctl(pvt->i7core_pci);
2166         else
2167                 i7core_printk(KERN_ERR,
2168                                 "Couldn't find mem_ctl_info for socket %d\n",
2169                                 pvt->i7core_dev->socket);
2170         pvt->i7core_pci = NULL;
2171 }
2172
2173 static void i7core_unregister_mci(struct i7core_dev *i7core_dev)
2174 {
2175         struct mem_ctl_info *mci = i7core_dev->mci;
2176         struct i7core_pvt *pvt;
2177
2178         if (unlikely(!mci || !mci->pvt_info)) {
2179                 debugf0("MC: " __FILE__ ": %s(): dev = %p\n",
2180                         __func__, &i7core_dev->pdev[0]->dev);
2181
2182                 i7core_printk(KERN_ERR, "Couldn't find mci handler\n");
2183                 return;
2184         }
2185
2186         pvt = mci->pvt_info;
2187
2188         debugf0("MC: " __FILE__ ": %s(): mci = %p, dev = %p\n",
2189                 __func__, mci, &i7core_dev->pdev[0]->dev);
2190
2191         /* Disable scrubrate setting */
2192         if (pvt->enable_scrub)
2193                 disable_sdram_scrub_setting(mci);
2194
2195         mce_unregister_decode_chain(&i7_mce_dec);
2196
2197         /* Disable EDAC polling */
2198         i7core_pci_ctl_release(pvt);
2199
2200         /* Remove MC sysfs nodes */
2201         i7core_delete_sysfs_devices(mci);
2202         edac_mc_del_mc(mci->pdev);
2203
2204         debugf1("%s: free mci struct\n", mci->ctl_name);
2205         kfree(mci->ctl_name);
2206         edac_mc_free(mci);
2207         i7core_dev->mci = NULL;
2208 }
2209
2210 static int i7core_register_mci(struct i7core_dev *i7core_dev)
2211 {
2212         struct mem_ctl_info *mci;
2213         struct i7core_pvt *pvt;
2214         int rc;
2215         struct edac_mc_layer layers[2];
2216
2217         /* allocate a new MC control structure */
2218
2219         layers[0].type = EDAC_MC_LAYER_CHANNEL;
2220         layers[0].size = NUM_CHANS;
2221         layers[0].is_virt_csrow = false;
2222         layers[1].type = EDAC_MC_LAYER_SLOT;
2223         layers[1].size = MAX_DIMMS;
2224         layers[1].is_virt_csrow = true;
2225         mci = edac_mc_alloc(i7core_dev->socket, ARRAY_SIZE(layers), layers,
2226                             sizeof(*pvt));
2227         if (unlikely(!mci))
2228                 return -ENOMEM;
2229
2230         debugf0("MC: " __FILE__ ": %s(): mci = %p, dev = %p\n",
2231                 __func__, mci, &i7core_dev->pdev[0]->dev);
2232
2233         pvt = mci->pvt_info;
2234         memset(pvt, 0, sizeof(*pvt));
2235
2236         /* Associates i7core_dev and mci for future usage */
2237         pvt->i7core_dev = i7core_dev;
2238         i7core_dev->mci = mci;
2239
2240         /*
2241          * FIXME: how to handle RDDR3 at MCI level? It is possible to have
2242          * Mixed RDDR3/UDDR3 with Nehalem, provided that they are on different
2243          * memory channels
2244          */
2245         mci->mtype_cap = MEM_FLAG_DDR3;
2246         mci->edac_ctl_cap = EDAC_FLAG_NONE;
2247         mci->edac_cap = EDAC_FLAG_NONE;
2248         mci->mod_name = "i7core_edac.c";
2249         mci->mod_ver = I7CORE_REVISION;
2250         mci->ctl_name = kasprintf(GFP_KERNEL, "i7 core #%d",
2251                                   i7core_dev->socket);
2252         mci->dev_name = pci_name(i7core_dev->pdev[0]);
2253         mci->ctl_page_to_phys = NULL;
2254
2255         /* Store pci devices at mci for faster access */
2256         rc = mci_bind_devs(mci, i7core_dev);
2257         if (unlikely(rc < 0))
2258                 goto fail0;
2259
2260
2261         /* Get dimm basic config */
2262         get_dimm_config(mci);
2263         /* record ptr to the generic device */
2264         mci->pdev = &i7core_dev->pdev[0]->dev;
2265         /* Set the function pointer to an actual operation function */
2266         mci->edac_check = i7core_check_error;
2267
2268         /* Enable scrubrate setting */
2269         if (pvt->enable_scrub)
2270                 enable_sdram_scrub_setting(mci);
2271
2272         /* add this new MC control structure to EDAC's list of MCs */
2273         if (unlikely(edac_mc_add_mc(mci))) {
2274                 debugf0("MC: " __FILE__
2275                         ": %s(): failed edac_mc_add_mc()\n", __func__);
2276                 /* FIXME: perhaps some code should go here that disables error
2277                  * reporting if we just enabled it
2278                  */
2279
2280                 rc = -EINVAL;
2281                 goto fail0;
2282         }
2283         if (i7core_create_sysfs_devices(mci)) {
2284                 debugf0("MC: " __FILE__
2285                         ": %s(): failed to create sysfs nodes\n", __func__);
2286                 edac_mc_del_mc(mci->pdev);
2287                 rc = -EINVAL;
2288                 goto fail0;
2289         }
2290
2291         /* Default error mask is any memory */
2292         pvt->inject.channel = 0;
2293         pvt->inject.dimm = -1;
2294         pvt->inject.rank = -1;
2295         pvt->inject.bank = -1;
2296         pvt->inject.page = -1;
2297         pvt->inject.col = -1;
2298
2299         /* allocating generic PCI control info */
2300         i7core_pci_ctl_create(pvt);
2301
2302         /* DCLK for scrub rate setting */
2303         pvt->dclk_freq = get_dclk_freq();
2304
2305         mce_register_decode_chain(&i7_mce_dec);
2306
2307         return 0;
2308
2309 fail0:
2310         kfree(mci->ctl_name);
2311         edac_mc_free(mci);
2312         i7core_dev->mci = NULL;
2313         return rc;
2314 }
2315
2316 /*
2317  *      i7core_probe    Probe for ONE instance of device to see if it is
2318  *                      present.
2319  *      return:
2320  *              0 for FOUND a device
2321  *              < 0 for error code
2322  */
2323
2324 static int __devinit i7core_probe(struct pci_dev *pdev,
2325                                   const struct pci_device_id *id)
2326 {
2327         int rc, count = 0;
2328         struct i7core_dev *i7core_dev;
2329
2330         /* get the pci devices we want to reserve for our use */
2331         mutex_lock(&i7core_edac_lock);
2332
2333         /*
2334          * All memory controllers are allocated at the first pass.
2335          */
2336         if (unlikely(probed >= 1)) {
2337                 mutex_unlock(&i7core_edac_lock);
2338                 return -ENODEV;
2339         }
2340         probed++;
2341
2342         rc = i7core_get_all_devices();
2343         if (unlikely(rc < 0))
2344                 goto fail0;
2345
2346         list_for_each_entry(i7core_dev, &i7core_edac_list, list) {
2347                 count++;
2348                 rc = i7core_register_mci(i7core_dev);
2349                 if (unlikely(rc < 0))
2350                         goto fail1;
2351         }
2352
2353         /*
2354          * Nehalem-EX uses a different memory controller. However, as the
2355          * memory controller is not visible on some Nehalem/Nehalem-EP, we
2356          * need to indirectly probe via a X58 PCI device. The same devices
2357          * are found on (some) Nehalem-EX. So, on those machines, the
2358          * probe routine needs to return -ENODEV, as the actual Memory
2359          * Controller registers won't be detected.
2360          */
2361         if (!count) {
2362                 rc = -ENODEV;
2363                 goto fail1;
2364         }
2365
2366         i7core_printk(KERN_INFO,
2367                       "Driver loaded, %d memory controller(s) found.\n",
2368                       count);
2369
2370         mutex_unlock(&i7core_edac_lock);
2371         return 0;
2372
2373 fail1:
2374         list_for_each_entry(i7core_dev, &i7core_edac_list, list)
2375                 i7core_unregister_mci(i7core_dev);
2376
2377         i7core_put_all_devices();
2378 fail0:
2379         mutex_unlock(&i7core_edac_lock);
2380         return rc;
2381 }
2382
2383 /*
2384  *      i7core_remove   destructor for one instance of device
2385  *
2386  */
2387 static void __devexit i7core_remove(struct pci_dev *pdev)
2388 {
2389         struct i7core_dev *i7core_dev;
2390
2391         debugf0(__FILE__ ": %s()\n", __func__);
2392
2393         /*
2394          * we have a trouble here: pdev value for removal will be wrong, since
2395          * it will point to the X58 register used to detect that the machine
2396          * is a Nehalem or upper design. However, due to the way several PCI
2397          * devices are grouped together to provide MC functionality, we need
2398          * to use a different method for releasing the devices
2399          */
2400
2401         mutex_lock(&i7core_edac_lock);
2402
2403         if (unlikely(!probed)) {
2404                 mutex_unlock(&i7core_edac_lock);
2405                 return;
2406         }
2407
2408         list_for_each_entry(i7core_dev, &i7core_edac_list, list)
2409                 i7core_unregister_mci(i7core_dev);
2410
2411         /* Release PCI resources */
2412         i7core_put_all_devices();
2413
2414         probed--;
2415
2416         mutex_unlock(&i7core_edac_lock);
2417 }
2418
2419 MODULE_DEVICE_TABLE(pci, i7core_pci_tbl);
2420
2421 /*
2422  *      i7core_driver   pci_driver structure for this module
2423  *
2424  */
2425 static struct pci_driver i7core_driver = {
2426         .name     = "i7core_edac",
2427         .probe    = i7core_probe,
2428         .remove   = __devexit_p(i7core_remove),
2429         .id_table = i7core_pci_tbl,
2430 };
2431
2432 /*
2433  *      i7core_init             Module entry function
2434  *                      Try to initialize this module for its devices
2435  */
2436 static int __init i7core_init(void)
2437 {
2438         int pci_rc;
2439
2440         debugf2("MC: " __FILE__ ": %s()\n", __func__);
2441
2442         /* Ensure that the OPSTATE is set correctly for POLL or NMI */
2443         opstate_init();
2444
2445         if (use_pci_fixup)
2446                 i7core_xeon_pci_fixup(pci_dev_table);
2447
2448         pci_rc = pci_register_driver(&i7core_driver);
2449
2450         if (pci_rc >= 0)
2451                 return 0;
2452
2453         i7core_printk(KERN_ERR, "Failed to register device with error %d.\n",
2454                       pci_rc);
2455
2456         return pci_rc;
2457 }
2458
2459 /*
2460  *      i7core_exit()   Module exit function
2461  *                      Unregister the driver
2462  */
2463 static void __exit i7core_exit(void)
2464 {
2465         debugf2("MC: " __FILE__ ": %s()\n", __func__);
2466         pci_unregister_driver(&i7core_driver);
2467 }
2468
2469 module_init(i7core_init);
2470 module_exit(i7core_exit);
2471
2472 MODULE_LICENSE("GPL");
2473 MODULE_AUTHOR("Mauro Carvalho Chehab <mchehab@redhat.com>");
2474 MODULE_AUTHOR("Red Hat Inc. (http://www.redhat.com)");
2475 MODULE_DESCRIPTION("MC Driver for Intel i7 Core memory controllers - "
2476                    I7CORE_REVISION);
2477
2478 module_param(edac_op_state, int, 0444);
2479 MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI");