drivers/edac/i7core_edac.c

   1 /* Intel i7 core/Nehalem Memory Controller kernel module
   2  *
   3  * This driver supports yhe memory controllers found on the Intel
   4  * processor families i7core, i7core 7xx/8xx, i5core, Xeon 35xx,
   5  * Xeon 55xx and Xeon 56xx also known as Nehalem, Nehalem-EP, Lynnfield
   6  * and Westmere-EP.
   7  *
   8  * This file may be distributed under the terms of the
   9  * GNU General Public License version 2 only.
  10  *
  11  * Copyright (c) 2009-2010 by:
  12  *       Mauro Carvalho Chehab <mchehab@redhat.com>
  13  *
  14  * Red Hat Inc. http://www.redhat.com
  15  *
  16  * Forked and adapted from the i5400_edac driver
  17  *
  18  * Based on the following public Intel datasheets:
  19  * Intel Core i7 Processor Extreme Edition and Intel Core i7 Processor
  20  * Datasheet, Volume 2:
  21  *      http://download.intel.com/design/processor/datashts/320835.pdf
  22  * Intel Xeon Processor 5500 Series Datasheet Volume 2
  23  *      http://www.intel.com/Assets/PDF/datasheet/321322.pdf
  24  * also available at:
  25  *      http://www.arrownac.com/manufacturers/intel/s/nehalem/5500-datasheet-v2.pdf
  26  */
  27
  28 #include <linux/module.h>
  29 #include <linux/init.h>
  30 #include <linux/pci.h>
  31 #include <linux/pci_ids.h>
  32 #include <linux/slab.h>
  33 #include <linux/delay.h>
  34 #include <linux/edac.h>
  35 #include <linux/mmzone.h>
  36 #include <linux/edac_mce.h>
  37 #include <linux/smp.h>
  38 #include <asm/processor.h>
  39
  40 #include "edac_core.h"
  41
  42 /* Static vars */
  43 static LIST_HEAD(i7core_edac_list);
  44 static DEFINE_MUTEX(i7core_edac_lock);
  45 static int probed;
  46
  47 static int use_pci_fixup;
  48 module_param(use_pci_fixup, int, 0444);
  49 MODULE_PARM_DESC(use_pci_fixup, "Enable PCI fixup to seek for hidden devices");
  50 /*
  51  * This is used for Nehalem-EP and Nehalem-EX devices, where the non-core
  52  * registers start at bus 255, and are not reported by BIOS.
  53  * We currently find devices with only 2 sockets. In order to support more QPI
  54  * Quick Path Interconnect, just increment this number.
  55  */
  56 #define MAX_SOCKET_BUSES        2
  57
  58
  59 /*
  60  * Alter this version for the module when modifications are made
  61  */
  62 #define I7CORE_REVISION    " Ver: 1.0.0 " __DATE__
  63 #define EDAC_MOD_STR      "i7core_edac"
  64
  65 /*
  66  * Debug macros
  67  */
  68 #define i7core_printk(level, fmt, arg...)                       \
  69         edac_printk(level, "i7core", fmt, ##arg)
  70
  71 #define i7core_mc_printk(mci, level, fmt, arg...)               \
  72         edac_mc_chipset_printk(mci, level, "i7core", fmt, ##arg)
  73
  74 /*
  75  * i7core Memory Controller Registers
  76  */
  77
  78         /* OFFSETS for Device 0 Function 0 */
  79
  80 #define MC_CFG_CONTROL  0x90
  81
  82         /* OFFSETS for Device 3 Function 0 */
  83
  84 #define MC_CONTROL      0x48
  85 #define MC_STATUS       0x4c
  86 #define MC_MAX_DOD      0x64
  87
  88 /*
  89  * OFFSETS for Device 3 Function 4, as inicated on Xeon 5500 datasheet:
  90  * http://www.arrownac.com/manufacturers/intel/s/nehalem/5500-datasheet-v2.pdf
  91  */
  92
  93 #define MC_TEST_ERR_RCV1        0x60
  94   #define DIMM2_COR_ERR(r)                      ((r) & 0x7fff)
  95
  96 #define MC_TEST_ERR_RCV0        0x64
  97   #define DIMM1_COR_ERR(r)                      (((r) >> 16) & 0x7fff)
  98   #define DIMM0_COR_ERR(r)                      ((r) & 0x7fff)
  99
 100 /* OFFSETS for Device 3 Function 2, as inicated on Xeon 5500 datasheet */
 101 #define MC_COR_ECC_CNT_0        0x80
 102 #define MC_COR_ECC_CNT_1        0x84
 103 #define MC_COR_ECC_CNT_2        0x88
 104 #define MC_COR_ECC_CNT_3        0x8c
 105 #define MC_COR_ECC_CNT_4        0x90
 106 #define MC_COR_ECC_CNT_5        0x94
 107
 108 #define DIMM_TOP_COR_ERR(r)                     (((r) >> 16) & 0x7fff)
 109 #define DIMM_BOT_COR_ERR(r)                     ((r) & 0x7fff)
 110
 111
 112         /* OFFSETS for Devices 4,5 and 6 Function 0 */
 113
 114 #define MC_CHANNEL_DIMM_INIT_PARAMS 0x58
 115   #define THREE_DIMMS_PRESENT           (1 << 24)
 116   #define SINGLE_QUAD_RANK_PRESENT      (1 << 23)
 117   #define QUAD_RANK_PRESENT             (1 << 22)
 118   #define REGISTERED_DIMM               (1 << 15)
 119
 120 #define MC_CHANNEL_MAPPER       0x60
 121   #define RDLCH(r, ch)          ((((r) >> (3 + (ch * 6))) & 0x07) - 1)
 122   #define WRLCH(r, ch)          ((((r) >> (ch * 6)) & 0x07) - 1)
 123
 124 #define MC_CHANNEL_RANK_PRESENT 0x7c
 125   #define RANK_PRESENT_MASK             0xffff
 126
 127 #define MC_CHANNEL_ADDR_MATCH   0xf0
 128 #define MC_CHANNEL_ERROR_MASK   0xf8
 129 #define MC_CHANNEL_ERROR_INJECT 0xfc
 130   #define INJECT_ADDR_PARITY    0x10
 131   #define INJECT_ECC            0x08
 132   #define MASK_CACHELINE        0x06
 133   #define MASK_FULL_CACHELINE   0x06
 134   #define MASK_MSB32_CACHELINE  0x04
 135   #define MASK_LSB32_CACHELINE  0x02
 136   #define NO_MASK_CACHELINE     0x00
 137   #define REPEAT_EN             0x01
 138
 139         /* OFFSETS for Devices 4,5 and 6 Function 1 */
 140
 141 #define MC_DOD_CH_DIMM0         0x48
 142 #define MC_DOD_CH_DIMM1         0x4c
 143 #define MC_DOD_CH_DIMM2         0x50
 144   #define RANKOFFSET_MASK       ((1 << 12) | (1 << 11) | (1 << 10))
 145   #define RANKOFFSET(x)         ((x & RANKOFFSET_MASK) >> 10)
 146   #define DIMM_PRESENT_MASK     (1 << 9)
 147   #define DIMM_PRESENT(x)       (((x) & DIMM_PRESENT_MASK) >> 9)
 148   #define MC_DOD_NUMBANK_MASK           ((1 << 8) | (1 << 7))
 149   #define MC_DOD_NUMBANK(x)             (((x) & MC_DOD_NUMBANK_MASK) >> 7)
 150   #define MC_DOD_NUMRANK_MASK           ((1 << 6) | (1 << 5))
 151   #define MC_DOD_NUMRANK(x)             (((x) & MC_DOD_NUMRANK_MASK) >> 5)
 152   #define MC_DOD_NUMROW_MASK            ((1 << 4) | (1 << 3) | (1 << 2))
 153   #define MC_DOD_NUMROW(x)              (((x) & MC_DOD_NUMROW_MASK) >> 2)
 154   #define MC_DOD_NUMCOL_MASK            3
 155   #define MC_DOD_NUMCOL(x)              ((x) & MC_DOD_NUMCOL_MASK)
 156
 157 #define MC_RANK_PRESENT         0x7c
 158
 159 #define MC_SAG_CH_0     0x80
 160 #define MC_SAG_CH_1     0x84
 161 #define MC_SAG_CH_2     0x88
 162 #define MC_SAG_CH_3     0x8c
 163 #define MC_SAG_CH_4     0x90
 164 #define MC_SAG_CH_5     0x94
 165 #define MC_SAG_CH_6     0x98
 166 #define MC_SAG_CH_7     0x9c
 167
 168 #define MC_RIR_LIMIT_CH_0       0x40
 169 #define MC_RIR_LIMIT_CH_1       0x44
 170 #define MC_RIR_LIMIT_CH_2       0x48
 171 #define MC_RIR_LIMIT_CH_3       0x4C
 172 #define MC_RIR_LIMIT_CH_4       0x50
 173 #define MC_RIR_LIMIT_CH_5       0x54
 174 #define MC_RIR_LIMIT_CH_6       0x58
 175 #define MC_RIR_LIMIT_CH_7       0x5C
 176 #define MC_RIR_LIMIT_MASK       ((1 << 10) - 1)
 177
 178 #define MC_RIR_WAY_CH           0x80
 179   #define MC_RIR_WAY_OFFSET_MASK        (((1 << 14) - 1) & ~0x7)
 180   #define MC_RIR_WAY_RANK_MASK          0x7
 181
 182 /*
 183  * i7core structs
 184  */
 185
 186 #define NUM_CHANS 3
 187 #define MAX_DIMMS 3             /* Max DIMMS per channel */
 188 #define MAX_MCR_FUNC  4
 189 #define MAX_CHAN_FUNC 3
 190
 191 struct i7core_info {
 192         u32     mc_control;
 193         u32     mc_status;
 194         u32     max_dod;
 195         u32     ch_map;
 196 };
 197
 198
 199 struct i7core_inject {
 200         int     enable;
 201
 202         u32     section;
 203         u32     type;
 204         u32     eccmask;
 205
 206         /* Error address mask */
 207         int channel, dimm, rank, bank, page, col;
 208 };
 209
 210 struct i7core_channel {
 211         u32             ranks;
 212         u32             dimms;
 213 };
 214
 215 struct pci_id_descr {
 216         int                     dev;
 217         int                     func;
 218         int                     dev_id;
 219         int                     optional;
 220 };
 221
 222 struct pci_id_table {
 223         const struct pci_id_descr       *descr;
 224         int                             n_devs;
 225 };
 226
 227 struct i7core_dev {
 228         struct list_head        list;
 229         u8                      socket;
 230         struct pci_dev          **pdev;
 231         int                     n_devs;
 232         struct mem_ctl_info     *mci;
 233 };
 234
 235 struct i7core_pvt {
 236         struct pci_dev  *pci_noncore;
 237         struct pci_dev  *pci_mcr[MAX_MCR_FUNC + 1];
 238         struct pci_dev  *pci_ch[NUM_CHANS][MAX_CHAN_FUNC + 1];
 239
 240         struct i7core_dev *i7core_dev;
 241
 242         struct i7core_info      info;
 243         struct i7core_inject    inject;
 244         struct i7core_channel   channel[NUM_CHANS];
 245
 246         int             ce_count_available;
 247         int             csrow_map[NUM_CHANS][MAX_DIMMS];
 248
 249                         /* ECC corrected errors counts per udimm */
 250         unsigned long   udimm_ce_count[MAX_DIMMS];
 251         int             udimm_last_ce_count[MAX_DIMMS];
 252                         /* ECC corrected errors counts per rdimm */
 253         unsigned long   rdimm_ce_count[NUM_CHANS][MAX_DIMMS];
 254         int             rdimm_last_ce_count[NUM_CHANS][MAX_DIMMS];
 255
 256         unsigned int    is_registered;
 257
 258         /* mcelog glue */
 259         struct edac_mce         edac_mce;
 260
 261         /* Fifo double buffers */
 262         struct mce              mce_entry[MCE_LOG_LEN];
 263         struct mce              mce_outentry[MCE_LOG_LEN];
 264
 265         /* Fifo in/out counters */
 266         unsigned                mce_in, mce_out;
 267
 268         /* Count indicator to show errors not got */
 269         unsigned                mce_overrun;
 270
 271         /* Struct to control EDAC polling */
 272         struct edac_pci_ctl_info *i7core_pci;
 273 };
 274
 275 #define PCI_DESCR(device, function, device_id)  \
 276         .dev = (device),                        \
 277         .func = (function),                     \
 278         .dev_id = (device_id)
 279
 280 static const struct pci_id_descr pci_dev_descr_i7core_nehalem[] = {
 281                 /* Memory controller */
 282         { PCI_DESCR(3, 0, PCI_DEVICE_ID_INTEL_I7_MCR)     },
 283         { PCI_DESCR(3, 1, PCI_DEVICE_ID_INTEL_I7_MC_TAD)  },
 284
 285                 /* Exists only for RDIMM */
 286         { PCI_DESCR(3, 2, PCI_DEVICE_ID_INTEL_I7_MC_RAS), .optional = 1  },
 287         { PCI_DESCR(3, 4, PCI_DEVICE_ID_INTEL_I7_MC_TEST) },
 288
 289                 /* Channel 0 */
 290         { PCI_DESCR(4, 0, PCI_DEVICE_ID_INTEL_I7_MC_CH0_CTRL) },
 291         { PCI_DESCR(4, 1, PCI_DEVICE_ID_INTEL_I7_MC_CH0_ADDR) },
 292         { PCI_DESCR(4, 2, PCI_DEVICE_ID_INTEL_I7_MC_CH0_RANK) },
 293         { PCI_DESCR(4, 3, PCI_DEVICE_ID_INTEL_I7_MC_CH0_TC)   },
 294
 295                 /* Channel 1 */
 296         { PCI_DESCR(5, 0, PCI_DEVICE_ID_INTEL_I7_MC_CH1_CTRL) },
 297         { PCI_DESCR(5, 1, PCI_DEVICE_ID_INTEL_I7_MC_CH1_ADDR) },
 298         { PCI_DESCR(5, 2, PCI_DEVICE_ID_INTEL_I7_MC_CH1_RANK) },
 299         { PCI_DESCR(5, 3, PCI_DEVICE_ID_INTEL_I7_MC_CH1_TC)   },
 300
 301                 /* Channel 2 */
 302         { PCI_DESCR(6, 0, PCI_DEVICE_ID_INTEL_I7_MC_CH2_CTRL) },
 303         { PCI_DESCR(6, 1, PCI_DEVICE_ID_INTEL_I7_MC_CH2_ADDR) },
 304         { PCI_DESCR(6, 2, PCI_DEVICE_ID_INTEL_I7_MC_CH2_RANK) },
 305         { PCI_DESCR(6, 3, PCI_DEVICE_ID_INTEL_I7_MC_CH2_TC)   },
 306 };
 307
 308 static const struct pci_id_descr pci_dev_descr_lynnfield[] = {
 309         { PCI_DESCR( 3, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MCR)         },
 310         { PCI_DESCR( 3, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_TAD)      },
 311         { PCI_DESCR( 3, 4, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_TEST)     },
 312
 313         { PCI_DESCR( 4, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_CTRL) },
 314         { PCI_DESCR( 4, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_ADDR) },
 315         { PCI_DESCR( 4, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_RANK) },
 316         { PCI_DESCR( 4, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_TC)   },
 317
 318         { PCI_DESCR( 5, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_CTRL) },
 319         { PCI_DESCR( 5, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_ADDR) },
 320         { PCI_DESCR( 5, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_RANK) },
 321         { PCI_DESCR( 5, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_TC)   },
 322 };
 323
 324 static const struct pci_id_descr pci_dev_descr_i7core_westmere[] = {
 325                 /* Memory controller */
 326         { PCI_DESCR(3, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MCR_REV2)     },
 327         { PCI_DESCR(3, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_TAD_REV2)  },
 328                         /* Exists only for RDIMM */
 329         { PCI_DESCR(3, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_RAS_REV2), .optional = 1  },
 330         { PCI_DESCR(3, 4, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_TEST_REV2) },
 331
 332                 /* Channel 0 */
 333         { PCI_DESCR(4, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_CTRL_REV2) },
 334         { PCI_DESCR(4, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_ADDR_REV2) },
 335         { PCI_DESCR(4, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_RANK_REV2) },
 336         { PCI_DESCR(4, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_TC_REV2)   },
 337
 338                 /* Channel 1 */
 339         { PCI_DESCR(5, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_CTRL_REV2) },
 340         { PCI_DESCR(5, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_ADDR_REV2) },
 341         { PCI_DESCR(5, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_RANK_REV2) },
 342         { PCI_DESCR(5, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_TC_REV2)   },
 343
 344                 /* Channel 2 */
 345         { PCI_DESCR(6, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH2_CTRL_REV2) },
 346         { PCI_DESCR(6, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH2_ADDR_REV2) },
 347         { PCI_DESCR(6, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH2_RANK_REV2) },
 348         { PCI_DESCR(6, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH2_TC_REV2)   },
 349 };
 350
 351 #define PCI_ID_TABLE_ENTRY(A) { .descr=A, .n_devs = ARRAY_SIZE(A) }
 352 static const struct pci_id_table pci_dev_table[] = {
 353         PCI_ID_TABLE_ENTRY(pci_dev_descr_i7core_nehalem),
 354         PCI_ID_TABLE_ENTRY(pci_dev_descr_lynnfield),
 355         PCI_ID_TABLE_ENTRY(pci_dev_descr_i7core_westmere),
 356 };
 357
 358 /*
 359  *      pci_device_id   table for which devices we are looking for
 360  */
 361 static const struct pci_device_id i7core_pci_tbl[] __devinitdata = {
 362         {PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_X58_HUB_MGMT)},
 363         {PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_LYNNFIELD_QPI_LINK0)},
 364         {0,}                    /* 0 terminated list. */
 365 };
 366
 367 /****************************************************************************
 368                         Anciliary status routines
 369  ****************************************************************************/
 370
 371         /* MC_CONTROL bits */
 372 #define CH_ACTIVE(pvt, ch)      ((pvt)->info.mc_control & (1 << (8 + ch)))
 373 #define ECCx8(pvt)              ((pvt)->info.mc_control & (1 << 1))
 374
 375         /* MC_STATUS bits */
 376 #define ECC_ENABLED(pvt)        ((pvt)->info.mc_status & (1 << 4))
 377 #define CH_DISABLED(pvt, ch)    ((pvt)->info.mc_status & (1 << ch))
 378
 379         /* MC_MAX_DOD read functions */
 380 static inline int numdimms(u32 dimms)
 381 {
 382         return (dimms & 0x3) + 1;
 383 }
 384
 385 static inline int numrank(u32 rank)
 386 {
 387         static int ranks[4] = { 1, 2, 4, -EINVAL };
 388
 389         return ranks[rank & 0x3];
 390 }
 391
 392 static inline int numbank(u32 bank)
 393 {
 394         static int banks[4] = { 4, 8, 16, -EINVAL };
 395
 396         return banks[bank & 0x3];
 397 }
 398
 399 static inline int numrow(u32 row)
 400 {
 401         static int rows[8] = {
 402                 1 << 12, 1 << 13, 1 << 14, 1 << 15,
 403                 1 << 16, -EINVAL, -EINVAL, -EINVAL,
 404         };
 405
 406         return rows[row & 0x7];
 407 }
 408
 409 static inline int numcol(u32 col)
 410 {
 411         static int cols[8] = {
 412                 1 << 10, 1 << 11, 1 << 12, -EINVAL,
 413         };
 414         return cols[col & 0x3];
 415 }
 416
 417 static struct i7core_dev *get_i7core_dev(u8 socket)
 418 {
 419         struct i7core_dev *i7core_dev;
 420
 421         list_for_each_entry(i7core_dev, &i7core_edac_list, list) {
 422                 if (i7core_dev->socket == socket)
 423                         return i7core_dev;
 424         }
 425
 426         return NULL;
 427 }
 428
 429 static struct i7core_dev *alloc_i7core_dev(u8 socket,
 430                                            const struct pci_id_table *table)
 431 {
 432         struct i7core_dev *i7core_dev;
 433
 434         i7core_dev = kzalloc(sizeof(*i7core_dev), GFP_KERNEL);
 435         if (!i7core_dev)
 436                 return NULL;
 437
 438         i7core_dev->pdev = kzalloc(sizeof(*i7core_dev->pdev) * table->n_devs,
 439                                    GFP_KERNEL);
 440         if (!i7core_dev->pdev) {
 441                 kfree(i7core_dev);
 442                 return NULL;
 443         }
 444
 445         i7core_dev->socket = socket;
 446         i7core_dev->n_devs = table->n_devs;
 447         list_add_tail(&i7core_dev->list, &i7core_edac_list);
 448
 449         return i7core_dev;
 450 }
 451
 452 static void free_i7core_dev(struct i7core_dev *i7core_dev)
 453 {
 454         list_del(&i7core_dev->list);
 455         kfree(i7core_dev->pdev);
 456         kfree(i7core_dev);
 457 }
 458
 459 /****************************************************************************
 460                         Memory check routines
 461  ****************************************************************************/
 462 static struct pci_dev *get_pdev_slot_func(u8 socket, unsigned slot,
 463                                           unsigned func)
 464 {
 465         struct i7core_dev *i7core_dev = get_i7core_dev(socket);
 466         int i;
 467
 468         if (!i7core_dev)
 469                 return NULL;
 470
 471         for (i = 0; i < i7core_dev->n_devs; i++) {
 472                 if (!i7core_dev->pdev[i])
 473                         continue;
 474
 475                 if (PCI_SLOT(i7core_dev->pdev[i]->devfn) == slot &&
 476                     PCI_FUNC(i7core_dev->pdev[i]->devfn) == func) {
 477                         return i7core_dev->pdev[i];
 478                 }
 479         }
 480
 481         return NULL;
 482 }
 483
 484 /**
 485  * i7core_get_active_channels() - gets the number of channels and csrows
 486  * @socket:     Quick Path Interconnect socket
 487  * @channels:   Number of channels that will be returned
 488  * @csrows:     Number of csrows found
 489  *
 490  * Since EDAC core needs to know in advance the number of available channels
 491  * and csrows, in order to allocate memory for csrows/channels, it is needed
 492  * to run two similar steps. At the first step, implemented on this function,
 493  * it checks the number of csrows/channels present at one socket.
 494  * this is used in order to properly allocate the size of mci components.
 495  *
 496  * It should be noticed that none of the current available datasheets explain
 497  * or even mention how csrows are seen by the memory controller. So, we need
 498  * to add a fake description for csrows.
 499  * So, this driver is attributing one DIMM memory for one csrow.
 500  */
 501 static int i7core_get_active_channels(const u8 socket, unsigned *channels,
 502                                       unsigned *csrows)
 503 {
 504         struct pci_dev *pdev = NULL;
 505         int i, j;
 506         u32 status, control;
 507
 508         *channels = 0;
 509         *csrows = 0;
 510
 511         pdev = get_pdev_slot_func(socket, 3, 0);
 512         if (!pdev) {
 513                 i7core_printk(KERN_ERR, "Couldn't find socket %d fn 3.0!!!\n",
 514                               socket);
 515                 return -ENODEV;
 516         }
 517
 518         /* Device 3 function 0 reads */
 519         pci_read_config_dword(pdev, MC_STATUS, &status);
 520         pci_read_config_dword(pdev, MC_CONTROL, &control);
 521
 522         for (i = 0; i < NUM_CHANS; i++) {
 523                 u32 dimm_dod[3];
 524                 /* Check if the channel is active */
 525                 if (!(control & (1 << (8 + i))))
 526                         continue;
 527
 528                 /* Check if the channel is disabled */
 529                 if (status & (1 << i))
 530                         continue;
 531
 532                 pdev = get_pdev_slot_func(socket, i + 4, 1);
 533                 if (!pdev) {
 534                         i7core_printk(KERN_ERR, "Couldn't find socket %d "
 535                                                 "fn %d.%d!!!\n",
 536                                                 socket, i + 4, 1);
 537                         return -ENODEV;
 538                 }
 539                 /* Devices 4-6 function 1 */
 540                 pci_read_config_dword(pdev,
 541                                 MC_DOD_CH_DIMM0, &dimm_dod[0]);
 542                 pci_read_config_dword(pdev,
 543                                 MC_DOD_CH_DIMM1, &dimm_dod[1]);
 544                 pci_read_config_dword(pdev,
 545                                 MC_DOD_CH_DIMM2, &dimm_dod[2]);
 546
 547                 (*channels)++;
 548
 549                 for (j = 0; j < 3; j++) {
 550                         if (!DIMM_PRESENT(dimm_dod[j]))
 551                                 continue;
 552                         (*csrows)++;
 553                 }
 554         }
 555
 556         debugf0("Number of active channels on socket %d: %d\n",
 557                 socket, *channels);
 558
 559         return 0;
 560 }
 561
 562 static int get_dimm_config(const struct mem_ctl_info *mci)
 563 {
 564         struct i7core_pvt *pvt = mci->pvt_info;
 565         struct csrow_info *csr;
 566         struct pci_dev *pdev;
 567         int i, j;
 568         int csrow = 0;
 569         unsigned long last_page = 0;
 570         enum edac_type mode;
 571         enum mem_type mtype;
 572
 573         /* Get data from the MC register, function 0 */
 574         pdev = pvt->pci_mcr[0];
 575         if (!pdev)
 576                 return -ENODEV;
 577
 578         /* Device 3 function 0 reads */
 579         pci_read_config_dword(pdev, MC_CONTROL, &pvt->info.mc_control);
 580         pci_read_config_dword(pdev, MC_STATUS, &pvt->info.mc_status);
 581         pci_read_config_dword(pdev, MC_MAX_DOD, &pvt->info.max_dod);
 582         pci_read_config_dword(pdev, MC_CHANNEL_MAPPER, &pvt->info.ch_map);
 583
 584         debugf0("QPI %d control=0x%08x status=0x%08x dod=0x%08x map=0x%08x\n",
 585                 pvt->i7core_dev->socket, pvt->info.mc_control, pvt->info.mc_status,
 586                 pvt->info.max_dod, pvt->info.ch_map);
 587
 588         if (ECC_ENABLED(pvt)) {
 589                 debugf0("ECC enabled with x%d SDCC\n", ECCx8(pvt) ? 8 : 4);
 590                 if (ECCx8(pvt))
 591                         mode = EDAC_S8ECD8ED;
 592                 else
 593                         mode = EDAC_S4ECD4ED;
 594         } else {
 595                 debugf0("ECC disabled\n");
 596                 mode = EDAC_NONE;
 597         }
 598
 599         /* FIXME: need to handle the error codes */
 600         debugf0("DOD Max limits: DIMMS: %d, %d-ranked, %d-banked "
 601                 "x%x x 0x%x\n",
 602                 numdimms(pvt->info.max_dod),
 603                 numrank(pvt->info.max_dod >> 2),
 604                 numbank(pvt->info.max_dod >> 4),
 605                 numrow(pvt->info.max_dod >> 6),
 606                 numcol(pvt->info.max_dod >> 9));
 607
 608         for (i = 0; i < NUM_CHANS; i++) {
 609                 u32 data, dimm_dod[3], value[8];
 610
 611                 if (!pvt->pci_ch[i][0])
 612                         continue;
 613
 614                 if (!CH_ACTIVE(pvt, i)) {
 615                         debugf0("Channel %i is not active\n", i);
 616                         continue;
 617                 }
 618                 if (CH_DISABLED(pvt, i)) {
 619                         debugf0("Channel %i is disabled\n", i);
 620                         continue;
 621                 }
 622
 623                 /* Devices 4-6 function 0 */
 624                 pci_read_config_dword(pvt->pci_ch[i][0],
 625                                 MC_CHANNEL_DIMM_INIT_PARAMS, &data);
 626
 627                 pvt->channel[i].ranks = (data & QUAD_RANK_PRESENT) ?
 628                                                 4 : 2;
 629
 630                 if (data & REGISTERED_DIMM)
 631                         mtype = MEM_RDDR3;
 632                 else
 633                         mtype = MEM_DDR3;
 634 #if 0
 635                 if (data & THREE_DIMMS_PRESENT)
 636                         pvt->channel[i].dimms = 3;
 637                 else if (data & SINGLE_QUAD_RANK_PRESENT)
 638                         pvt->channel[i].dimms = 1;
 639                 else
 640                         pvt->channel[i].dimms = 2;
 641 #endif
 642
 643                 /* Devices 4-6 function 1 */
 644                 pci_read_config_dword(pvt->pci_ch[i][1],
 645                                 MC_DOD_CH_DIMM0, &dimm_dod[0]);
 646                 pci_read_config_dword(pvt->pci_ch[i][1],
 647                                 MC_DOD_CH_DIMM1, &dimm_dod[1]);
 648                 pci_read_config_dword(pvt->pci_ch[i][1],
 649                                 MC_DOD_CH_DIMM2, &dimm_dod[2]);
 650
 651                 debugf0("Ch%d phy rd%d, wr%d (0x%08x): "
 652                         "%d ranks, %cDIMMs\n",
 653                         i,
 654                         RDLCH(pvt->info.ch_map, i), WRLCH(pvt->info.ch_map, i),
 655                         data,
 656                         pvt->channel[i].ranks,
 657                         (data & REGISTERED_DIMM) ? 'R' : 'U');
 658
 659                 for (j = 0; j < 3; j++) {
 660                         u32 banks, ranks, rows, cols;
 661                         u32 size, npages;
 662
 663                         if (!DIMM_PRESENT(dimm_dod[j]))
 664                                 continue;
 665
 666                         banks = numbank(MC_DOD_NUMBANK(dimm_dod[j]));
 667                         ranks = numrank(MC_DOD_NUMRANK(dimm_dod[j]));
 668                         rows = numrow(MC_DOD_NUMROW(dimm_dod[j]));
 669                         cols = numcol(MC_DOD_NUMCOL(dimm_dod[j]));
 670
 671                         /* DDR3 has 8 I/O banks */
 672                         size = (rows * cols * banks * ranks) >> (20 - 3);
 673
 674                         pvt->channel[i].dimms++;
 675
 676                         debugf0("\tdimm %d %d Mb offset: %x, "
 677                                 "bank: %d, rank: %d, row: %#x, col: %#x\n",
 678                                 j, size,
 679                                 RANKOFFSET(dimm_dod[j]),
 680                                 banks, ranks, rows, cols);
 681
 682                         npages = MiB_TO_PAGES(size);
 683
 684                         csr = &mci->csrows[csrow];
 685                         csr->first_page = last_page + 1;
 686                         last_page += npages;
 687                         csr->last_page = last_page;
 688                         csr->nr_pages = npages;
 689
 690                         csr->page_mask = 0;
 691                         csr->grain = 8;
 692                         csr->csrow_idx = csrow;
 693                         csr->nr_channels = 1;
 694
 695                         csr->channels[0].chan_idx = i;
 696                         csr->channels[0].ce_count = 0;
 697
 698                         pvt->csrow_map[i][j] = csrow;
 699
 700                         switch (banks) {
 701                         case 4:
 702                                 csr->dtype = DEV_X4;
 703                                 break;
 704                         case 8:
 705                                 csr->dtype = DEV_X8;
 706                                 break;
 707                         case 16:
 708                                 csr->dtype = DEV_X16;
 709                                 break;
 710                         default:
 711                                 csr->dtype = DEV_UNKNOWN;
 712                         }
 713
 714                         csr->edac_mode = mode;
 715                         csr->mtype = mtype;
 716
 717                         csrow++;
 718                 }
 719
 720                 pci_read_config_dword(pdev, MC_SAG_CH_0, &value[0]);
 721                 pci_read_config_dword(pdev, MC_SAG_CH_1, &value[1]);
 722                 pci_read_config_dword(pdev, MC_SAG_CH_2, &value[2]);
 723                 pci_read_config_dword(pdev, MC_SAG_CH_3, &value[3]);
 724                 pci_read_config_dword(pdev, MC_SAG_CH_4, &value[4]);
 725                 pci_read_config_dword(pdev, MC_SAG_CH_5, &value[5]);
 726                 pci_read_config_dword(pdev, MC_SAG_CH_6, &value[6]);
 727                 pci_read_config_dword(pdev, MC_SAG_CH_7, &value[7]);
 728                 debugf1("\t[%i] DIVBY3\tREMOVED\tOFFSET\n", i);
 729                 for (j = 0; j < 8; j++)
 730                         debugf1("\t\t%#x\t%#x\t%#x\n",
 731                                 (value[j] >> 27) & 0x1,
 732                                 (value[j] >> 24) & 0x7,
 733                                 (value[j] && ((1 << 24) - 1)));
 734         }
 735
 736         return 0;
 737 }
 738
 739 /****************************************************************************
 740                         Error insertion routines
 741  ****************************************************************************/
 742
 743 /* The i7core has independent error injection features per channel.
 744    However, to have a simpler code, we don't allow enabling error injection
 745    on more than one channel.
 746    Also, since a change at an inject parameter will be applied only at enable,
 747    we're disabling error injection on all write calls to the sysfs nodes that
 748    controls the error code injection.
 749  */
 750 static int disable_inject(const struct mem_ctl_info *mci)
 751 {
 752         struct i7core_pvt *pvt = mci->pvt_info;
 753
 754         pvt->inject.enable = 0;
 755
 756         if (!pvt->pci_ch[pvt->inject.channel][0])
 757                 return -ENODEV;
 758
 759         pci_write_config_dword(pvt->pci_ch[pvt->inject.channel][0],
 760                                 MC_CHANNEL_ERROR_INJECT, 0);
 761
 762         return 0;
 763 }
 764
 765 /*
 766  * i7core inject inject.section
 767  *
 768  *      accept and store error injection inject.section value
 769  *      bit 0 - refers to the lower 32-byte half cacheline
 770  *      bit 1 - refers to the upper 32-byte half cacheline
 771  */
 772 static ssize_t i7core_inject_section_store(struct mem_ctl_info *mci,
 773                                            const char *data, size_t count)
 774 {
 775         struct i7core_pvt *pvt = mci->pvt_info;
 776         unsigned long value;
 777         int rc;
 778
 779         if (pvt->inject.enable)
 780                 disable_inject(mci);
 781
 782         rc = strict_strtoul(data, 10, &value);
 783         if ((rc < 0) || (value > 3))
 784                 return -EIO;
 785
 786         pvt->inject.section = (u32) value;
 787         return count;
 788 }
 789
 790 static ssize_t i7core_inject_section_show(struct mem_ctl_info *mci,
 791                                               char *data)
 792 {
 793         struct i7core_pvt *pvt = mci->pvt_info;
 794         return sprintf(data, "0x%08x\n", pvt->inject.section);
 795 }
 796
 797 /*
 798  * i7core inject.type
 799  *
 800  *      accept and store error injection inject.section value
 801  *      bit 0 - repeat enable - Enable error repetition
 802  *      bit 1 - inject ECC error
 803  *      bit 2 - inject parity error
 804  */
 805 static ssize_t i7core_inject_type_store(struct mem_ctl_info *mci,
 806                                         const char *data, size_t count)
 807 {
 808         struct i7core_pvt *pvt = mci->pvt_info;
 809         unsigned long value;
 810         int rc;
 811
 812         if (pvt->inject.enable)
 813                 disable_inject(mci);
 814
 815         rc = strict_strtoul(data, 10, &value);
 816         if ((rc < 0) || (value > 7))
 817                 return -EIO;
 818
 819         pvt->inject.type = (u32) value;
 820         return count;
 821 }
 822
 823 static ssize_t i7core_inject_type_show(struct mem_ctl_info *mci,
 824                                               char *data)
 825 {
 826         struct i7core_pvt *pvt = mci->pvt_info;
 827         return sprintf(data, "0x%08x\n", pvt->inject.type);
 828 }
 829
 830 /*
 831  * i7core_inject_inject.eccmask_store
 832  *
 833  * The type of error (UE/CE) will depend on the inject.eccmask value:
 834  *   Any bits set to a 1 will flip the corresponding ECC bit
 835  *   Correctable errors can be injected by flipping 1 bit or the bits within
 836  *   a symbol pair (2 consecutive aligned 8-bit pairs - i.e. 7:0 and 15:8 or
 837  *   23:16 and 31:24). Flipping bits in two symbol pairs will cause an
 838  *   uncorrectable error to be injected.
 839  */
 840 static ssize_t i7core_inject_eccmask_store(struct mem_ctl_info *mci,
 841                                         const char *data, size_t count)
 842 {
 843         struct i7core_pvt *pvt = mci->pvt_info;
 844         unsigned long value;
 845         int rc;
 846
 847         if (pvt->inject.enable)
 848                 disable_inject(mci);
 849
 850         rc = strict_strtoul(data, 10, &value);
 851         if (rc < 0)
 852                 return -EIO;
 853
 854         pvt->inject.eccmask = (u32) value;
 855         return count;
 856 }
 857
 858 static ssize_t i7core_inject_eccmask_show(struct mem_ctl_info *mci,
 859                                               char *data)
 860 {
 861         struct i7core_pvt *pvt = mci->pvt_info;
 862         return sprintf(data, "0x%08x\n", pvt->inject.eccmask);
 863 }
 864
 865 /*
 866  * i7core_addrmatch
 867  *
 868  * The type of error (UE/CE) will depend on the inject.eccmask value:
 869  *   Any bits set to a 1 will flip the corresponding ECC bit
 870  *   Correctable errors can be injected by flipping 1 bit or the bits within
 871  *   a symbol pair (2 consecutive aligned 8-bit pairs - i.e. 7:0 and 15:8 or
 872  *   23:16 and 31:24). Flipping bits in two symbol pairs will cause an
 873  *   uncorrectable error to be injected.
 874  */
 875
 876 #define DECLARE_ADDR_MATCH(param, limit)                        \
 877 static ssize_t i7core_inject_store_##param(                     \
 878                 struct mem_ctl_info *mci,                       \
 879                 const char *data, size_t count)                 \
 880 {                                                               \
 881         struct i7core_pvt *pvt;                                 \
 882         long value;                                             \
 883         int rc;                                                 \
 884                                                                 \
 885         debugf1("%s()\n", __func__);                            \
 886         pvt = mci->pvt_info;                                    \
 887                                                                 \
 888         if (pvt->inject.enable)                                 \
 889                 disable_inject(mci);                            \
 890                                                                 \
 891         if (!strcasecmp(data, "any") || !strcasecmp(data, "any\n"))\
 892                 value = -1;                                     \
 893         else {                                                  \
 894                 rc = strict_strtoul(data, 10, &value);          \
 895                 if ((rc < 0) || (value >= limit))               \
 896                         return -EIO;                            \
 897         }                                                       \
 898                                                                 \
 899         pvt->inject.param = value;                              \
 900                                                                 \
 901         return count;                                           \
 902 }                                                               \
 903                                                                 \
 904 static ssize_t i7core_inject_show_##param(                      \
 905                 struct mem_ctl_info *mci,                       \
 906                 char *data)                                     \
 907 {                                                               \
 908         struct i7core_pvt *pvt;                                 \
 909                                                                 \
 910         pvt = mci->pvt_info;                                    \
 911         debugf1("%s() pvt=%p\n", __func__, pvt);                \
 912         if (pvt->inject.param < 0)                              \
 913                 return sprintf(data, "any\n");                  \
 914         else                                                    \
 915                 return sprintf(data, "%d\n", pvt->inject.param);\
 916 }
 917
 918 #define ATTR_ADDR_MATCH(param)                                  \
 919         {                                                       \
 920                 .attr = {                                       \
 921                         .name = #param,                         \
 922                         .mode = (S_IRUGO | S_IWUSR)             \
 923                 },                                              \
 924                 .show  = i7core_inject_show_##param,            \
 925                 .store = i7core_inject_store_##param,           \
 926         }
 927
 928 DECLARE_ADDR_MATCH(channel, 3);
 929 DECLARE_ADDR_MATCH(dimm, 3);
 930 DECLARE_ADDR_MATCH(rank, 4);
 931 DECLARE_ADDR_MATCH(bank, 32);
 932 DECLARE_ADDR_MATCH(page, 0x10000);
 933 DECLARE_ADDR_MATCH(col, 0x4000);
 934
 935 static int write_and_test(struct pci_dev *dev, const int where, const u32 val)
 936 {
 937         u32 read;
 938         int count;
 939
 940         debugf0("setting pci %02x:%02x.%x reg=%02x value=%08x\n",
 941                 dev->bus->number, PCI_SLOT(dev->devfn), PCI_FUNC(dev->devfn),
 942                 where, val);
 943
 944         for (count = 0; count < 10; count++) {
 945                 if (count)
 946                         msleep(100);
 947                 pci_write_config_dword(dev, where, val);
 948                 pci_read_config_dword(dev, where, &read);
 949
 950                 if (read == val)
 951                         return 0;
 952         }
 953
 954         i7core_printk(KERN_ERR, "Error during set pci %02x:%02x.%x reg=%02x "
 955                 "write=%08x. Read=%08x\n",
 956                 dev->bus->number, PCI_SLOT(dev->devfn), PCI_FUNC(dev->devfn),
 957                 where, val, read);
 958
 959         return -EINVAL;
 960 }
 961
 962 /*
 963  * This routine prepares the Memory Controller for error injection.
 964  * The error will be injected when some process tries to write to the
 965  * memory that matches the given criteria.
 966  * The criteria can be set in terms of a mask where dimm, rank, bank, page
 967  * and col can be specified.
 968  * A -1 value for any of the mask items will make the MCU to ignore
 969  * that matching criteria for error injection.
 970  *
 971  * It should be noticed that the error will only happen after a write operation
 972  * on a memory that matches the condition. if REPEAT_EN is not enabled at
 973  * inject mask, then it will produce just one error. Otherwise, it will repeat
 974  * until the injectmask would be cleaned.
 975  *
 976  * FIXME: This routine assumes that MAXNUMDIMMS value of MC_MAX_DOD
 977  *    is reliable enough to check if the MC is using the
 978  *    three channels. However, this is not clear at the datasheet.
 979  */
 980 static ssize_t i7core_inject_enable_store(struct mem_ctl_info *mci,
 981                                        const char *data, size_t count)
 982 {
 983         struct i7core_pvt *pvt = mci->pvt_info;
 984         u32 injectmask;
 985         u64 mask = 0;
 986         int  rc;
 987         long enable;
 988
 989         if (!pvt->pci_ch[pvt->inject.channel][0])
 990                 return 0;
 991
 992         rc = strict_strtoul(data, 10, &enable);
 993         if ((rc < 0))
 994                 return 0;
 995
 996         if (enable) {
 997                 pvt->inject.enable = 1;
 998         } else {
 999                 disable_inject(mci);
1000                 return count;
1001         }
1002
1003         /* Sets pvt->inject.dimm mask */
1004         if (pvt->inject.dimm < 0)
1005                 mask |= 1LL << 41;
1006         else {
1007                 if (pvt->channel[pvt->inject.channel].dimms > 2)
1008                         mask |= (pvt->inject.dimm & 0x3LL) << 35;
1009                 else
1010                         mask |= (pvt->inject.dimm & 0x1LL) << 36;
1011         }
1012
1013         /* Sets pvt->inject.rank mask */
1014         if (pvt->inject.rank < 0)
1015                 mask |= 1LL << 40;
1016         else {
1017                 if (pvt->channel[pvt->inject.channel].dimms > 2)
1018                         mask |= (pvt->inject.rank & 0x1LL) << 34;
1019                 else
1020                         mask |= (pvt->inject.rank & 0x3LL) << 34;
1021         }
1022
1023         /* Sets pvt->inject.bank mask */
1024         if (pvt->inject.bank < 0)
1025                 mask |= 1LL << 39;
1026         else
1027                 mask |= (pvt->inject.bank & 0x15LL) << 30;
1028
1029         /* Sets pvt->inject.page mask */
1030         if (pvt->inject.page < 0)
1031                 mask |= 1LL << 38;
1032         else
1033                 mask |= (pvt->inject.page & 0xffff) << 14;
1034
1035         /* Sets pvt->inject.column mask */
1036         if (pvt->inject.col < 0)
1037                 mask |= 1LL << 37;
1038         else
1039                 mask |= (pvt->inject.col & 0x3fff);
1040
1041         /*
1042          * bit    0: REPEAT_EN
1043          * bits 1-2: MASK_HALF_CACHELINE
1044          * bit    3: INJECT_ECC
1045          * bit    4: INJECT_ADDR_PARITY
1046          */
1047
1048         injectmask = (pvt->inject.type & 1) |
1049                      (pvt->inject.section & 0x3) << 1 |
1050                      (pvt->inject.type & 0x6) << (3 - 1);
1051
1052         /* Unlock writes to registers - this register is write only */
1053         pci_write_config_dword(pvt->pci_noncore,
1054                                MC_CFG_CONTROL, 0x2);
1055
1056         write_and_test(pvt->pci_ch[pvt->inject.channel][0],
1057                                MC_CHANNEL_ADDR_MATCH, mask);
1058         write_and_test(pvt->pci_ch[pvt->inject.channel][0],
1059                                MC_CHANNEL_ADDR_MATCH + 4, mask >> 32L);
1060
1061         write_and_test(pvt->pci_ch[pvt->inject.channel][0],
1062                                MC_CHANNEL_ERROR_MASK, pvt->inject.eccmask);
1063
1064         write_and_test(pvt->pci_ch[pvt->inject.channel][0],
1065                                MC_CHANNEL_ERROR_INJECT, injectmask);
1066
1067         /*
1068          * This is something undocumented, based on my tests
1069          * Without writing 8 to this register, errors aren't injected. Not sure
1070          * why.
1071          */
1072         pci_write_config_dword(pvt->pci_noncore,
1073                                MC_CFG_CONTROL, 8);
1074
1075         debugf0("Error inject addr match 0x%016llx, ecc 0x%08x,"
1076                 " inject 0x%08x\n",
1077                 mask, pvt->inject.eccmask, injectmask);
1078
1079
1080         return count;
1081 }
1082
1083 static ssize_t i7core_inject_enable_show(struct mem_ctl_info *mci,
1084                                         char *data)
1085 {
1086         struct i7core_pvt *pvt = mci->pvt_info;
1087         u32 injectmask;
1088
1089         if (!pvt->pci_ch[pvt->inject.channel][0])
1090                 return 0;
1091
1092         pci_read_config_dword(pvt->pci_ch[pvt->inject.channel][0],
1093                                MC_CHANNEL_ERROR_INJECT, &injectmask);
1094
1095         debugf0("Inject error read: 0x%018x\n", injectmask);
1096
1097         if (injectmask & 0x0c)
1098                 pvt->inject.enable = 1;
1099
1100         return sprintf(data, "%d\n", pvt->inject.enable);
1101 }
1102
1103 #define DECLARE_COUNTER(param)                                  \
1104 static ssize_t i7core_show_counter_##param(                     \
1105                 struct mem_ctl_info *mci,                       \
1106                 char *data)                                     \
1107 {                                                               \
1108         struct i7core_pvt *pvt = mci->pvt_info;                 \
1109                                                                 \
1110         debugf1("%s() \n", __func__);                           \
1111         if (!pvt->ce_count_available || (pvt->is_registered))   \
1112                 return sprintf(data, "data unavailable\n");     \
1113         return sprintf(data, "%lu\n",                           \
1114                         pvt->udimm_ce_count[param]);            \
1115 }
1116
1117 #define ATTR_COUNTER(param)                                     \
1118         {                                                       \
1119                 .attr = {                                       \
1120                         .name = __stringify(udimm##param),      \
1121                         .mode = (S_IRUGO | S_IWUSR)             \
1122                 },                                              \
1123                 .show  = i7core_show_counter_##param            \
1124         }
1125
1126 DECLARE_COUNTER(0);
1127 DECLARE_COUNTER(1);
1128 DECLARE_COUNTER(2);
1129
1130 /*
1131  * Sysfs struct
1132  */
1133
1134 static const struct mcidev_sysfs_attribute i7core_addrmatch_attrs[] = {
1135         ATTR_ADDR_MATCH(channel),
1136         ATTR_ADDR_MATCH(dimm),
1137         ATTR_ADDR_MATCH(rank),
1138         ATTR_ADDR_MATCH(bank),
1139         ATTR_ADDR_MATCH(page),
1140         ATTR_ADDR_MATCH(col),
1141         { } /* End of list */
1142 };
1143
1144 static const struct mcidev_sysfs_group i7core_inject_addrmatch = {
1145         .name  = "inject_addrmatch",
1146         .mcidev_attr = i7core_addrmatch_attrs,
1147 };
1148
1149 static const struct mcidev_sysfs_attribute i7core_udimm_counters_attrs[] = {
1150         ATTR_COUNTER(0),
1151         ATTR_COUNTER(1),
1152         ATTR_COUNTER(2),
1153         { .attr = { .name = NULL } }
1154 };
1155
1156 static const struct mcidev_sysfs_group i7core_udimm_counters = {
1157         .name  = "all_channel_counts",
1158         .mcidev_attr = i7core_udimm_counters_attrs,
1159 };
1160
1161 static const struct mcidev_sysfs_attribute i7core_sysfs_rdimm_attrs[] = {
1162         {
1163                 .attr = {
1164                         .name = "inject_section",
1165                         .mode = (S_IRUGO | S_IWUSR)
1166                 },
1167                 .show  = i7core_inject_section_show,
1168                 .store = i7core_inject_section_store,
1169         }, {
1170                 .attr = {
1171                         .name = "inject_type",
1172                         .mode = (S_IRUGO | S_IWUSR)
1173                 },
1174                 .show  = i7core_inject_type_show,
1175                 .store = i7core_inject_type_store,
1176         }, {
1177                 .attr = {
1178                         .name = "inject_eccmask",
1179                         .mode = (S_IRUGO | S_IWUSR)
1180                 },
1181                 .show  = i7core_inject_eccmask_show,
1182                 .store = i7core_inject_eccmask_store,
1183         }, {
1184                 .grp = &i7core_inject_addrmatch,
1185         }, {
1186                 .attr = {
1187                         .name = "inject_enable",
1188                         .mode = (S_IRUGO | S_IWUSR)
1189                 },
1190                 .show  = i7core_inject_enable_show,
1191                 .store = i7core_inject_enable_store,
1192         },
1193         { }     /* End of list */
1194 };
1195
1196 static const struct mcidev_sysfs_attribute i7core_sysfs_udimm_attrs[] = {
1197         {
1198                 .attr = {
1199                         .name = "inject_section",
1200                         .mode = (S_IRUGO | S_IWUSR)
1201                 },
1202                 .show  = i7core_inject_section_show,
1203                 .store = i7core_inject_section_store,
1204         }, {
1205                 .attr = {
1206                         .name = "inject_type",
1207                         .mode = (S_IRUGO | S_IWUSR)
1208                 },
1209                 .show  = i7core_inject_type_show,
1210                 .store = i7core_inject_type_store,
1211         }, {
1212                 .attr = {
1213                         .name = "inject_eccmask",
1214                         .mode = (S_IRUGO | S_IWUSR)
1215                 },
1216                 .show  = i7core_inject_eccmask_show,
1217                 .store = i7core_inject_eccmask_store,
1218         }, {
1219                 .grp = &i7core_inject_addrmatch,
1220         }, {
1221                 .attr = {
1222                         .name = "inject_enable",
1223                         .mode = (S_IRUGO | S_IWUSR)
1224                 },
1225                 .show  = i7core_inject_enable_show,
1226                 .store = i7core_inject_enable_store,
1227         }, {
1228                 .grp = &i7core_udimm_counters,
1229         },
1230         { }     /* End of list */
1231 };
1232
1233 /****************************************************************************
1234         Device initialization routines: put/get, init/exit
1235  ****************************************************************************/
1236
1237 /*
1238  *      i7core_put_all_devices  'put' all the devices that we have
1239  *                              reserved via 'get'
1240  */
1241 static void i7core_put_devices(struct i7core_dev *i7core_dev)
1242 {
1243         int i;
1244
1245         debugf0(__FILE__ ": %s()\n", __func__);
1246         for (i = 0; i < i7core_dev->n_devs; i++) {
1247                 struct pci_dev *pdev = i7core_dev->pdev[i];
1248                 if (!pdev)
1249                         continue;
1250                 debugf0("Removing dev %02x:%02x.%d\n",
1251                         pdev->bus->number,
1252                         PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
1253                 pci_dev_put(pdev);
1254         }
1255 }
1256
1257 static void i7core_put_all_devices(void)
1258 {
1259         struct i7core_dev *i7core_dev, *tmp;
1260
1261         list_for_each_entry_safe(i7core_dev, tmp, &i7core_edac_list, list) {
1262                 i7core_put_devices(i7core_dev);
1263                 free_i7core_dev(i7core_dev);
1264         }
1265 }
1266
1267 static void __init i7core_xeon_pci_fixup(const struct pci_id_table *table)
1268 {
1269         struct pci_dev *pdev = NULL;
1270         int i;
1271
1272         /*
1273          * On Xeon 55xx, the Intel Quckpath Arch Generic Non-core pci buses
1274          * aren't announced by acpi. So, we need to use a legacy scan probing
1275          * to detect them
1276          */
1277         while (table && table->descr) {
1278                 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, table->descr[0].dev_id, NULL);
1279                 if (unlikely(!pdev)) {
1280                         for (i = 0; i < MAX_SOCKET_BUSES; i++)
1281                                 pcibios_scan_specific_bus(255-i);
1282                 }
1283                 pci_dev_put(pdev);
1284                 table++;
1285         }
1286 }
1287
1288 static unsigned i7core_pci_lastbus(void)
1289 {
1290         int last_bus = 0, bus;
1291         struct pci_bus *b = NULL;
1292
1293         while ((b = pci_find_next_bus(b)) != NULL) {
1294                 bus = b->number;
1295                 debugf0("Found bus %d\n", bus);
1296                 if (bus > last_bus)
1297                         last_bus = bus;
1298         }
1299
1300         debugf0("Last bus %d\n", last_bus);
1301
1302         return last_bus;
1303 }
1304
1305 /*
1306  *      i7core_get_all_devices  Find and perform 'get' operation on the MCH's
1307  *                      device/functions we want to reference for this driver
1308  *
1309  *                      Need to 'get' device 16 func 1 and func 2
1310  */
1311 static int i7core_get_onedevice(struct pci_dev **prev,
1312                                 const struct pci_id_table *table,
1313                                 const unsigned devno,
1314                                 const unsigned last_bus)
1315 {
1316         struct i7core_dev *i7core_dev;
1317         const struct pci_id_descr *dev_descr = &table->descr[devno];
1318
1319         struct pci_dev *pdev = NULL;
1320         u8 bus = 0;
1321         u8 socket = 0;
1322
1323         pdev = pci_get_device(PCI_VENDOR_ID_INTEL,
1324                               dev_descr->dev_id, *prev);
1325
1326         if (!pdev) {
1327                 if (*prev) {
1328                         *prev = pdev;
1329                         return 0;
1330                 }
1331
1332                 if (dev_descr->optional)
1333                         return 0;
1334
1335                 if (devno == 0)
1336                         return -ENODEV;
1337
1338                 i7core_printk(KERN_INFO,
1339                         "Device not found: dev %02x.%d PCI ID %04x:%04x\n",
1340                         dev_descr->dev, dev_descr->func,
1341                         PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
1342
1343                 /* End of list, leave */
1344                 return -ENODEV;
1345         }
1346         bus = pdev->bus->number;
1347
1348         socket = last_bus - bus;
1349
1350         i7core_dev = get_i7core_dev(socket);
1351         if (!i7core_dev) {
1352                 i7core_dev = alloc_i7core_dev(socket, table);
1353                 if (!i7core_dev) {
1354                         pci_dev_put(pdev);
1355                         return -ENOMEM;
1356                 }
1357         }
1358
1359         if (i7core_dev->pdev[devno]) {
1360                 i7core_printk(KERN_ERR,
1361                         "Duplicated device for "
1362                         "dev %02x:%02x.%d PCI ID %04x:%04x\n",
1363                         bus, dev_descr->dev, dev_descr->func,
1364                         PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
1365                 pci_dev_put(pdev);
1366                 return -ENODEV;
1367         }
1368
1369         i7core_dev->pdev[devno] = pdev;
1370
1371         /* Sanity check */
1372         if (unlikely(PCI_SLOT(pdev->devfn) != dev_descr->dev ||
1373                         PCI_FUNC(pdev->devfn) != dev_descr->func)) {
1374                 i7core_printk(KERN_ERR,
1375                         "Device PCI ID %04x:%04x "
1376                         "has dev %02x:%02x.%d instead of dev %02x:%02x.%d\n",
1377                         PCI_VENDOR_ID_INTEL, dev_descr->dev_id,
1378                         bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn),
1379                         bus, dev_descr->dev, dev_descr->func);
1380                 return -ENODEV;
1381         }
1382
1383         /* Be sure that the device is enabled */
1384         if (unlikely(pci_enable_device(pdev) < 0)) {
1385                 i7core_printk(KERN_ERR,
1386                         "Couldn't enable "
1387                         "dev %02x:%02x.%d PCI ID %04x:%04x\n",
1388                         bus, dev_descr->dev, dev_descr->func,
1389                         PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
1390                 return -ENODEV;
1391         }
1392
1393         debugf0("Detected socket %d dev %02x:%02x.%d PCI ID %04x:%04x\n",
1394                 socket, bus, dev_descr->dev,
1395                 dev_descr->func,
1396                 PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
1397
1398         /*
1399          * As stated on drivers/pci/search.c, the reference count for
1400          * @from is always decremented if it is not %NULL. So, as we need
1401          * to get all devices up to null, we need to do a get for the device
1402          */
1403         pci_dev_get(pdev);
1404
1405         *prev = pdev;
1406
1407         return 0;
1408 }
1409
1410 static int i7core_get_all_devices(void)
1411 {
1412         int i, j, rc, last_bus;
1413         struct pci_dev *pdev = NULL;
1414         const struct pci_id_table *table;
1415
1416         last_bus = i7core_pci_lastbus();
1417
1418         for (j = 0; j < ARRAY_SIZE(pci_dev_table); j++) {
1419                 table = &pci_dev_table[j];
1420                 for (i = 0; i < table->n_devs; i++) {
1421                         pdev = NULL;
1422                         do {
1423                                 rc = i7core_get_onedevice(&pdev, table, i,
1424                                                           last_bus);
1425                                 if (rc < 0) {
1426                                         if (i == 0) {
1427                                                 i = table->n_devs;
1428                                                 break;
1429                                         }
1430                                         i7core_put_all_devices();
1431                                         return -ENODEV;
1432                                 }
1433                         } while (pdev);
1434                 }
1435         }
1436
1437         return 0;
1438 }
1439
1440 static int mci_bind_devs(struct mem_ctl_info *mci,
1441                          struct i7core_dev *i7core_dev)
1442 {
1443         struct i7core_pvt *pvt = mci->pvt_info;
1444         struct pci_dev *pdev;
1445         int i, func, slot;
1446
1447         pvt->is_registered = 0;
1448         for (i = 0; i < i7core_dev->n_devs; i++) {
1449                 pdev = i7core_dev->pdev[i];
1450                 if (!pdev)
1451                         continue;
1452
1453                 func = PCI_FUNC(pdev->devfn);
1454                 slot = PCI_SLOT(pdev->devfn);
1455                 if (slot == 3) {
1456                         if (unlikely(func > MAX_MCR_FUNC))
1457                                 goto error;
1458                         pvt->pci_mcr[func] = pdev;
1459                 } else if (likely(slot >= 4 && slot < 4 + NUM_CHANS)) {
1460                         if (unlikely(func > MAX_CHAN_FUNC))
1461                                 goto error;
1462                         pvt->pci_ch[slot - 4][func] = pdev;
1463                 } else if (!slot && !func)
1464                         pvt->pci_noncore = pdev;
1465                 else
1466                         goto error;
1467
1468                 debugf0("Associated fn %d.%d, dev = %p, socket %d\n",
1469                         PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn),
1470                         pdev, i7core_dev->socket);
1471
1472                 if (PCI_SLOT(pdev->devfn) == 3 &&
1473                         PCI_FUNC(pdev->devfn) == 2)
1474                         pvt->is_registered = 1;
1475         }
1476
1477         return 0;
1478
1479 error:
1480         i7core_printk(KERN_ERR, "Device %d, function %d "
1481                       "is out of the expected range\n",
1482                       slot, func);
1483         return -EINVAL;
1484 }
1485
1486 /****************************************************************************
1487                         Error check routines
1488  ****************************************************************************/
1489 static void i7core_rdimm_update_csrow(struct mem_ctl_info *mci,
1490                                       const int chan,
1491                                       const int dimm,
1492                                       const int add)
1493 {
1494         char *msg;
1495         struct i7core_pvt *pvt = mci->pvt_info;
1496         int row = pvt->csrow_map[chan][dimm], i;
1497
1498         for (i = 0; i < add; i++) {
1499                 msg = kasprintf(GFP_KERNEL, "Corrected error "
1500                                 "(Socket=%d channel=%d dimm=%d)",
1501                                 pvt->i7core_dev->socket, chan, dimm);
1502
1503                 edac_mc_handle_fbd_ce(mci, row, 0, msg);
1504                 kfree (msg);
1505         }
1506 }
1507
1508 static void i7core_rdimm_update_ce_count(struct mem_ctl_info *mci,
1509                                          const int chan,
1510                                          const int new0,
1511                                          const int new1,
1512                                          const int new2)
1513 {
1514         struct i7core_pvt *pvt = mci->pvt_info;
1515         int add0 = 0, add1 = 0, add2 = 0;
1516         /* Updates CE counters if it is not the first time here */
1517         if (pvt->ce_count_available) {
1518                 /* Updates CE counters */
1519
1520                 add2 = new2 - pvt->rdimm_last_ce_count[chan][2];
1521                 add1 = new1 - pvt->rdimm_last_ce_count[chan][1];
1522                 add0 = new0 - pvt->rdimm_last_ce_count[chan][0];
1523
1524                 if (add2 < 0)
1525                         add2 += 0x7fff;
1526                 pvt->rdimm_ce_count[chan][2] += add2;
1527
1528                 if (add1 < 0)
1529                         add1 += 0x7fff;
1530                 pvt->rdimm_ce_count[chan][1] += add1;
1531
1532                 if (add0 < 0)
1533                         add0 += 0x7fff;
1534                 pvt->rdimm_ce_count[chan][0] += add0;
1535         } else
1536                 pvt->ce_count_available = 1;
1537
1538         /* Store the new values */
1539         pvt->rdimm_last_ce_count[chan][2] = new2;
1540         pvt->rdimm_last_ce_count[chan][1] = new1;
1541         pvt->rdimm_last_ce_count[chan][0] = new0;
1542
1543         /*updated the edac core */
1544         if (add0 != 0)
1545                 i7core_rdimm_update_csrow(mci, chan, 0, add0);
1546         if (add1 != 0)
1547                 i7core_rdimm_update_csrow(mci, chan, 1, add1);
1548         if (add2 != 0)
1549                 i7core_rdimm_update_csrow(mci, chan, 2, add2);
1550
1551 }
1552
1553 static void i7core_rdimm_check_mc_ecc_err(struct mem_ctl_info *mci)
1554 {
1555         struct i7core_pvt *pvt = mci->pvt_info;
1556         u32 rcv[3][2];
1557         int i, new0, new1, new2;
1558
1559         /*Read DEV 3: FUN 2:  MC_COR_ECC_CNT regs directly*/
1560         pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_0,
1561                                                                 &rcv[0][0]);
1562         pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_1,
1563                                                                 &rcv[0][1]);
1564         pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_2,
1565                                                                 &rcv[1][0]);
1566         pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_3,
1567                                                                 &rcv[1][1]);
1568         pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_4,
1569                                                                 &rcv[2][0]);
1570         pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_5,
1571                                                                 &rcv[2][1]);
1572         for (i = 0 ; i < 3; i++) {
1573                 debugf3("MC_COR_ECC_CNT%d = 0x%x; MC_COR_ECC_CNT%d = 0x%x\n",
1574                         (i * 2), rcv[i][0], (i * 2) + 1, rcv[i][1]);
1575                 /*if the channel has 3 dimms*/
1576                 if (pvt->channel[i].dimms > 2) {
1577                         new0 = DIMM_BOT_COR_ERR(rcv[i][0]);
1578                         new1 = DIMM_TOP_COR_ERR(rcv[i][0]);
1579                         new2 = DIMM_BOT_COR_ERR(rcv[i][1]);
1580                 } else {
1581                         new0 = DIMM_TOP_COR_ERR(rcv[i][0]) +
1582                                         DIMM_BOT_COR_ERR(rcv[i][0]);
1583                         new1 = DIMM_TOP_COR_ERR(rcv[i][1]) +
1584                                         DIMM_BOT_COR_ERR(rcv[i][1]);
1585                         new2 = 0;
1586                 }
1587
1588                 i7core_rdimm_update_ce_count(mci, i, new0, new1, new2);
1589         }
1590 }
1591
1592 /* This function is based on the device 3 function 4 registers as described on:
1593  * Intel Xeon Processor 5500 Series Datasheet Volume 2
1594  *      http://www.intel.com/Assets/PDF/datasheet/321322.pdf
1595  * also available at:
1596  *      http://www.arrownac.com/manufacturers/intel/s/nehalem/5500-datasheet-v2.pdf
1597  */
1598 static void i7core_udimm_check_mc_ecc_err(struct mem_ctl_info *mci)
1599 {
1600         struct i7core_pvt *pvt = mci->pvt_info;
1601         u32 rcv1, rcv0;
1602         int new0, new1, new2;
1603
1604         if (!pvt->pci_mcr[4]) {
1605                 debugf0("%s MCR registers not found\n", __func__);
1606                 return;
1607         }
1608
1609         /* Corrected test errors */
1610         pci_read_config_dword(pvt->pci_mcr[4], MC_TEST_ERR_RCV1, &rcv1);
1611         pci_read_config_dword(pvt->pci_mcr[4], MC_TEST_ERR_RCV0, &rcv0);
1612
1613         /* Store the new values */
1614         new2 = DIMM2_COR_ERR(rcv1);
1615         new1 = DIMM1_COR_ERR(rcv0);
1616         new0 = DIMM0_COR_ERR(rcv0);
1617
1618         /* Updates CE counters if it is not the first time here */
1619         if (pvt->ce_count_available) {
1620                 /* Updates CE counters */
1621                 int add0, add1, add2;
1622
1623                 add2 = new2 - pvt->udimm_last_ce_count[2];
1624                 add1 = new1 - pvt->udimm_last_ce_count[1];
1625                 add0 = new0 - pvt->udimm_last_ce_count[0];
1626
1627                 if (add2 < 0)
1628                         add2 += 0x7fff;
1629                 pvt->udimm_ce_count[2] += add2;
1630
1631                 if (add1 < 0)
1632                         add1 += 0x7fff;
1633                 pvt->udimm_ce_count[1] += add1;
1634
1635                 if (add0 < 0)
1636                         add0 += 0x7fff;
1637                 pvt->udimm_ce_count[0] += add0;
1638
1639                 if (add0 | add1 | add2)
1640                         i7core_printk(KERN_ERR, "New Corrected error(s): "
1641                                       "dimm0: +%d, dimm1: +%d, dimm2 +%d\n",
1642                                       add0, add1, add2);
1643         } else
1644                 pvt->ce_count_available = 1;
1645
1646         /* Store the new values */
1647         pvt->udimm_last_ce_count[2] = new2;
1648         pvt->udimm_last_ce_count[1] = new1;
1649         pvt->udimm_last_ce_count[0] = new0;
1650 }
1651
1652 /*
1653  * According with tables E-11 and E-12 of chapter E.3.3 of Intel 64 and IA-32
1654  * Architectures Software Developer’s Manual Volume 3B.
1655  * Nehalem are defined as family 0x06, model 0x1a
1656  *
1657  * The MCA registers used here are the following ones:
1658  *     struct mce field MCA Register
1659  *     m->status        MSR_IA32_MC8_STATUS
1660  *     m->addr          MSR_IA32_MC8_ADDR
1661  *     m->misc          MSR_IA32_MC8_MISC
1662  * In the case of Nehalem, the error information is masked at .status and .misc
1663  * fields
1664  */
1665 static void i7core_mce_output_error(struct mem_ctl_info *mci,
1666                                     const struct mce *m)
1667 {
1668         struct i7core_pvt *pvt = mci->pvt_info;
1669         char *type, *optype, *err, *msg;
1670         unsigned long error = m->status & 0x1ff0000l;
1671         u32 optypenum = (m->status >> 4) & 0x07;
1672         u32 core_err_cnt = (m->status >> 38) && 0x7fff;
1673         u32 dimm = (m->misc >> 16) & 0x3;
1674         u32 channel = (m->misc >> 18) & 0x3;
1675         u32 syndrome = m->misc >> 32;
1676         u32 errnum = find_first_bit(&error, 32);
1677         int csrow;
1678
1679         if (m->mcgstatus & 1)
1680                 type = "FATAL";
1681         else
1682                 type = "NON_FATAL";
1683
1684         switch (optypenum) {
1685         case 0:
1686                 optype = "generic undef request";
1687                 break;
1688         case 1:
1689                 optype = "read error";
1690                 break;
1691         case 2:
1692                 optype = "write error";
1693                 break;
1694         case 3:
1695                 optype = "addr/cmd error";
1696                 break;
1697         case 4:
1698                 optype = "scrubbing error";
1699                 break;
1700         default:
1701                 optype = "reserved";
1702                 break;
1703         }
1704
1705         switch (errnum) {
1706         case 16:
1707                 err = "read ECC error";
1708                 break;
1709         case 17:
1710                 err = "RAS ECC error";
1711                 break;
1712         case 18:
1713                 err = "write parity error";
1714                 break;
1715         case 19:
1716                 err = "redundacy loss";
1717                 break;
1718         case 20:
1719                 err = "reserved";
1720                 break;
1721         case 21:
1722                 err = "memory range error";
1723                 break;
1724         case 22:
1725                 err = "RTID out of range";
1726                 break;
1727         case 23:
1728                 err = "address parity error";
1729                 break;
1730         case 24:
1731                 err = "byte enable parity error";
1732                 break;
1733         default:
1734                 err = "unknown";
1735         }
1736
1737         /* FIXME: should convert addr into bank and rank information */
1738         msg = kasprintf(GFP_ATOMIC,
1739                 "%s (addr = 0x%08llx, cpu=%d, Dimm=%d, Channel=%d, "
1740                 "syndrome=0x%08x, count=%d, Err=%08llx:%08llx (%s: %s))\n",
1741                 type, (long long) m->addr, m->cpu, dimm, channel,
1742                 syndrome, core_err_cnt, (long long)m->status,
1743                 (long long)m->misc, optype, err);
1744
1745         debugf0("%s", msg);
1746
1747         csrow = pvt->csrow_map[channel][dimm];
1748
1749         /* Call the helper to output message */
1750         if (m->mcgstatus & 1)
1751                 edac_mc_handle_fbd_ue(mci, csrow, 0,
1752                                 0 /* FIXME: should be channel here */, msg);
1753         else if (!pvt->is_registered)
1754                 edac_mc_handle_fbd_ce(mci, csrow,
1755                                 0 /* FIXME: should be channel here */, msg);
1756
1757         kfree(msg);
1758 }
1759
1760 /*
1761  *      i7core_check_error      Retrieve and process errors reported by the
1762  *                              hardware. Called by the Core module.
1763  */
1764 static void i7core_check_error(struct mem_ctl_info *mci)
1765 {
1766         struct i7core_pvt *pvt = mci->pvt_info;
1767         int i;
1768         unsigned count = 0;
1769         struct mce *m;
1770
1771         /*
1772          * MCE first step: Copy all mce errors into a temporary buffer
1773          * We use a double buffering here, to reduce the risk of
1774          * loosing an error.
1775          */
1776         smp_rmb();
1777         count = (pvt->mce_out + MCE_LOG_LEN - pvt->mce_in)
1778                 % MCE_LOG_LEN;
1779         if (!count)
1780                 goto check_ce_error;
1781
1782         m = pvt->mce_outentry;
1783         if (pvt->mce_in + count > MCE_LOG_LEN) {
1784                 unsigned l = MCE_LOG_LEN - pvt->mce_in;
1785
1786                 memcpy(m, &pvt->mce_entry[pvt->mce_in], sizeof(*m) * l);
1787                 smp_wmb();
1788                 pvt->mce_in = 0;
1789                 count -= l;
1790                 m += l;
1791         }
1792         memcpy(m, &pvt->mce_entry[pvt->mce_in], sizeof(*m) * count);
1793         smp_wmb();
1794         pvt->mce_in += count;
1795
1796         smp_rmb();
1797         if (pvt->mce_overrun) {
1798                 i7core_printk(KERN_ERR, "Lost %d memory errors\n",
1799                               pvt->mce_overrun);
1800                 smp_wmb();
1801                 pvt->mce_overrun = 0;
1802         }
1803
1804         /*
1805          * MCE second step: parse errors and display
1806          */
1807         for (i = 0; i < count; i++)
1808                 i7core_mce_output_error(mci, &pvt->mce_outentry[i]);
1809
1810         /*
1811          * Now, let's increment CE error counts
1812          */
1813 check_ce_error:
1814         if (!pvt->is_registered)
1815                 i7core_udimm_check_mc_ecc_err(mci);
1816         else
1817                 i7core_rdimm_check_mc_ecc_err(mci);
1818 }
1819
1820 /*
1821  * i7core_mce_check_error       Replicates mcelog routine to get errors
1822  *                              This routine simply queues mcelog errors, and
1823  *                              return. The error itself should be handled later
1824  *                              by i7core_check_error.
1825  * WARNING: As this routine should be called at NMI time, extra care should
1826  * be taken to avoid deadlocks, and to be as fast as possible.
1827  */
1828 static int i7core_mce_check_error(void *priv, struct mce *mce)
1829 {
1830         struct mem_ctl_info *mci = priv;
1831         struct i7core_pvt *pvt = mci->pvt_info;
1832
1833         /*
1834          * Just let mcelog handle it if the error is
1835          * outside the memory controller
1836          */
1837         if (((mce->status & 0xffff) >> 7) != 1)
1838                 return 0;
1839
1840         /* Bank 8 registers are the only ones that we know how to handle */
1841         if (mce->bank != 8)
1842                 return 0;
1843
1844 #ifdef CONFIG_SMP
1845         /* Only handle if it is the right mc controller */
1846         if (cpu_data(mce->cpu).phys_proc_id != pvt->i7core_dev->socket)
1847                 return 0;
1848 #endif
1849
1850         smp_rmb();
1851         if ((pvt->mce_out + 1) % MCE_LOG_LEN == pvt->mce_in) {
1852                 smp_wmb();
1853                 pvt->mce_overrun++;
1854                 return 0;
1855         }
1856
1857         /* Copy memory error at the ringbuffer */
1858         memcpy(&pvt->mce_entry[pvt->mce_out], mce, sizeof(*mce));
1859         smp_wmb();
1860         pvt->mce_out = (pvt->mce_out + 1) % MCE_LOG_LEN;
1861
1862         /* Handle fatal errors immediately */
1863         if (mce->mcgstatus & 1)
1864                 i7core_check_error(mci);
1865
1866         /* Advice mcelog that the error were handled */
1867         return 1;
1868 }
1869
1870 static void i7core_pci_ctl_create(struct i7core_pvt *pvt)
1871 {
1872         pvt->i7core_pci = edac_pci_create_generic_ctl(
1873                                                 &pvt->i7core_dev->pdev[0]->dev,
1874                                                 EDAC_MOD_STR);
1875         if (unlikely(!pvt->i7core_pci))
1876                 pr_warn("Unable to setup PCI error report via EDAC\n");
1877 }
1878
1879 static void i7core_pci_ctl_release(struct i7core_pvt *pvt)
1880 {
1881         if (likely(pvt->i7core_pci))
1882                 edac_pci_release_generic_ctl(pvt->i7core_pci);
1883         else
1884                 i7core_printk(KERN_ERR,
1885                                 "Couldn't find mem_ctl_info for socket %d\n",
1886                                 pvt->i7core_dev->socket);
1887         pvt->i7core_pci = NULL;
1888 }
1889
1890 static void i7core_unregister_mci(struct i7core_dev *i7core_dev)
1891 {
1892         struct mem_ctl_info *mci = i7core_dev->mci;
1893         struct i7core_pvt *pvt;
1894
1895         if (unlikely(!mci || !mci->pvt_info)) {
1896                 debugf0("MC: " __FILE__ ": %s(): dev = %p\n",
1897                         __func__, &i7core_dev->pdev[0]->dev);
1898
1899                 i7core_printk(KERN_ERR, "Couldn't find mci handler\n");
1900                 return;
1901         }
1902
1903         pvt = mci->pvt_info;
1904
1905         debugf0("MC: " __FILE__ ": %s(): mci = %p, dev = %p\n",
1906                 __func__, mci, &i7core_dev->pdev[0]->dev);
1907
1908         /* Disable MCE NMI handler */
1909         edac_mce_unregister(&pvt->edac_mce);
1910
1911         /* Disable EDAC polling */
1912         i7core_pci_ctl_release(pvt);
1913
1914         /* Remove MC sysfs nodes */
1915         edac_mc_del_mc(mci->dev);
1916
1917         debugf1("%s: free mci struct\n", mci->ctl_name);
1918         kfree(mci->ctl_name);
1919         edac_mc_free(mci);
1920         i7core_dev->mci = NULL;
1921 }
1922
1923 static int i7core_register_mci(struct i7core_dev *i7core_dev)
1924 {
1925         struct mem_ctl_info *mci;
1926         struct i7core_pvt *pvt;
1927         int rc, channels, csrows;
1928
1929         /* Check the number of active and not disabled channels */
1930         rc = i7core_get_active_channels(i7core_dev->socket, &channels, &csrows);
1931         if (unlikely(rc < 0))
1932                 return rc;
1933
1934         /* allocate a new MC control structure */
1935         mci = edac_mc_alloc(sizeof(*pvt), csrows, channels, i7core_dev->socket);
1936         if (unlikely(!mci))
1937                 return -ENOMEM;
1938
1939         debugf0("MC: " __FILE__ ": %s(): mci = %p, dev = %p\n",
1940                 __func__, mci, &i7core_dev->pdev[0]->dev);
1941
1942         pvt = mci->pvt_info;
1943         memset(pvt, 0, sizeof(*pvt));
1944
1945         /* Associates i7core_dev and mci for future usage */
1946         pvt->i7core_dev = i7core_dev;
1947         i7core_dev->mci = mci;
1948
1949         /*
1950          * FIXME: how to handle RDDR3 at MCI level? It is possible to have
1951          * Mixed RDDR3/UDDR3 with Nehalem, provided that they are on different
1952          * memory channels
1953          */
1954         mci->mtype_cap = MEM_FLAG_DDR3;
1955         mci->edac_ctl_cap = EDAC_FLAG_NONE;
1956         mci->edac_cap = EDAC_FLAG_NONE;
1957         mci->mod_name = "i7core_edac.c";
1958         mci->mod_ver = I7CORE_REVISION;
1959         mci->ctl_name = kasprintf(GFP_KERNEL, "i7 core #%d",
1960                                   i7core_dev->socket);
1961         mci->dev_name = pci_name(i7core_dev->pdev[0]);
1962         mci->ctl_page_to_phys = NULL;
1963
1964         /* Store pci devices at mci for faster access */
1965         rc = mci_bind_devs(mci, i7core_dev);
1966         if (unlikely(rc < 0))
1967                 goto fail0;
1968
1969         if (pvt->is_registered)
1970                 mci->mc_driver_sysfs_attributes = i7core_sysfs_rdimm_attrs;
1971         else
1972                 mci->mc_driver_sysfs_attributes = i7core_sysfs_udimm_attrs;
1973
1974         /* Get dimm basic config */
1975         get_dimm_config(mci);
1976         /* record ptr to the generic device */
1977         mci->dev = &i7core_dev->pdev[0]->dev;
1978         /* Set the function pointer to an actual operation function */
1979         mci->edac_check = i7core_check_error;
1980
1981         /* add this new MC control structure to EDAC's list of MCs */
1982         if (unlikely(edac_mc_add_mc(mci))) {
1983                 debugf0("MC: " __FILE__
1984                         ": %s(): failed edac_mc_add_mc()\n", __func__);
1985                 /* FIXME: perhaps some code should go here that disables error
1986                  * reporting if we just enabled it
1987                  */
1988
1989                 rc = -EINVAL;
1990                 goto fail0;
1991         }
1992
1993         /* Default error mask is any memory */
1994         pvt->inject.channel = 0;
1995         pvt->inject.dimm = -1;
1996         pvt->inject.rank = -1;
1997         pvt->inject.bank = -1;
1998         pvt->inject.page = -1;
1999         pvt->inject.col = -1;
2000
2001         /* allocating generic PCI control info */
2002         i7core_pci_ctl_create(pvt);
2003
2004         /* Registers on edac_mce in order to receive memory errors */
2005         pvt->edac_mce.priv = mci;
2006         pvt->edac_mce.check_error = i7core_mce_check_error;
2007         rc = edac_mce_register(&pvt->edac_mce);
2008         if (unlikely(rc < 0)) {
2009                 debugf0("MC: " __FILE__
2010                         ": %s(): failed edac_mce_register()\n", __func__);
2011                 goto fail1;
2012         }
2013
2014         return 0;
2015
2016 fail1:
2017         i7core_pci_ctl_release(pvt);
2018         edac_mc_del_mc(mci->dev);
2019 fail0:
2020         kfree(mci->ctl_name);
2021         edac_mc_free(mci);
2022         i7core_dev->mci = NULL;
2023         return rc;
2024 }
2025
2026 /*
2027  *      i7core_probe    Probe for ONE instance of device to see if it is
2028  *                      present.
2029  *      return:
2030  *              0 for FOUND a device
2031  *              < 0 for error code
2032  */
2033
2034 static int __devinit i7core_probe(struct pci_dev *pdev,
2035                                   const struct pci_device_id *id)
2036 {
2037         int rc;
2038         struct i7core_dev *i7core_dev;
2039
2040         /* get the pci devices we want to reserve for our use */
2041         mutex_lock(&i7core_edac_lock);
2042
2043         /*
2044          * All memory controllers are allocated at the first pass.
2045          */
2046         if (unlikely(probed >= 1)) {
2047                 mutex_unlock(&i7core_edac_lock);
2048                 return -EINVAL;
2049         }
2050         probed++;
2051
2052         rc = i7core_get_all_devices();
2053         if (unlikely(rc < 0))
2054                 goto fail0;
2055
2056         list_for_each_entry(i7core_dev, &i7core_edac_list, list) {
2057                 rc = i7core_register_mci(i7core_dev);
2058                 if (unlikely(rc < 0))
2059                         goto fail1;
2060         }
2061
2062         i7core_printk(KERN_INFO, "Driver loaded.\n");
2063
2064         mutex_unlock(&i7core_edac_lock);
2065         return 0;
2066
2067 fail1:
2068         list_for_each_entry(i7core_dev, &i7core_edac_list, list)
2069                 i7core_unregister_mci(i7core_dev);
2070
2071         i7core_put_all_devices();
2072 fail0:
2073         mutex_unlock(&i7core_edac_lock);
2074         return rc;
2075 }
2076
2077 /*
2078  *      i7core_remove   destructor for one instance of device
2079  *
2080  */
2081 static void __devexit i7core_remove(struct pci_dev *pdev)
2082 {
2083         struct i7core_dev *i7core_dev;
2084
2085         debugf0(__FILE__ ": %s()\n", __func__);
2086
2087         /*
2088          * we have a trouble here: pdev value for removal will be wrong, since
2089          * it will point to the X58 register used to detect that the machine
2090          * is a Nehalem or upper design. However, due to the way several PCI
2091          * devices are grouped together to provide MC functionality, we need
2092          * to use a different method for releasing the devices
2093          */
2094
2095         mutex_lock(&i7core_edac_lock);
2096
2097         if (unlikely(!probed)) {
2098                 mutex_unlock(&i7core_edac_lock);
2099                 return;
2100         }
2101
2102         list_for_each_entry(i7core_dev, &i7core_edac_list, list)
2103                 i7core_unregister_mci(i7core_dev);
2104
2105         /* Release PCI resources */
2106         i7core_put_all_devices();
2107
2108         probed--;
2109
2110         mutex_unlock(&i7core_edac_lock);
2111 }
2112
2113 MODULE_DEVICE_TABLE(pci, i7core_pci_tbl);
2114
2115 /*
2116  *      i7core_driver   pci_driver structure for this module
2117  *
2118  */
2119 static struct pci_driver i7core_driver = {
2120         .name     = "i7core_edac",
2121         .probe    = i7core_probe,
2122         .remove   = __devexit_p(i7core_remove),
2123         .id_table = i7core_pci_tbl,
2124 };
2125
2126 /*
2127  *      i7core_init             Module entry function
2128  *                      Try to initialize this module for its devices
2129  */
2130 static int __init i7core_init(void)
2131 {
2132         int pci_rc;
2133
2134         debugf2("MC: " __FILE__ ": %s()\n", __func__);
2135
2136         /* Ensure that the OPSTATE is set correctly for POLL or NMI */
2137         opstate_init();
2138
2139         if (use_pci_fixup)
2140                 i7core_xeon_pci_fixup(pci_dev_table);
2141
2142         pci_rc = pci_register_driver(&i7core_driver);
2143
2144         if (pci_rc >= 0)
2145                 return 0;
2146
2147         i7core_printk(KERN_ERR, "Failed to register device with error %d.\n",
2148                       pci_rc);
2149
2150         return pci_rc;
2151 }
2152
2153 /*
2154  *      i7core_exit()   Module exit function
2155  *                      Unregister the driver
2156  */
2157 static void __exit i7core_exit(void)
2158 {
2159         debugf2("MC: " __FILE__ ": %s()\n", __func__);
2160         pci_unregister_driver(&i7core_driver);
2161 }
2162
2163 module_init(i7core_init);
2164 module_exit(i7core_exit);
2165
2166 MODULE_LICENSE("GPL");
2167 MODULE_AUTHOR("Mauro Carvalho Chehab <mchehab@redhat.com>");
2168 MODULE_AUTHOR("Red Hat Inc. (http://www.redhat.com)");
2169 MODULE_DESCRIPTION("MC Driver for Intel i7 Core memory controllers - "
2170                    I7CORE_REVISION);
2171
2172 module_param(edac_op_state, int, 0444);
2173 MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI");