]> git.karo-electronics.de Git - karo-tx-linux.git/blob - drivers/edac/edac_mc.c
Merge remote-tracking branch 'usb-chipidea-next/ci-for-usb-next'
[karo-tx-linux.git] / drivers / edac / edac_mc.c
1 /*
2  * edac_mc kernel module
3  * (C) 2005, 2006 Linux Networx (http://lnxi.com)
4  * This file may be distributed under the terms of the
5  * GNU General Public License.
6  *
7  * Written by Thayne Harbaugh
8  * Based on work by Dan Hollis <goemon at anime dot net> and others.
9  *      http://www.anime.net/~goemon/linux-ecc/
10  *
11  * Modified by Dave Peterson and Doug Thompson
12  *
13  */
14
15 #include <linux/module.h>
16 #include <linux/proc_fs.h>
17 #include <linux/kernel.h>
18 #include <linux/types.h>
19 #include <linux/smp.h>
20 #include <linux/init.h>
21 #include <linux/sysctl.h>
22 #include <linux/highmem.h>
23 #include <linux/timer.h>
24 #include <linux/slab.h>
25 #include <linux/jiffies.h>
26 #include <linux/spinlock.h>
27 #include <linux/list.h>
28 #include <linux/ctype.h>
29 #include <linux/edac.h>
30 #include <linux/bitops.h>
31 #include <asm/uaccess.h>
32 #include <asm/page.h>
33 #include "edac_core.h"
34 #include "edac_module.h"
35 #include <ras/ras_event.h>
36
37 #ifdef CONFIG_EDAC_ATOMIC_SCRUB
38 #include <asm/edac.h>
39 #else
40 #define edac_atomic_scrub(va, size) do { } while (0)
41 #endif
42
43 /* lock to memory controller's control array */
44 static DEFINE_MUTEX(mem_ctls_mutex);
45 static LIST_HEAD(mc_devices);
46
47 /*
48  * Used to lock EDAC MC to just one module, avoiding two drivers e. g.
49  *      apei/ghes and i7core_edac to be used at the same time.
50  */
51 static void const *edac_mc_owner;
52
53 static struct bus_type mc_bus[EDAC_MAX_MCS];
54
55 unsigned edac_dimm_info_location(struct dimm_info *dimm, char *buf,
56                                  unsigned len)
57 {
58         struct mem_ctl_info *mci = dimm->mci;
59         int i, n, count = 0;
60         char *p = buf;
61
62         for (i = 0; i < mci->n_layers; i++) {
63                 n = snprintf(p, len, "%s %d ",
64                               edac_layer_name[mci->layers[i].type],
65                               dimm->location[i]);
66                 p += n;
67                 len -= n;
68                 count += n;
69                 if (!len)
70                         break;
71         }
72
73         return count;
74 }
75
76 #ifdef CONFIG_EDAC_DEBUG
77
78 static void edac_mc_dump_channel(struct rank_info *chan)
79 {
80         edac_dbg(4, "  channel->chan_idx = %d\n", chan->chan_idx);
81         edac_dbg(4, "    channel = %p\n", chan);
82         edac_dbg(4, "    channel->csrow = %p\n", chan->csrow);
83         edac_dbg(4, "    channel->dimm = %p\n", chan->dimm);
84 }
85
86 static void edac_mc_dump_dimm(struct dimm_info *dimm, int number)
87 {
88         char location[80];
89
90         edac_dimm_info_location(dimm, location, sizeof(location));
91
92         edac_dbg(4, "%s%i: %smapped as virtual row %d, chan %d\n",
93                  dimm->mci->csbased ? "rank" : "dimm",
94                  number, location, dimm->csrow, dimm->cschannel);
95         edac_dbg(4, "  dimm = %p\n", dimm);
96         edac_dbg(4, "  dimm->label = '%s'\n", dimm->label);
97         edac_dbg(4, "  dimm->nr_pages = 0x%x\n", dimm->nr_pages);
98         edac_dbg(4, "  dimm->grain = %d\n", dimm->grain);
99         edac_dbg(4, "  dimm->nr_pages = 0x%x\n", dimm->nr_pages);
100 }
101
102 static void edac_mc_dump_csrow(struct csrow_info *csrow)
103 {
104         edac_dbg(4, "csrow->csrow_idx = %d\n", csrow->csrow_idx);
105         edac_dbg(4, "  csrow = %p\n", csrow);
106         edac_dbg(4, "  csrow->first_page = 0x%lx\n", csrow->first_page);
107         edac_dbg(4, "  csrow->last_page = 0x%lx\n", csrow->last_page);
108         edac_dbg(4, "  csrow->page_mask = 0x%lx\n", csrow->page_mask);
109         edac_dbg(4, "  csrow->nr_channels = %d\n", csrow->nr_channels);
110         edac_dbg(4, "  csrow->channels = %p\n", csrow->channels);
111         edac_dbg(4, "  csrow->mci = %p\n", csrow->mci);
112 }
113
114 static void edac_mc_dump_mci(struct mem_ctl_info *mci)
115 {
116         edac_dbg(3, "\tmci = %p\n", mci);
117         edac_dbg(3, "\tmci->mtype_cap = %lx\n", mci->mtype_cap);
118         edac_dbg(3, "\tmci->edac_ctl_cap = %lx\n", mci->edac_ctl_cap);
119         edac_dbg(3, "\tmci->edac_cap = %lx\n", mci->edac_cap);
120         edac_dbg(4, "\tmci->edac_check = %p\n", mci->edac_check);
121         edac_dbg(3, "\tmci->nr_csrows = %d, csrows = %p\n",
122                  mci->nr_csrows, mci->csrows);
123         edac_dbg(3, "\tmci->nr_dimms = %d, dimms = %p\n",
124                  mci->tot_dimms, mci->dimms);
125         edac_dbg(3, "\tdev = %p\n", mci->pdev);
126         edac_dbg(3, "\tmod_name:ctl_name = %s:%s\n",
127                  mci->mod_name, mci->ctl_name);
128         edac_dbg(3, "\tpvt_info = %p\n\n", mci->pvt_info);
129 }
130
131 #endif                          /* CONFIG_EDAC_DEBUG */
132
133 const char * const edac_mem_types[] = {
134         [MEM_EMPTY]     = "Empty csrow",
135         [MEM_RESERVED]  = "Reserved csrow type",
136         [MEM_UNKNOWN]   = "Unknown csrow type",
137         [MEM_FPM]       = "Fast page mode RAM",
138         [MEM_EDO]       = "Extended data out RAM",
139         [MEM_BEDO]      = "Burst Extended data out RAM",
140         [MEM_SDR]       = "Single data rate SDRAM",
141         [MEM_RDR]       = "Registered single data rate SDRAM",
142         [MEM_DDR]       = "Double data rate SDRAM",
143         [MEM_RDDR]      = "Registered Double data rate SDRAM",
144         [MEM_RMBS]      = "Rambus DRAM",
145         [MEM_DDR2]      = "Unbuffered DDR2 RAM",
146         [MEM_FB_DDR2]   = "Fully buffered DDR2",
147         [MEM_RDDR2]     = "Registered DDR2 RAM",
148         [MEM_XDR]       = "Rambus XDR",
149         [MEM_DDR3]      = "Unbuffered DDR3 RAM",
150         [MEM_RDDR3]     = "Registered DDR3 RAM",
151         [MEM_LRDDR3]    = "Load-Reduced DDR3 RAM",
152         [MEM_DDR4]      = "Unbuffered DDR4 RAM",
153         [MEM_RDDR4]     = "Registered DDR4 RAM",
154 };
155 EXPORT_SYMBOL_GPL(edac_mem_types);
156
157 /**
158  * edac_align_ptr - Prepares the pointer offsets for a single-shot allocation
159  * @p:          pointer to a pointer with the memory offset to be used. At
160  *              return, this will be incremented to point to the next offset
161  * @size:       Size of the data structure to be reserved
162  * @n_elems:    Number of elements that should be reserved
163  *
164  * If 'size' is a constant, the compiler will optimize this whole function
165  * down to either a no-op or the addition of a constant to the value of '*p'.
166  *
167  * The 'p' pointer is absolutely needed to keep the proper advancing
168  * further in memory to the proper offsets when allocating the struct along
169  * with its embedded structs, as edac_device_alloc_ctl_info() does it
170  * above, for example.
171  *
172  * At return, the pointer 'p' will be incremented to be used on a next call
173  * to this function.
174  */
175 void *edac_align_ptr(void **p, unsigned size, int n_elems)
176 {
177         unsigned align, r;
178         void *ptr = *p;
179
180         *p += size * n_elems;
181
182         /*
183          * 'p' can possibly be an unaligned item X such that sizeof(X) is
184          * 'size'.  Adjust 'p' so that its alignment is at least as
185          * stringent as what the compiler would provide for X and return
186          * the aligned result.
187          * Here we assume that the alignment of a "long long" is the most
188          * stringent alignment that the compiler will ever provide by default.
189          * As far as I know, this is a reasonable assumption.
190          */
191         if (size > sizeof(long))
192                 align = sizeof(long long);
193         else if (size > sizeof(int))
194                 align = sizeof(long);
195         else if (size > sizeof(short))
196                 align = sizeof(int);
197         else if (size > sizeof(char))
198                 align = sizeof(short);
199         else
200                 return (char *)ptr;
201
202         r = (unsigned long)p % align;
203
204         if (r == 0)
205                 return (char *)ptr;
206
207         *p += align - r;
208
209         return (void *)(((unsigned long)ptr) + align - r);
210 }
211
212 static void _edac_mc_free(struct mem_ctl_info *mci)
213 {
214         int i, chn, row;
215         struct csrow_info *csr;
216         const unsigned int tot_dimms = mci->tot_dimms;
217         const unsigned int tot_channels = mci->num_cschannel;
218         const unsigned int tot_csrows = mci->nr_csrows;
219
220         if (mci->dimms) {
221                 for (i = 0; i < tot_dimms; i++)
222                         kfree(mci->dimms[i]);
223                 kfree(mci->dimms);
224         }
225         if (mci->csrows) {
226                 for (row = 0; row < tot_csrows; row++) {
227                         csr = mci->csrows[row];
228                         if (csr) {
229                                 if (csr->channels) {
230                                         for (chn = 0; chn < tot_channels; chn++)
231                                                 kfree(csr->channels[chn]);
232                                         kfree(csr->channels);
233                                 }
234                                 kfree(csr);
235                         }
236                 }
237                 kfree(mci->csrows);
238         }
239         kfree(mci);
240 }
241
242 /**
243  * edac_mc_alloc: Allocate and partially fill a struct mem_ctl_info structure
244  * @mc_num:             Memory controller number
245  * @n_layers:           Number of MC hierarchy layers
246  * layers:              Describes each layer as seen by the Memory Controller
247  * @size_pvt:           size of private storage needed
248  *
249  *
250  * Everything is kmalloc'ed as one big chunk - more efficient.
251  * Only can be used if all structures have the same lifetime - otherwise
252  * you have to allocate and initialize your own structures.
253  *
254  * Use edac_mc_free() to free mc structures allocated by this function.
255  *
256  * NOTE: drivers handle multi-rank memories in different ways: in some
257  * drivers, one multi-rank memory stick is mapped as one entry, while, in
258  * others, a single multi-rank memory stick would be mapped into several
259  * entries. Currently, this function will allocate multiple struct dimm_info
260  * on such scenarios, as grouping the multiple ranks require drivers change.
261  *
262  * Returns:
263  *      On failure: NULL
264  *      On success: struct mem_ctl_info pointer
265  */
266 struct mem_ctl_info *edac_mc_alloc(unsigned mc_num,
267                                    unsigned n_layers,
268                                    struct edac_mc_layer *layers,
269                                    unsigned sz_pvt)
270 {
271         struct mem_ctl_info *mci;
272         struct edac_mc_layer *layer;
273         struct csrow_info *csr;
274         struct rank_info *chan;
275         struct dimm_info *dimm;
276         u32 *ce_per_layer[EDAC_MAX_LAYERS], *ue_per_layer[EDAC_MAX_LAYERS];
277         unsigned pos[EDAC_MAX_LAYERS];
278         unsigned size, tot_dimms = 1, count = 1;
279         unsigned tot_csrows = 1, tot_channels = 1, tot_errcount = 0;
280         void *pvt, *p, *ptr = NULL;
281         int i, j, row, chn, n, len, off;
282         bool per_rank = false;
283
284         BUG_ON(n_layers > EDAC_MAX_LAYERS || n_layers == 0);
285         /*
286          * Calculate the total amount of dimms and csrows/cschannels while
287          * in the old API emulation mode
288          */
289         for (i = 0; i < n_layers; i++) {
290                 tot_dimms *= layers[i].size;
291                 if (layers[i].is_virt_csrow)
292                         tot_csrows *= layers[i].size;
293                 else
294                         tot_channels *= layers[i].size;
295
296                 if (layers[i].type == EDAC_MC_LAYER_CHIP_SELECT)
297                         per_rank = true;
298         }
299
300         /* Figure out the offsets of the various items from the start of an mc
301          * structure.  We want the alignment of each item to be at least as
302          * stringent as what the compiler would provide if we could simply
303          * hardcode everything into a single struct.
304          */
305         mci = edac_align_ptr(&ptr, sizeof(*mci), 1);
306         layer = edac_align_ptr(&ptr, sizeof(*layer), n_layers);
307         for (i = 0; i < n_layers; i++) {
308                 count *= layers[i].size;
309                 edac_dbg(4, "errcount layer %d size %d\n", i, count);
310                 ce_per_layer[i] = edac_align_ptr(&ptr, sizeof(u32), count);
311                 ue_per_layer[i] = edac_align_ptr(&ptr, sizeof(u32), count);
312                 tot_errcount += 2 * count;
313         }
314
315         edac_dbg(4, "allocating %d error counters\n", tot_errcount);
316         pvt = edac_align_ptr(&ptr, sz_pvt, 1);
317         size = ((unsigned long)pvt) + sz_pvt;
318
319         edac_dbg(1, "allocating %u bytes for mci data (%d %s, %d csrows/channels)\n",
320                  size,
321                  tot_dimms,
322                  per_rank ? "ranks" : "dimms",
323                  tot_csrows * tot_channels);
324
325         mci = kzalloc(size, GFP_KERNEL);
326         if (mci == NULL)
327                 return NULL;
328
329         /* Adjust pointers so they point within the memory we just allocated
330          * rather than an imaginary chunk of memory located at address 0.
331          */
332         layer = (struct edac_mc_layer *)(((char *)mci) + ((unsigned long)layer));
333         for (i = 0; i < n_layers; i++) {
334                 mci->ce_per_layer[i] = (u32 *)((char *)mci + ((unsigned long)ce_per_layer[i]));
335                 mci->ue_per_layer[i] = (u32 *)((char *)mci + ((unsigned long)ue_per_layer[i]));
336         }
337         pvt = sz_pvt ? (((char *)mci) + ((unsigned long)pvt)) : NULL;
338
339         /* setup index and various internal pointers */
340         mci->mc_idx = mc_num;
341         mci->tot_dimms = tot_dimms;
342         mci->pvt_info = pvt;
343         mci->n_layers = n_layers;
344         mci->layers = layer;
345         memcpy(mci->layers, layers, sizeof(*layer) * n_layers);
346         mci->nr_csrows = tot_csrows;
347         mci->num_cschannel = tot_channels;
348         mci->csbased = per_rank;
349
350         /*
351          * Alocate and fill the csrow/channels structs
352          */
353         mci->csrows = kcalloc(tot_csrows, sizeof(*mci->csrows), GFP_KERNEL);
354         if (!mci->csrows)
355                 goto error;
356         for (row = 0; row < tot_csrows; row++) {
357                 csr = kzalloc(sizeof(**mci->csrows), GFP_KERNEL);
358                 if (!csr)
359                         goto error;
360                 mci->csrows[row] = csr;
361                 csr->csrow_idx = row;
362                 csr->mci = mci;
363                 csr->nr_channels = tot_channels;
364                 csr->channels = kcalloc(tot_channels, sizeof(*csr->channels),
365                                         GFP_KERNEL);
366                 if (!csr->channels)
367                         goto error;
368
369                 for (chn = 0; chn < tot_channels; chn++) {
370                         chan = kzalloc(sizeof(**csr->channels), GFP_KERNEL);
371                         if (!chan)
372                                 goto error;
373                         csr->channels[chn] = chan;
374                         chan->chan_idx = chn;
375                         chan->csrow = csr;
376                 }
377         }
378
379         /*
380          * Allocate and fill the dimm structs
381          */
382         mci->dimms  = kcalloc(tot_dimms, sizeof(*mci->dimms), GFP_KERNEL);
383         if (!mci->dimms)
384                 goto error;
385
386         memset(&pos, 0, sizeof(pos));
387         row = 0;
388         chn = 0;
389         for (i = 0; i < tot_dimms; i++) {
390                 chan = mci->csrows[row]->channels[chn];
391                 off = EDAC_DIMM_OFF(layer, n_layers, pos[0], pos[1], pos[2]);
392                 if (off < 0 || off >= tot_dimms) {
393                         edac_mc_printk(mci, KERN_ERR, "EDAC core bug: EDAC_DIMM_OFF is trying to do an illegal data access\n");
394                         goto error;
395                 }
396
397                 dimm = kzalloc(sizeof(**mci->dimms), GFP_KERNEL);
398                 if (!dimm)
399                         goto error;
400                 mci->dimms[off] = dimm;
401                 dimm->mci = mci;
402
403                 /*
404                  * Copy DIMM location and initialize it.
405                  */
406                 len = sizeof(dimm->label);
407                 p = dimm->label;
408                 n = snprintf(p, len, "mc#%u", mc_num);
409                 p += n;
410                 len -= n;
411                 for (j = 0; j < n_layers; j++) {
412                         n = snprintf(p, len, "%s#%u",
413                                      edac_layer_name[layers[j].type],
414                                      pos[j]);
415                         p += n;
416                         len -= n;
417                         dimm->location[j] = pos[j];
418
419                         if (len <= 0)
420                                 break;
421                 }
422
423                 /* Link it to the csrows old API data */
424                 chan->dimm = dimm;
425                 dimm->csrow = row;
426                 dimm->cschannel = chn;
427
428                 /* Increment csrow location */
429                 if (layers[0].is_virt_csrow) {
430                         chn++;
431                         if (chn == tot_channels) {
432                                 chn = 0;
433                                 row++;
434                         }
435                 } else {
436                         row++;
437                         if (row == tot_csrows) {
438                                 row = 0;
439                                 chn++;
440                         }
441                 }
442
443                 /* Increment dimm location */
444                 for (j = n_layers - 1; j >= 0; j--) {
445                         pos[j]++;
446                         if (pos[j] < layers[j].size)
447                                 break;
448                         pos[j] = 0;
449                 }
450         }
451
452         mci->op_state = OP_ALLOC;
453
454         return mci;
455
456 error:
457         _edac_mc_free(mci);
458
459         return NULL;
460 }
461 EXPORT_SYMBOL_GPL(edac_mc_alloc);
462
463 /**
464  * edac_mc_free
465  *      'Free' a previously allocated 'mci' structure
466  * @mci: pointer to a struct mem_ctl_info structure
467  */
468 void edac_mc_free(struct mem_ctl_info *mci)
469 {
470         edac_dbg(1, "\n");
471
472         /* If we're not yet registered with sysfs free only what was allocated
473          * in edac_mc_alloc().
474          */
475         if (!device_is_registered(&mci->dev)) {
476                 _edac_mc_free(mci);
477                 return;
478         }
479
480         /* the mci instance is freed here, when the sysfs object is dropped */
481         edac_unregister_sysfs(mci);
482 }
483 EXPORT_SYMBOL_GPL(edac_mc_free);
484
485
486 /**
487  * find_mci_by_dev
488  *
489  *      scan list of controllers looking for the one that manages
490  *      the 'dev' device
491  * @dev: pointer to a struct device related with the MCI
492  */
493 struct mem_ctl_info *find_mci_by_dev(struct device *dev)
494 {
495         struct mem_ctl_info *mci;
496         struct list_head *item;
497
498         edac_dbg(3, "\n");
499
500         list_for_each(item, &mc_devices) {
501                 mci = list_entry(item, struct mem_ctl_info, link);
502
503                 if (mci->pdev == dev)
504                         return mci;
505         }
506
507         return NULL;
508 }
509 EXPORT_SYMBOL_GPL(find_mci_by_dev);
510
511 /*
512  * handler for EDAC to check if NMI type handler has asserted interrupt
513  */
514 static int edac_mc_assert_error_check_and_clear(void)
515 {
516         int old_state;
517
518         if (edac_op_state == EDAC_OPSTATE_POLL)
519                 return 1;
520
521         old_state = edac_err_assert;
522         edac_err_assert = 0;
523
524         return old_state;
525 }
526
527 /*
528  * edac_mc_workq_function
529  *      performs the operation scheduled by a workq request
530  */
531 static void edac_mc_workq_function(struct work_struct *work_req)
532 {
533         struct delayed_work *d_work = to_delayed_work(work_req);
534         struct mem_ctl_info *mci = to_edac_mem_ctl_work(d_work);
535
536         mutex_lock(&mem_ctls_mutex);
537
538         if (mci->op_state != OP_RUNNING_POLL) {
539                 mutex_unlock(&mem_ctls_mutex);
540                 return;
541         }
542
543         if (edac_mc_assert_error_check_and_clear())
544                 mci->edac_check(mci);
545
546         mutex_unlock(&mem_ctls_mutex);
547
548         /* Queue ourselves again. */
549         edac_queue_work(&mci->work, msecs_to_jiffies(edac_mc_get_poll_msec()));
550 }
551
552 /*
553  * edac_mc_reset_delay_period(unsigned long value)
554  *
555  *      user space has updated our poll period value, need to
556  *      reset our workq delays
557  */
558 void edac_mc_reset_delay_period(unsigned long value)
559 {
560         struct mem_ctl_info *mci;
561         struct list_head *item;
562
563         mutex_lock(&mem_ctls_mutex);
564
565         list_for_each(item, &mc_devices) {
566                 mci = list_entry(item, struct mem_ctl_info, link);
567
568                 edac_mod_work(&mci->work, value);
569         }
570         mutex_unlock(&mem_ctls_mutex);
571 }
572
573
574
575 /* Return 0 on success, 1 on failure.
576  * Before calling this function, caller must
577  * assign a unique value to mci->mc_idx.
578  *
579  *      locking model:
580  *
581  *              called with the mem_ctls_mutex lock held
582  */
583 static int add_mc_to_global_list(struct mem_ctl_info *mci)
584 {
585         struct list_head *item, *insert_before;
586         struct mem_ctl_info *p;
587
588         insert_before = &mc_devices;
589
590         p = find_mci_by_dev(mci->pdev);
591         if (unlikely(p != NULL))
592                 goto fail0;
593
594         list_for_each(item, &mc_devices) {
595                 p = list_entry(item, struct mem_ctl_info, link);
596
597                 if (p->mc_idx >= mci->mc_idx) {
598                         if (unlikely(p->mc_idx == mci->mc_idx))
599                                 goto fail1;
600
601                         insert_before = item;
602                         break;
603                 }
604         }
605
606         list_add_tail_rcu(&mci->link, insert_before);
607         atomic_inc(&edac_handlers);
608         return 0;
609
610 fail0:
611         edac_printk(KERN_WARNING, EDAC_MC,
612                 "%s (%s) %s %s already assigned %d\n", dev_name(p->pdev),
613                 edac_dev_name(mci), p->mod_name, p->ctl_name, p->mc_idx);
614         return 1;
615
616 fail1:
617         edac_printk(KERN_WARNING, EDAC_MC,
618                 "bug in low-level driver: attempt to assign\n"
619                 "    duplicate mc_idx %d in %s()\n", p->mc_idx, __func__);
620         return 1;
621 }
622
623 static int del_mc_from_global_list(struct mem_ctl_info *mci)
624 {
625         int handlers = atomic_dec_return(&edac_handlers);
626         list_del_rcu(&mci->link);
627
628         /* these are for safe removal of devices from global list while
629          * NMI handlers may be traversing list
630          */
631         synchronize_rcu();
632         INIT_LIST_HEAD(&mci->link);
633
634         return handlers;
635 }
636
637 /**
638  * edac_mc_find: Search for a mem_ctl_info structure whose index is 'idx'.
639  *
640  * If found, return a pointer to the structure.
641  * Else return NULL.
642  *
643  * Caller must hold mem_ctls_mutex.
644  */
645 struct mem_ctl_info *edac_mc_find(int idx)
646 {
647         struct list_head *item;
648         struct mem_ctl_info *mci;
649
650         list_for_each(item, &mc_devices) {
651                 mci = list_entry(item, struct mem_ctl_info, link);
652
653                 if (mci->mc_idx >= idx) {
654                         if (mci->mc_idx == idx)
655                                 return mci;
656
657                         break;
658                 }
659         }
660
661         return NULL;
662 }
663 EXPORT_SYMBOL(edac_mc_find);
664
665 /**
666  * edac_mc_add_mc_with_groups: Insert the 'mci' structure into the mci
667  *      global list and create sysfs entries associated with mci structure
668  * @mci: pointer to the mci structure to be added to the list
669  * @groups: optional attribute groups for the driver-specific sysfs entries
670  *
671  * Return:
672  *      0       Success
673  *      !0      Failure
674  */
675
676 /* FIXME - should a warning be printed if no error detection? correction? */
677 int edac_mc_add_mc_with_groups(struct mem_ctl_info *mci,
678                                const struct attribute_group **groups)
679 {
680         int ret = -EINVAL;
681         edac_dbg(0, "\n");
682
683         if (mci->mc_idx >= EDAC_MAX_MCS) {
684                 pr_warn_once("Too many memory controllers: %d\n", mci->mc_idx);
685                 return -ENODEV;
686         }
687
688 #ifdef CONFIG_EDAC_DEBUG
689         if (edac_debug_level >= 3)
690                 edac_mc_dump_mci(mci);
691
692         if (edac_debug_level >= 4) {
693                 int i;
694
695                 for (i = 0; i < mci->nr_csrows; i++) {
696                         struct csrow_info *csrow = mci->csrows[i];
697                         u32 nr_pages = 0;
698                         int j;
699
700                         for (j = 0; j < csrow->nr_channels; j++)
701                                 nr_pages += csrow->channels[j]->dimm->nr_pages;
702                         if (!nr_pages)
703                                 continue;
704                         edac_mc_dump_csrow(csrow);
705                         for (j = 0; j < csrow->nr_channels; j++)
706                                 if (csrow->channels[j]->dimm->nr_pages)
707                                         edac_mc_dump_channel(csrow->channels[j]);
708                 }
709                 for (i = 0; i < mci->tot_dimms; i++)
710                         if (mci->dimms[i]->nr_pages)
711                                 edac_mc_dump_dimm(mci->dimms[i], i);
712         }
713 #endif
714         mutex_lock(&mem_ctls_mutex);
715
716         if (edac_mc_owner && edac_mc_owner != mci->mod_name) {
717                 ret = -EPERM;
718                 goto fail0;
719         }
720
721         if (add_mc_to_global_list(mci))
722                 goto fail0;
723
724         /* set load time so that error rate can be tracked */
725         mci->start_time = jiffies;
726
727         mci->bus = &mc_bus[mci->mc_idx];
728
729         if (edac_create_sysfs_mci_device(mci, groups)) {
730                 edac_mc_printk(mci, KERN_WARNING,
731                         "failed to create sysfs device\n");
732                 goto fail1;
733         }
734
735         if (mci->edac_check) {
736                 mci->op_state = OP_RUNNING_POLL;
737
738                 INIT_DELAYED_WORK(&mci->work, edac_mc_workq_function);
739                 edac_queue_work(&mci->work, msecs_to_jiffies(edac_mc_get_poll_msec()));
740
741         } else {
742                 mci->op_state = OP_RUNNING_INTERRUPT;
743         }
744
745         /* Report action taken */
746         edac_mc_printk(mci, KERN_INFO,
747                 "Giving out device to module %s controller %s: DEV %s (%s)\n",
748                 mci->mod_name, mci->ctl_name, mci->dev_name,
749                 edac_op_state_to_string(mci->op_state));
750
751         edac_mc_owner = mci->mod_name;
752
753         mutex_unlock(&mem_ctls_mutex);
754         return 0;
755
756 fail1:
757         del_mc_from_global_list(mci);
758
759 fail0:
760         mutex_unlock(&mem_ctls_mutex);
761         return ret;
762 }
763 EXPORT_SYMBOL_GPL(edac_mc_add_mc_with_groups);
764
765 /**
766  * edac_mc_del_mc: Remove sysfs entries for specified mci structure and
767  *                 remove mci structure from global list
768  * @pdev: Pointer to 'struct device' representing mci structure to remove.
769  *
770  * Return pointer to removed mci structure, or NULL if device not found.
771  */
772 struct mem_ctl_info *edac_mc_del_mc(struct device *dev)
773 {
774         struct mem_ctl_info *mci;
775
776         edac_dbg(0, "\n");
777
778         mutex_lock(&mem_ctls_mutex);
779
780         /* find the requested mci struct in the global list */
781         mci = find_mci_by_dev(dev);
782         if (mci == NULL) {
783                 mutex_unlock(&mem_ctls_mutex);
784                 return NULL;
785         }
786
787         /* mark MCI offline: */
788         mci->op_state = OP_OFFLINE;
789
790         if (!del_mc_from_global_list(mci))
791                 edac_mc_owner = NULL;
792
793         mutex_unlock(&mem_ctls_mutex);
794
795         if (mci->edac_check)
796                 edac_stop_work(&mci->work);
797
798         /* remove from sysfs */
799         edac_remove_sysfs_mci_device(mci);
800
801         edac_printk(KERN_INFO, EDAC_MC,
802                 "Removed device %d for %s %s: DEV %s\n", mci->mc_idx,
803                 mci->mod_name, mci->ctl_name, edac_dev_name(mci));
804
805         return mci;
806 }
807 EXPORT_SYMBOL_GPL(edac_mc_del_mc);
808
809 static void edac_mc_scrub_block(unsigned long page, unsigned long offset,
810                                 u32 size)
811 {
812         struct page *pg;
813         void *virt_addr;
814         unsigned long flags = 0;
815
816         edac_dbg(3, "\n");
817
818         /* ECC error page was not in our memory. Ignore it. */
819         if (!pfn_valid(page))
820                 return;
821
822         /* Find the actual page structure then map it and fix */
823         pg = pfn_to_page(page);
824
825         if (PageHighMem(pg))
826                 local_irq_save(flags);
827
828         virt_addr = kmap_atomic(pg);
829
830         /* Perform architecture specific atomic scrub operation */
831         edac_atomic_scrub(virt_addr + offset, size);
832
833         /* Unmap and complete */
834         kunmap_atomic(virt_addr);
835
836         if (PageHighMem(pg))
837                 local_irq_restore(flags);
838 }
839
840 /* FIXME - should return -1 */
841 int edac_mc_find_csrow_by_page(struct mem_ctl_info *mci, unsigned long page)
842 {
843         struct csrow_info **csrows = mci->csrows;
844         int row, i, j, n;
845
846         edac_dbg(1, "MC%d: 0x%lx\n", mci->mc_idx, page);
847         row = -1;
848
849         for (i = 0; i < mci->nr_csrows; i++) {
850                 struct csrow_info *csrow = csrows[i];
851                 n = 0;
852                 for (j = 0; j < csrow->nr_channels; j++) {
853                         struct dimm_info *dimm = csrow->channels[j]->dimm;
854                         n += dimm->nr_pages;
855                 }
856                 if (n == 0)
857                         continue;
858
859                 edac_dbg(3, "MC%d: first(0x%lx) page(0x%lx) last(0x%lx) mask(0x%lx)\n",
860                          mci->mc_idx,
861                          csrow->first_page, page, csrow->last_page,
862                          csrow->page_mask);
863
864                 if ((page >= csrow->first_page) &&
865                     (page <= csrow->last_page) &&
866                     ((page & csrow->page_mask) ==
867                      (csrow->first_page & csrow->page_mask))) {
868                         row = i;
869                         break;
870                 }
871         }
872
873         if (row == -1)
874                 edac_mc_printk(mci, KERN_ERR,
875                         "could not look up page error address %lx\n",
876                         (unsigned long)page);
877
878         return row;
879 }
880 EXPORT_SYMBOL_GPL(edac_mc_find_csrow_by_page);
881
882 const char *edac_layer_name[] = {
883         [EDAC_MC_LAYER_BRANCH] = "branch",
884         [EDAC_MC_LAYER_CHANNEL] = "channel",
885         [EDAC_MC_LAYER_SLOT] = "slot",
886         [EDAC_MC_LAYER_CHIP_SELECT] = "csrow",
887         [EDAC_MC_LAYER_ALL_MEM] = "memory",
888 };
889 EXPORT_SYMBOL_GPL(edac_layer_name);
890
891 static void edac_inc_ce_error(struct mem_ctl_info *mci,
892                               bool enable_per_layer_report,
893                               const int pos[EDAC_MAX_LAYERS],
894                               const u16 count)
895 {
896         int i, index = 0;
897
898         mci->ce_mc += count;
899
900         if (!enable_per_layer_report) {
901                 mci->ce_noinfo_count += count;
902                 return;
903         }
904
905         for (i = 0; i < mci->n_layers; i++) {
906                 if (pos[i] < 0)
907                         break;
908                 index += pos[i];
909                 mci->ce_per_layer[i][index] += count;
910
911                 if (i < mci->n_layers - 1)
912                         index *= mci->layers[i + 1].size;
913         }
914 }
915
916 static void edac_inc_ue_error(struct mem_ctl_info *mci,
917                                     bool enable_per_layer_report,
918                                     const int pos[EDAC_MAX_LAYERS],
919                                     const u16 count)
920 {
921         int i, index = 0;
922
923         mci->ue_mc += count;
924
925         if (!enable_per_layer_report) {
926                 mci->ce_noinfo_count += count;
927                 return;
928         }
929
930         for (i = 0; i < mci->n_layers; i++) {
931                 if (pos[i] < 0)
932                         break;
933                 index += pos[i];
934                 mci->ue_per_layer[i][index] += count;
935
936                 if (i < mci->n_layers - 1)
937                         index *= mci->layers[i + 1].size;
938         }
939 }
940
941 static void edac_ce_error(struct mem_ctl_info *mci,
942                           const u16 error_count,
943                           const int pos[EDAC_MAX_LAYERS],
944                           const char *msg,
945                           const char *location,
946                           const char *label,
947                           const char *detail,
948                           const char *other_detail,
949                           const bool enable_per_layer_report,
950                           const unsigned long page_frame_number,
951                           const unsigned long offset_in_page,
952                           long grain)
953 {
954         unsigned long remapped_page;
955         char *msg_aux = "";
956
957         if (*msg)
958                 msg_aux = " ";
959
960         if (edac_mc_get_log_ce()) {
961                 if (other_detail && *other_detail)
962                         edac_mc_printk(mci, KERN_WARNING,
963                                        "%d CE %s%son %s (%s %s - %s)\n",
964                                        error_count, msg, msg_aux, label,
965                                        location, detail, other_detail);
966                 else
967                         edac_mc_printk(mci, KERN_WARNING,
968                                        "%d CE %s%son %s (%s %s)\n",
969                                        error_count, msg, msg_aux, label,
970                                        location, detail);
971         }
972         edac_inc_ce_error(mci, enable_per_layer_report, pos, error_count);
973
974         if (mci->scrub_mode == SCRUB_SW_SRC) {
975                 /*
976                         * Some memory controllers (called MCs below) can remap
977                         * memory so that it is still available at a different
978                         * address when PCI devices map into memory.
979                         * MC's that can't do this, lose the memory where PCI
980                         * devices are mapped. This mapping is MC-dependent
981                         * and so we call back into the MC driver for it to
982                         * map the MC page to a physical (CPU) page which can
983                         * then be mapped to a virtual page - which can then
984                         * be scrubbed.
985                         */
986                 remapped_page = mci->ctl_page_to_phys ?
987                         mci->ctl_page_to_phys(mci, page_frame_number) :
988                         page_frame_number;
989
990                 edac_mc_scrub_block(remapped_page,
991                                         offset_in_page, grain);
992         }
993 }
994
995 static void edac_ue_error(struct mem_ctl_info *mci,
996                           const u16 error_count,
997                           const int pos[EDAC_MAX_LAYERS],
998                           const char *msg,
999                           const char *location,
1000                           const char *label,
1001                           const char *detail,
1002                           const char *other_detail,
1003                           const bool enable_per_layer_report)
1004 {
1005         char *msg_aux = "";
1006
1007         if (*msg)
1008                 msg_aux = " ";
1009
1010         if (edac_mc_get_log_ue()) {
1011                 if (other_detail && *other_detail)
1012                         edac_mc_printk(mci, KERN_WARNING,
1013                                        "%d UE %s%son %s (%s %s - %s)\n",
1014                                        error_count, msg, msg_aux, label,
1015                                        location, detail, other_detail);
1016                 else
1017                         edac_mc_printk(mci, KERN_WARNING,
1018                                        "%d UE %s%son %s (%s %s)\n",
1019                                        error_count, msg, msg_aux, label,
1020                                        location, detail);
1021         }
1022
1023         if (edac_mc_get_panic_on_ue()) {
1024                 if (other_detail && *other_detail)
1025                         panic("UE %s%son %s (%s%s - %s)\n",
1026                               msg, msg_aux, label, location, detail, other_detail);
1027                 else
1028                         panic("UE %s%son %s (%s%s)\n",
1029                               msg, msg_aux, label, location, detail);
1030         }
1031
1032         edac_inc_ue_error(mci, enable_per_layer_report, pos, error_count);
1033 }
1034
1035 /**
1036  * edac_raw_mc_handle_error - reports a memory event to userspace without doing
1037  *                            anything to discover the error location
1038  *
1039  * @type:               severity of the error (CE/UE/Fatal)
1040  * @mci:                a struct mem_ctl_info pointer
1041  * @e:                  error description
1042  *
1043  * This raw function is used internally by edac_mc_handle_error(). It should
1044  * only be called directly when the hardware error come directly from BIOS,
1045  * like in the case of APEI GHES driver.
1046  */
1047 void edac_raw_mc_handle_error(const enum hw_event_mc_err_type type,
1048                               struct mem_ctl_info *mci,
1049                               struct edac_raw_error_desc *e)
1050 {
1051         char detail[80];
1052         int pos[EDAC_MAX_LAYERS] = { e->top_layer, e->mid_layer, e->low_layer };
1053
1054         /* Memory type dependent details about the error */
1055         if (type == HW_EVENT_ERR_CORRECTED) {
1056                 snprintf(detail, sizeof(detail),
1057                         "page:0x%lx offset:0x%lx grain:%ld syndrome:0x%lx",
1058                         e->page_frame_number, e->offset_in_page,
1059                         e->grain, e->syndrome);
1060                 edac_ce_error(mci, e->error_count, pos, e->msg, e->location, e->label,
1061                               detail, e->other_detail, e->enable_per_layer_report,
1062                               e->page_frame_number, e->offset_in_page, e->grain);
1063         } else {
1064                 snprintf(detail, sizeof(detail),
1065                         "page:0x%lx offset:0x%lx grain:%ld",
1066                         e->page_frame_number, e->offset_in_page, e->grain);
1067
1068                 edac_ue_error(mci, e->error_count, pos, e->msg, e->location, e->label,
1069                               detail, e->other_detail, e->enable_per_layer_report);
1070         }
1071
1072
1073 }
1074 EXPORT_SYMBOL_GPL(edac_raw_mc_handle_error);
1075
1076 /**
1077  * edac_mc_handle_error - reports a memory event to userspace
1078  *
1079  * @type:               severity of the error (CE/UE/Fatal)
1080  * @mci:                a struct mem_ctl_info pointer
1081  * @error_count:        Number of errors of the same type
1082  * @page_frame_number:  mem page where the error occurred
1083  * @offset_in_page:     offset of the error inside the page
1084  * @syndrome:           ECC syndrome
1085  * @top_layer:          Memory layer[0] position
1086  * @mid_layer:          Memory layer[1] position
1087  * @low_layer:          Memory layer[2] position
1088  * @msg:                Message meaningful to the end users that
1089  *                      explains the event
1090  * @other_detail:       Technical details about the event that
1091  *                      may help hardware manufacturers and
1092  *                      EDAC developers to analyse the event
1093  */
1094 void edac_mc_handle_error(const enum hw_event_mc_err_type type,
1095                           struct mem_ctl_info *mci,
1096                           const u16 error_count,
1097                           const unsigned long page_frame_number,
1098                           const unsigned long offset_in_page,
1099                           const unsigned long syndrome,
1100                           const int top_layer,
1101                           const int mid_layer,
1102                           const int low_layer,
1103                           const char *msg,
1104                           const char *other_detail)
1105 {
1106         char *p;
1107         int row = -1, chan = -1;
1108         int pos[EDAC_MAX_LAYERS] = { top_layer, mid_layer, low_layer };
1109         int i, n_labels = 0;
1110         u8 grain_bits;
1111         struct edac_raw_error_desc *e = &mci->error_desc;
1112
1113         edac_dbg(3, "MC%d\n", mci->mc_idx);
1114
1115         /* Fills the error report buffer */
1116         memset(e, 0, sizeof (*e));
1117         e->error_count = error_count;
1118         e->top_layer = top_layer;
1119         e->mid_layer = mid_layer;
1120         e->low_layer = low_layer;
1121         e->page_frame_number = page_frame_number;
1122         e->offset_in_page = offset_in_page;
1123         e->syndrome = syndrome;
1124         e->msg = msg;
1125         e->other_detail = other_detail;
1126
1127         /*
1128          * Check if the event report is consistent and if the memory
1129          * location is known. If it is known, enable_per_layer_report will be
1130          * true, the DIMM(s) label info will be filled and the per-layer
1131          * error counters will be incremented.
1132          */
1133         for (i = 0; i < mci->n_layers; i++) {
1134                 if (pos[i] >= (int)mci->layers[i].size) {
1135
1136                         edac_mc_printk(mci, KERN_ERR,
1137                                        "INTERNAL ERROR: %s value is out of range (%d >= %d)\n",
1138                                        edac_layer_name[mci->layers[i].type],
1139                                        pos[i], mci->layers[i].size);
1140                         /*
1141                          * Instead of just returning it, let's use what's
1142                          * known about the error. The increment routines and
1143                          * the DIMM filter logic will do the right thing by
1144                          * pointing the likely damaged DIMMs.
1145                          */
1146                         pos[i] = -1;
1147                 }
1148                 if (pos[i] >= 0)
1149                         e->enable_per_layer_report = true;
1150         }
1151
1152         /*
1153          * Get the dimm label/grain that applies to the match criteria.
1154          * As the error algorithm may not be able to point to just one memory
1155          * stick, the logic here will get all possible labels that could
1156          * pottentially be affected by the error.
1157          * On FB-DIMM memory controllers, for uncorrected errors, it is common
1158          * to have only the MC channel and the MC dimm (also called "branch")
1159          * but the channel is not known, as the memory is arranged in pairs,
1160          * where each memory belongs to a separate channel within the same
1161          * branch.
1162          */
1163         p = e->label;
1164         *p = '\0';
1165
1166         for (i = 0; i < mci->tot_dimms; i++) {
1167                 struct dimm_info *dimm = mci->dimms[i];
1168
1169                 if (top_layer >= 0 && top_layer != dimm->location[0])
1170                         continue;
1171                 if (mid_layer >= 0 && mid_layer != dimm->location[1])
1172                         continue;
1173                 if (low_layer >= 0 && low_layer != dimm->location[2])
1174                         continue;
1175
1176                 /* get the max grain, over the error match range */
1177                 if (dimm->grain > e->grain)
1178                         e->grain = dimm->grain;
1179
1180                 /*
1181                  * If the error is memory-controller wide, there's no need to
1182                  * seek for the affected DIMMs because the whole
1183                  * channel/memory controller/...  may be affected.
1184                  * Also, don't show errors for empty DIMM slots.
1185                  */
1186                 if (e->enable_per_layer_report && dimm->nr_pages) {
1187                         if (n_labels >= EDAC_MAX_LABELS) {
1188                                 e->enable_per_layer_report = false;
1189                                 break;
1190                         }
1191                         n_labels++;
1192                         if (p != e->label) {
1193                                 strcpy(p, OTHER_LABEL);
1194                                 p += strlen(OTHER_LABEL);
1195                         }
1196                         strcpy(p, dimm->label);
1197                         p += strlen(p);
1198                         *p = '\0';
1199
1200                         /*
1201                          * get csrow/channel of the DIMM, in order to allow
1202                          * incrementing the compat API counters
1203                          */
1204                         edac_dbg(4, "%s csrows map: (%d,%d)\n",
1205                                  mci->csbased ? "rank" : "dimm",
1206                                  dimm->csrow, dimm->cschannel);
1207                         if (row == -1)
1208                                 row = dimm->csrow;
1209                         else if (row >= 0 && row != dimm->csrow)
1210                                 row = -2;
1211
1212                         if (chan == -1)
1213                                 chan = dimm->cschannel;
1214                         else if (chan >= 0 && chan != dimm->cschannel)
1215                                 chan = -2;
1216                 }
1217         }
1218
1219         if (!e->enable_per_layer_report) {
1220                 strcpy(e->label, "any memory");
1221         } else {
1222                 edac_dbg(4, "csrow/channel to increment: (%d,%d)\n", row, chan);
1223                 if (p == e->label)
1224                         strcpy(e->label, "unknown memory");
1225                 if (type == HW_EVENT_ERR_CORRECTED) {
1226                         if (row >= 0) {
1227                                 mci->csrows[row]->ce_count += error_count;
1228                                 if (chan >= 0)
1229                                         mci->csrows[row]->channels[chan]->ce_count += error_count;
1230                         }
1231                 } else
1232                         if (row >= 0)
1233                                 mci->csrows[row]->ue_count += error_count;
1234         }
1235
1236         /* Fill the RAM location data */
1237         p = e->location;
1238
1239         for (i = 0; i < mci->n_layers; i++) {
1240                 if (pos[i] < 0)
1241                         continue;
1242
1243                 p += sprintf(p, "%s:%d ",
1244                              edac_layer_name[mci->layers[i].type],
1245                              pos[i]);
1246         }
1247         if (p > e->location)
1248                 *(p - 1) = '\0';
1249
1250         /* Report the error via the trace interface */
1251         grain_bits = fls_long(e->grain) + 1;
1252         trace_mc_event(type, e->msg, e->label, e->error_count,
1253                        mci->mc_idx, e->top_layer, e->mid_layer, e->low_layer,
1254                        (e->page_frame_number << PAGE_SHIFT) | e->offset_in_page,
1255                        grain_bits, e->syndrome, e->other_detail);
1256
1257         edac_raw_mc_handle_error(type, mci, e);
1258 }
1259 EXPORT_SYMBOL_GPL(edac_mc_handle_error);