Pileus Git - ~andy/linux/blob - drivers/edac/i7core_edac.c

   1 /* Intel i7 core/Nehalem Memory Controller kernel module
   2  *
   3  * This driver supports the memory controllers found on the Intel
   4  * processor families i7core, i7core 7xx/8xx, i5core, Xeon 35xx,
   5  * Xeon 55xx and Xeon 56xx also known as Nehalem, Nehalem-EP, Lynnfield
   6  * and Westmere-EP.
   7  *
   8  * This file may be distributed under the terms of the
   9  * GNU General Public License version 2 only.
  10  *
  11  * Copyright (c) 2009-2010 by:
  12  *       Mauro Carvalho Chehab <mchehab@redhat.com>
  13  *
  14  * Red Hat Inc. http://www.redhat.com
  15  *
  16  * Forked and adapted from the i5400_edac driver
  17  *
  18  * Based on the following public Intel datasheets:
  19  * Intel Core i7 Processor Extreme Edition and Intel Core i7 Processor
  20  * Datasheet, Volume 2:
  21  *      http://download.intel.com/design/processor/datashts/320835.pdf
  22  * Intel Xeon Processor 5500 Series Datasheet Volume 2
  23  *      http://www.intel.com/Assets/PDF/datasheet/321322.pdf
  24  * also available at:
  25  *      http://www.arrownac.com/manufacturers/intel/s/nehalem/5500-datasheet-v2.pdf
  26  */
  27
  28 #include <linux/module.h>
  29 #include <linux/init.h>
  30 #include <linux/pci.h>
  31 #include <linux/pci_ids.h>
  32 #include <linux/slab.h>
  33 #include <linux/delay.h>
  34 #include <linux/dmi.h>
  35 #include <linux/edac.h>
  36 #include <linux/mmzone.h>
  37 #include <linux/smp.h>
  38 #include <asm/mce.h>
  39 #include <asm/processor.h>
  40 #include <asm/div64.h>
  41
  42 #include "edac_core.h"
  43
  44 /* Static vars */
  45 static LIST_HEAD(i7core_edac_list);
  46 static DEFINE_MUTEX(i7core_edac_lock);
  47 static int probed;
  48
  49 static int use_pci_fixup;
  50 module_param(use_pci_fixup, int, 0444);
  51 MODULE_PARM_DESC(use_pci_fixup, "Enable PCI fixup to seek for hidden devices");
  52 /*
  53  * This is used for Nehalem-EP and Nehalem-EX devices, where the non-core
  54  * registers start at bus 255, and are not reported by BIOS.
  55  * We currently find devices with only 2 sockets. In order to support more QPI
  56  * Quick Path Interconnect, just increment this number.
  57  */
  58 #define MAX_SOCKET_BUSES        2
  59
  60
  61 /*
  62  * Alter this version for the module when modifications are made
  63  */
  64 #define I7CORE_REVISION    " Ver: 1.0.0"
  65 #define EDAC_MOD_STR      "i7core_edac"
  66
  67 /*
  68  * Debug macros
  69  */
  70 #define i7core_printk(level, fmt, arg...)                       \
  71         edac_printk(level, "i7core", fmt, ##arg)
  72
  73 #define i7core_mc_printk(mci, level, fmt, arg...)               \
  74         edac_mc_chipset_printk(mci, level, "i7core", fmt, ##arg)
  75
  76 /*
  77  * i7core Memory Controller Registers
  78  */
  79
  80         /* OFFSETS for Device 0 Function 0 */
  81
  82 #define MC_CFG_CONTROL  0x90
  83   #define MC_CFG_UNLOCK         0x02
  84   #define MC_CFG_LOCK           0x00
  85
  86         /* OFFSETS for Device 3 Function 0 */
  87
  88 #define MC_CONTROL      0x48
  89 #define MC_STATUS       0x4c
  90 #define MC_MAX_DOD      0x64
  91
  92 /*
  93  * OFFSETS for Device 3 Function 4, as inicated on Xeon 5500 datasheet:
  94  * http://www.arrownac.com/manufacturers/intel/s/nehalem/5500-datasheet-v2.pdf
  95  */
  96
  97 #define MC_TEST_ERR_RCV1        0x60
  98   #define DIMM2_COR_ERR(r)                      ((r) & 0x7fff)
  99
 100 #define MC_TEST_ERR_RCV0        0x64
 101   #define DIMM1_COR_ERR(r)                      (((r) >> 16) & 0x7fff)
 102   #define DIMM0_COR_ERR(r)                      ((r) & 0x7fff)
 103
 104 /* OFFSETS for Device 3 Function 2, as inicated on Xeon 5500 datasheet */
 105 #define MC_SSRCONTROL           0x48
 106   #define SSR_MODE_DISABLE      0x00
 107   #define SSR_MODE_ENABLE       0x01
 108   #define SSR_MODE_MASK         0x03
 109
 110 #define MC_SCRUB_CONTROL        0x4c
 111   #define STARTSCRUB            (1 << 24)
 112   #define SCRUBINTERVAL_MASK    0xffffff
 113
 114 #define MC_COR_ECC_CNT_0        0x80
 115 #define MC_COR_ECC_CNT_1        0x84
 116 #define MC_COR_ECC_CNT_2        0x88
 117 #define MC_COR_ECC_CNT_3        0x8c
 118 #define MC_COR_ECC_CNT_4        0x90
 119 #define MC_COR_ECC_CNT_5        0x94
 120
 121 #define DIMM_TOP_COR_ERR(r)                     (((r) >> 16) & 0x7fff)
 122 #define DIMM_BOT_COR_ERR(r)                     ((r) & 0x7fff)
 123
 124
 125         /* OFFSETS for Devices 4,5 and 6 Function 0 */
 126
 127 #define MC_CHANNEL_DIMM_INIT_PARAMS 0x58
 128   #define THREE_DIMMS_PRESENT           (1 << 24)
 129   #define SINGLE_QUAD_RANK_PRESENT      (1 << 23)
 130   #define QUAD_RANK_PRESENT             (1 << 22)
 131   #define REGISTERED_DIMM               (1 << 15)
 132
 133 #define MC_CHANNEL_MAPPER       0x60
 134   #define RDLCH(r, ch)          ((((r) >> (3 + (ch * 6))) & 0x07) - 1)
 135   #define WRLCH(r, ch)          ((((r) >> (ch * 6)) & 0x07) - 1)
 136
 137 #define MC_CHANNEL_RANK_PRESENT 0x7c
 138   #define RANK_PRESENT_MASK             0xffff
 139
 140 #define MC_CHANNEL_ADDR_MATCH   0xf0
 141 #define MC_CHANNEL_ERROR_MASK   0xf8
 142 #define MC_CHANNEL_ERROR_INJECT 0xfc
 143   #define INJECT_ADDR_PARITY    0x10
 144   #define INJECT_ECC            0x08
 145   #define MASK_CACHELINE        0x06
 146   #define MASK_FULL_CACHELINE   0x06
 147   #define MASK_MSB32_CACHELINE  0x04
 148   #define MASK_LSB32_CACHELINE  0x02
 149   #define NO_MASK_CACHELINE     0x00
 150   #define REPEAT_EN             0x01
 151
 152         /* OFFSETS for Devices 4,5 and 6 Function 1 */
 153
 154 #define MC_DOD_CH_DIMM0         0x48
 155 #define MC_DOD_CH_DIMM1         0x4c
 156 #define MC_DOD_CH_DIMM2         0x50
 157   #define RANKOFFSET_MASK       ((1 << 12) | (1 << 11) | (1 << 10))
 158   #define RANKOFFSET(x)         ((x & RANKOFFSET_MASK) >> 10)
 159   #define DIMM_PRESENT_MASK     (1 << 9)
 160   #define DIMM_PRESENT(x)       (((x) & DIMM_PRESENT_MASK) >> 9)
 161   #define MC_DOD_NUMBANK_MASK           ((1 << 8) | (1 << 7))
 162   #define MC_DOD_NUMBANK(x)             (((x) & MC_DOD_NUMBANK_MASK) >> 7)
 163   #define MC_DOD_NUMRANK_MASK           ((1 << 6) | (1 << 5))
 164   #define MC_DOD_NUMRANK(x)             (((x) & MC_DOD_NUMRANK_MASK) >> 5)
 165   #define MC_DOD_NUMROW_MASK            ((1 << 4) | (1 << 3) | (1 << 2))
 166   #define MC_DOD_NUMROW(x)              (((x) & MC_DOD_NUMROW_MASK) >> 2)
 167   #define MC_DOD_NUMCOL_MASK            3
 168   #define MC_DOD_NUMCOL(x)              ((x) & MC_DOD_NUMCOL_MASK)
 169
 170 #define MC_RANK_PRESENT         0x7c
 171
 172 #define MC_SAG_CH_0     0x80
 173 #define MC_SAG_CH_1     0x84
 174 #define MC_SAG_CH_2     0x88
 175 #define MC_SAG_CH_3     0x8c
 176 #define MC_SAG_CH_4     0x90
 177 #define MC_SAG_CH_5     0x94
 178 #define MC_SAG_CH_6     0x98
 179 #define MC_SAG_CH_7     0x9c
 180
 181 #define MC_RIR_LIMIT_CH_0       0x40
 182 #define MC_RIR_LIMIT_CH_1       0x44
 183 #define MC_RIR_LIMIT_CH_2       0x48
 184 #define MC_RIR_LIMIT_CH_3       0x4C
 185 #define MC_RIR_LIMIT_CH_4       0x50
 186 #define MC_RIR_LIMIT_CH_5       0x54
 187 #define MC_RIR_LIMIT_CH_6       0x58
 188 #define MC_RIR_LIMIT_CH_7       0x5C
 189 #define MC_RIR_LIMIT_MASK       ((1 << 10) - 1)
 190
 191 #define MC_RIR_WAY_CH           0x80
 192   #define MC_RIR_WAY_OFFSET_MASK        (((1 << 14) - 1) & ~0x7)
 193   #define MC_RIR_WAY_RANK_MASK          0x7
 194
 195 /*
 196  * i7core structs
 197  */
 198
 199 #define NUM_CHANS 3
 200 #define MAX_DIMMS 3             /* Max DIMMS per channel */
 201 #define MAX_MCR_FUNC  4
 202 #define MAX_CHAN_FUNC 3
 203
 204 struct i7core_info {
 205         u32     mc_control;
 206         u32     mc_status;
 207         u32     max_dod;
 208         u32     ch_map;
 209 };
 210
 211
 212 struct i7core_inject {
 213         int     enable;
 214
 215         u32     section;
 216         u32     type;
 217         u32     eccmask;
 218
 219         /* Error address mask */
 220         int channel, dimm, rank, bank, page, col;
 221 };
 222
 223 struct i7core_channel {
 224         bool            is_3dimms_present;
 225         bool            is_single_4rank;
 226         bool            has_4rank;
 227         u32             dimms;
 228 };
 229
 230 struct pci_id_descr {
 231         int                     dev;
 232         int                     func;
 233         int                     dev_id;
 234         int                     optional;
 235 };
 236
 237 struct pci_id_table {
 238         const struct pci_id_descr       *descr;
 239         int                             n_devs;
 240 };
 241
 242 struct i7core_dev {
 243         struct list_head        list;
 244         u8                      socket;
 245         struct pci_dev          **pdev;
 246         int                     n_devs;
 247         struct mem_ctl_info     *mci;
 248 };
 249
 250 struct i7core_pvt {
 251         struct device *addrmatch_dev, *chancounts_dev;
 252
 253         struct pci_dev  *pci_noncore;
 254         struct pci_dev  *pci_mcr[MAX_MCR_FUNC + 1];
 255         struct pci_dev  *pci_ch[NUM_CHANS][MAX_CHAN_FUNC + 1];
 256
 257         struct i7core_dev *i7core_dev;
 258
 259         struct i7core_info      info;
 260         struct i7core_inject    inject;
 261         struct i7core_channel   channel[NUM_CHANS];
 262
 263         int             ce_count_available;
 264
 265                         /* ECC corrected errors counts per udimm */
 266         unsigned long   udimm_ce_count[MAX_DIMMS];
 267         int             udimm_last_ce_count[MAX_DIMMS];
 268                         /* ECC corrected errors counts per rdimm */
 269         unsigned long   rdimm_ce_count[NUM_CHANS][MAX_DIMMS];
 270         int             rdimm_last_ce_count[NUM_CHANS][MAX_DIMMS];
 271
 272         bool            is_registered, enable_scrub;
 273
 274         /* Fifo double buffers */
 275         struct mce              mce_entry[MCE_LOG_LEN];
 276         struct mce              mce_outentry[MCE_LOG_LEN];
 277
 278         /* Fifo in/out counters */
 279         unsigned                mce_in, mce_out;
 280
 281         /* Count indicator to show errors not got */
 282         unsigned                mce_overrun;
 283
 284         /* DCLK Frequency used for computing scrub rate */
 285         int                     dclk_freq;
 286
 287         /* Struct to control EDAC polling */
 288         struct edac_pci_ctl_info *i7core_pci;
 289 };
 290
 291 #define PCI_DESCR(device, function, device_id)  \
 292         .dev = (device),                        \
 293         .func = (function),                     \
 294         .dev_id = (device_id)
 295
 296 static const struct pci_id_descr pci_dev_descr_i7core_nehalem[] = {
 297                 /* Memory controller */
 298         { PCI_DESCR(3, 0, PCI_DEVICE_ID_INTEL_I7_MCR)     },
 299         { PCI_DESCR(3, 1, PCI_DEVICE_ID_INTEL_I7_MC_TAD)  },
 300                         /* Exists only for RDIMM */
 301         { PCI_DESCR(3, 2, PCI_DEVICE_ID_INTEL_I7_MC_RAS), .optional = 1  },
 302         { PCI_DESCR(3, 4, PCI_DEVICE_ID_INTEL_I7_MC_TEST) },
 303
 304                 /* Channel 0 */
 305         { PCI_DESCR(4, 0, PCI_DEVICE_ID_INTEL_I7_MC_CH0_CTRL) },
 306         { PCI_DESCR(4, 1, PCI_DEVICE_ID_INTEL_I7_MC_CH0_ADDR) },
 307         { PCI_DESCR(4, 2, PCI_DEVICE_ID_INTEL_I7_MC_CH0_RANK) },
 308         { PCI_DESCR(4, 3, PCI_DEVICE_ID_INTEL_I7_MC_CH0_TC)   },
 309
 310                 /* Channel 1 */
 311         { PCI_DESCR(5, 0, PCI_DEVICE_ID_INTEL_I7_MC_CH1_CTRL) },
 312         { PCI_DESCR(5, 1, PCI_DEVICE_ID_INTEL_I7_MC_CH1_ADDR) },
 313         { PCI_DESCR(5, 2, PCI_DEVICE_ID_INTEL_I7_MC_CH1_RANK) },
 314         { PCI_DESCR(5, 3, PCI_DEVICE_ID_INTEL_I7_MC_CH1_TC)   },
 315
 316                 /* Channel 2 */
 317         { PCI_DESCR(6, 0, PCI_DEVICE_ID_INTEL_I7_MC_CH2_CTRL) },
 318         { PCI_DESCR(6, 1, PCI_DEVICE_ID_INTEL_I7_MC_CH2_ADDR) },
 319         { PCI_DESCR(6, 2, PCI_DEVICE_ID_INTEL_I7_MC_CH2_RANK) },
 320         { PCI_DESCR(6, 3, PCI_DEVICE_ID_INTEL_I7_MC_CH2_TC)   },
 321
 322                 /* Generic Non-core registers */
 323         /*
 324          * This is the PCI device on i7core and on Xeon 35xx (8086:2c41)
 325          * On Xeon 55xx, however, it has a different id (8086:2c40). So,
 326          * the probing code needs to test for the other address in case of
 327          * failure of this one
 328          */
 329         { PCI_DESCR(0, 0, PCI_DEVICE_ID_INTEL_I7_NONCORE)  },
 330
 331 };
 332
 333 static const struct pci_id_descr pci_dev_descr_lynnfield[] = {
 334         { PCI_DESCR( 3, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MCR)         },
 335         { PCI_DESCR( 3, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_TAD)      },
 336         { PCI_DESCR( 3, 4, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_TEST)     },
 337
 338         { PCI_DESCR( 4, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_CTRL) },
 339         { PCI_DESCR( 4, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_ADDR) },
 340         { PCI_DESCR( 4, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_RANK) },
 341         { PCI_DESCR( 4, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_TC)   },
 342
 343         { PCI_DESCR( 5, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_CTRL) },
 344         { PCI_DESCR( 5, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_ADDR) },
 345         { PCI_DESCR( 5, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_RANK) },
 346         { PCI_DESCR( 5, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_TC)   },
 347
 348         /*
 349          * This is the PCI device has an alternate address on some
 350          * processors like Core i7 860
 351          */
 352         { PCI_DESCR( 0, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE)     },
 353 };
 354
 355 static const struct pci_id_descr pci_dev_descr_i7core_westmere[] = {
 356                 /* Memory controller */
 357         { PCI_DESCR(3, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MCR_REV2)     },
 358         { PCI_DESCR(3, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_TAD_REV2)  },
 359                         /* Exists only for RDIMM */
 360         { PCI_DESCR(3, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_RAS_REV2), .optional = 1  },
 361         { PCI_DESCR(3, 4, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_TEST_REV2) },
 362
 363                 /* Channel 0 */
 364         { PCI_DESCR(4, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_CTRL_REV2) },
 365         { PCI_DESCR(4, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_ADDR_REV2) },
 366         { PCI_DESCR(4, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_RANK_REV2) },
 367         { PCI_DESCR(4, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_TC_REV2)   },
 368
 369                 /* Channel 1 */
 370         { PCI_DESCR(5, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_CTRL_REV2) },
 371         { PCI_DESCR(5, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_ADDR_REV2) },
 372         { PCI_DESCR(5, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_RANK_REV2) },
 373         { PCI_DESCR(5, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_TC_REV2)   },
 374
 375                 /* Channel 2 */
 376         { PCI_DESCR(6, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH2_CTRL_REV2) },
 377         { PCI_DESCR(6, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH2_ADDR_REV2) },
 378         { PCI_DESCR(6, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH2_RANK_REV2) },
 379         { PCI_DESCR(6, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH2_TC_REV2)   },
 380
 381                 /* Generic Non-core registers */
 382         { PCI_DESCR(0, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE_REV2)  },
 383
 384 };
 385
 386 #define PCI_ID_TABLE_ENTRY(A) { .descr=A, .n_devs = ARRAY_SIZE(A) }
 387 static const struct pci_id_table pci_dev_table[] = {
 388         PCI_ID_TABLE_ENTRY(pci_dev_descr_i7core_nehalem),
 389         PCI_ID_TABLE_ENTRY(pci_dev_descr_lynnfield),
 390         PCI_ID_TABLE_ENTRY(pci_dev_descr_i7core_westmere),
 391         {0,}                    /* 0 terminated list. */
 392 };
 393
 394 /*
 395  *      pci_device_id   table for which devices we are looking for
 396  */
 397 static DEFINE_PCI_DEVICE_TABLE(i7core_pci_tbl) = {
 398         {PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_X58_HUB_MGMT)},
 399         {PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_LYNNFIELD_QPI_LINK0)},
 400         {0,}                    /* 0 terminated list. */
 401 };
 402
 403 /****************************************************************************
 404                         Anciliary status routines
 405  ****************************************************************************/
 406
 407         /* MC_CONTROL bits */
 408 #define CH_ACTIVE(pvt, ch)      ((pvt)->info.mc_control & (1 << (8 + ch)))
 409 #define ECCx8(pvt)              ((pvt)->info.mc_control & (1 << 1))
 410
 411         /* MC_STATUS bits */
 412 #define ECC_ENABLED(pvt)        ((pvt)->info.mc_status & (1 << 4))
 413 #define CH_DISABLED(pvt, ch)    ((pvt)->info.mc_status & (1 << ch))
 414
 415         /* MC_MAX_DOD read functions */
 416 static inline int numdimms(u32 dimms)
 417 {
 418         return (dimms & 0x3) + 1;
 419 }
 420
 421 static inline int numrank(u32 rank)
 422 {
 423         static int ranks[4] = { 1, 2, 4, -EINVAL };
 424
 425         return ranks[rank & 0x3];
 426 }
 427
 428 static inline int numbank(u32 bank)
 429 {
 430         static int banks[4] = { 4, 8, 16, -EINVAL };
 431
 432         return banks[bank & 0x3];
 433 }
 434
 435 static inline int numrow(u32 row)
 436 {
 437         static int rows[8] = {
 438                 1 << 12, 1 << 13, 1 << 14, 1 << 15,
 439                 1 << 16, -EINVAL, -EINVAL, -EINVAL,
 440         };
 441
 442         return rows[row & 0x7];
 443 }
 444
 445 static inline int numcol(u32 col)
 446 {
 447         static int cols[8] = {
 448                 1 << 10, 1 << 11, 1 << 12, -EINVAL,
 449         };
 450         return cols[col & 0x3];
 451 }
 452
 453 static struct i7core_dev *get_i7core_dev(u8 socket)
 454 {
 455         struct i7core_dev *i7core_dev;
 456
 457         list_for_each_entry(i7core_dev, &i7core_edac_list, list) {
 458                 if (i7core_dev->socket == socket)
 459                         return i7core_dev;
 460         }
 461
 462         return NULL;
 463 }
 464
 465 static struct i7core_dev *alloc_i7core_dev(u8 socket,
 466                                            const struct pci_id_table *table)
 467 {
 468         struct i7core_dev *i7core_dev;
 469
 470         i7core_dev = kzalloc(sizeof(*i7core_dev), GFP_KERNEL);
 471         if (!i7core_dev)
 472                 return NULL;
 473
 474         i7core_dev->pdev = kzalloc(sizeof(*i7core_dev->pdev) * table->n_devs,
 475                                    GFP_KERNEL);
 476         if (!i7core_dev->pdev) {
 477                 kfree(i7core_dev);
 478                 return NULL;
 479         }
 480
 481         i7core_dev->socket = socket;
 482         i7core_dev->n_devs = table->n_devs;
 483         list_add_tail(&i7core_dev->list, &i7core_edac_list);
 484
 485         return i7core_dev;
 486 }
 487
 488 static void free_i7core_dev(struct i7core_dev *i7core_dev)
 489 {
 490         list_del(&i7core_dev->list);
 491         kfree(i7core_dev->pdev);
 492         kfree(i7core_dev);
 493 }
 494
 495 /****************************************************************************
 496                         Memory check routines
 497  ****************************************************************************/
 498
 499 static int get_dimm_config(struct mem_ctl_info *mci)
 500 {
 501         struct i7core_pvt *pvt = mci->pvt_info;
 502         struct pci_dev *pdev;
 503         int i, j;
 504         enum edac_type mode;
 505         enum mem_type mtype;
 506         struct dimm_info *dimm;
 507
 508         /* Get data from the MC register, function 0 */
 509         pdev = pvt->pci_mcr[0];
 510         if (!pdev)
 511                 return -ENODEV;
 512
 513         /* Device 3 function 0 reads */
 514         pci_read_config_dword(pdev, MC_CONTROL, &pvt->info.mc_control);
 515         pci_read_config_dword(pdev, MC_STATUS, &pvt->info.mc_status);
 516         pci_read_config_dword(pdev, MC_MAX_DOD, &pvt->info.max_dod);
 517         pci_read_config_dword(pdev, MC_CHANNEL_MAPPER, &pvt->info.ch_map);
 518
 519         debugf0("QPI %d control=0x%08x status=0x%08x dod=0x%08x map=0x%08x\n",
 520                 pvt->i7core_dev->socket, pvt->info.mc_control, pvt->info.mc_status,
 521                 pvt->info.max_dod, pvt->info.ch_map);
 522
 523         if (ECC_ENABLED(pvt)) {
 524                 debugf0("ECC enabled with x%d SDCC\n", ECCx8(pvt) ? 8 : 4);
 525                 if (ECCx8(pvt))
 526                         mode = EDAC_S8ECD8ED;
 527                 else
 528                         mode = EDAC_S4ECD4ED;
 529         } else {
 530                 debugf0("ECC disabled\n");
 531                 mode = EDAC_NONE;
 532         }
 533
 534         /* FIXME: need to handle the error codes */
 535         debugf0("DOD Max limits: DIMMS: %d, %d-ranked, %d-banked "
 536                 "x%x x 0x%x\n",
 537                 numdimms(pvt->info.max_dod),
 538                 numrank(pvt->info.max_dod >> 2),
 539                 numbank(pvt->info.max_dod >> 4),
 540                 numrow(pvt->info.max_dod >> 6),
 541                 numcol(pvt->info.max_dod >> 9));
 542
 543         for (i = 0; i < NUM_CHANS; i++) {
 544                 u32 data, dimm_dod[3], value[8];
 545
 546                 if (!pvt->pci_ch[i][0])
 547                         continue;
 548
 549                 if (!CH_ACTIVE(pvt, i)) {
 550                         debugf0("Channel %i is not active\n", i);
 551                         continue;
 552                 }
 553                 if (CH_DISABLED(pvt, i)) {
 554                         debugf0("Channel %i is disabled\n", i);
 555                         continue;
 556                 }
 557
 558                 /* Devices 4-6 function 0 */
 559                 pci_read_config_dword(pvt->pci_ch[i][0],
 560                                 MC_CHANNEL_DIMM_INIT_PARAMS, &data);
 561
 562
 563                 if (data & THREE_DIMMS_PRESENT)
 564                         pvt->channel[i].is_3dimms_present = true;
 565
 566                 if (data & SINGLE_QUAD_RANK_PRESENT)
 567                         pvt->channel[i].is_single_4rank = true;
 568
 569                 if (data & QUAD_RANK_PRESENT)
 570                         pvt->channel[i].has_4rank = true;
 571
 572                 if (data & REGISTERED_DIMM)
 573                         mtype = MEM_RDDR3;
 574                 else
 575                         mtype = MEM_DDR3;
 576
 577                 /* Devices 4-6 function 1 */
 578                 pci_read_config_dword(pvt->pci_ch[i][1],
 579                                 MC_DOD_CH_DIMM0, &dimm_dod[0]);
 580                 pci_read_config_dword(pvt->pci_ch[i][1],
 581                                 MC_DOD_CH_DIMM1, &dimm_dod[1]);
 582                 pci_read_config_dword(pvt->pci_ch[i][1],
 583                                 MC_DOD_CH_DIMM2, &dimm_dod[2]);
 584
 585                 debugf0("Ch%d phy rd%d, wr%d (0x%08x): "
 586                         "%s%s%s%cDIMMs\n",
 587                         i,
 588                         RDLCH(pvt->info.ch_map, i), WRLCH(pvt->info.ch_map, i),
 589                         data,
 590                         pvt->channel[i].is_3dimms_present ? "3DIMMS " : "",
 591                         pvt->channel[i].is_3dimms_present ? "SINGLE_4R " : "",
 592                         pvt->channel[i].has_4rank ? "HAS_4R " : "",
 593                         (data & REGISTERED_DIMM) ? 'R' : 'U');
 594
 595                 for (j = 0; j < 3; j++) {
 596                         u32 banks, ranks, rows, cols;
 597                         u32 size, npages;
 598
 599                         if (!DIMM_PRESENT(dimm_dod[j]))
 600                                 continue;
 601
 602                         dimm = EDAC_DIMM_PTR(mci->layers, mci->dimms, mci->n_layers,
 603                                        i, j, 0);
 604                         banks = numbank(MC_DOD_NUMBANK(dimm_dod[j]));
 605                         ranks = numrank(MC_DOD_NUMRANK(dimm_dod[j]));
 606                         rows = numrow(MC_DOD_NUMROW(dimm_dod[j]));
 607                         cols = numcol(MC_DOD_NUMCOL(dimm_dod[j]));
 608
 609                         /* DDR3 has 8 I/O banks */
 610                         size = (rows * cols * banks * ranks) >> (20 - 3);
 611
 612                         debugf0("\tdimm %d %d Mb offset: %x, "
 613                                 "bank: %d, rank: %d, row: %#x, col: %#x\n",
 614                                 j, size,
 615                                 RANKOFFSET(dimm_dod[j]),
 616                                 banks, ranks, rows, cols);
 617
 618                         npages = MiB_TO_PAGES(size);
 619
 620                         dimm->nr_pages = npages;
 621
 622                         switch (banks) {
 623                         case 4:
 624                                 dimm->dtype = DEV_X4;
 625                                 break;
 626                         case 8:
 627                                 dimm->dtype = DEV_X8;
 628                                 break;
 629                         case 16:
 630                                 dimm->dtype = DEV_X16;
 631                                 break;
 632                         default:
 633                                 dimm->dtype = DEV_UNKNOWN;
 634                         }
 635
 636                         snprintf(dimm->label, sizeof(dimm->label),
 637                                  "CPU#%uChannel#%u_DIMM#%u",
 638                                  pvt->i7core_dev->socket, i, j);
 639                         dimm->grain = 8;
 640                         dimm->edac_mode = mode;
 641                         dimm->mtype = mtype;
 642                 }
 643
 644                 pci_read_config_dword(pdev, MC_SAG_CH_0, &value[0]);
 645                 pci_read_config_dword(pdev, MC_SAG_CH_1, &value[1]);
 646                 pci_read_config_dword(pdev, MC_SAG_CH_2, &value[2]);
 647                 pci_read_config_dword(pdev, MC_SAG_CH_3, &value[3]);
 648                 pci_read_config_dword(pdev, MC_SAG_CH_4, &value[4]);
 649                 pci_read_config_dword(pdev, MC_SAG_CH_5, &value[5]);
 650                 pci_read_config_dword(pdev, MC_SAG_CH_6, &value[6]);
 651                 pci_read_config_dword(pdev, MC_SAG_CH_7, &value[7]);
 652                 debugf1("\t[%i] DIVBY3\tREMOVED\tOFFSET\n", i);
 653                 for (j = 0; j < 8; j++)
 654                         debugf1("\t\t%#x\t%#x\t%#x\n",
 655                                 (value[j] >> 27) & 0x1,
 656                                 (value[j] >> 24) & 0x7,
 657                                 (value[j] & ((1 << 24) - 1)));
 658         }
 659
 660         return 0;
 661 }
 662
 663 /****************************************************************************
 664                         Error insertion routines
 665  ****************************************************************************/
 666
 667 #define to_mci(k) container_of(k, struct mem_ctl_info, dev)
 668
 669 /* The i7core has independent error injection features per channel.
 670    However, to have a simpler code, we don't allow enabling error injection
 671    on more than one channel.
 672    Also, since a change at an inject parameter will be applied only at enable,
 673    we're disabling error injection on all write calls to the sysfs nodes that
 674    controls the error code injection.
 675  */
 676 static int disable_inject(const struct mem_ctl_info *mci)
 677 {
 678         struct i7core_pvt *pvt = mci->pvt_info;
 679
 680         pvt->inject.enable = 0;
 681
 682         if (!pvt->pci_ch[pvt->inject.channel][0])
 683                 return -ENODEV;
 684
 685         pci_write_config_dword(pvt->pci_ch[pvt->inject.channel][0],
 686                                 MC_CHANNEL_ERROR_INJECT, 0);
 687
 688         return 0;
 689 }
 690
 691 /*
 692  * i7core inject inject.section
 693  *
 694  *      accept and store error injection inject.section value
 695  *      bit 0 - refers to the lower 32-byte half cacheline
 696  *      bit 1 - refers to the upper 32-byte half cacheline
 697  */
 698 static ssize_t i7core_inject_section_store(struct device *dev,
 699                                            struct device_attribute *mattr,
 700                                            const char *data, size_t count)
 701 {
 702         struct mem_ctl_info *mci = to_mci(dev);
 703         struct i7core_pvt *pvt = mci->pvt_info;
 704         unsigned long value;
 705         int rc;
 706
 707         if (pvt->inject.enable)
 708                 disable_inject(mci);
 709
 710         rc = strict_strtoul(data, 10, &value);
 711         if ((rc < 0) || (value > 3))
 712                 return -EIO;
 713
 714         pvt->inject.section = (u32) value;
 715         return count;
 716 }
 717
 718 static ssize_t i7core_inject_section_show(struct device *dev,
 719                                           struct device_attribute *mattr,
 720                                           char *data)
 721 {
 722         struct mem_ctl_info *mci = to_mci(dev);
 723         struct i7core_pvt *pvt = mci->pvt_info;
 724         return sprintf(data, "0x%08x\n", pvt->inject.section);
 725 }
 726
 727 /*
 728  * i7core inject.type
 729  *
 730  *      accept and store error injection inject.section value
 731  *      bit 0 - repeat enable - Enable error repetition
 732  *      bit 1 - inject ECC error
 733  *      bit 2 - inject parity error
 734  */
 735 static ssize_t i7core_inject_type_store(struct device *dev,
 736                                         struct device_attribute *mattr,
 737                                         const char *data, size_t count)
 738 {
 739         struct mem_ctl_info *mci = to_mci(dev);
 740 struct i7core_pvt *pvt = mci->pvt_info;
 741         unsigned long value;
 742         int rc;
 743
 744         if (pvt->inject.enable)
 745                 disable_inject(mci);
 746
 747         rc = strict_strtoul(data, 10, &value);
 748         if ((rc < 0) || (value > 7))
 749                 return -EIO;
 750
 751         pvt->inject.type = (u32) value;
 752         return count;
 753 }
 754
 755 static ssize_t i7core_inject_type_show(struct device *dev,
 756                                        struct device_attribute *mattr,
 757                                        char *data)
 758 {
 759         struct mem_ctl_info *mci = to_mci(dev);
 760         struct i7core_pvt *pvt = mci->pvt_info;
 761
 762         return sprintf(data, "0x%08x\n", pvt->inject.type);
 763 }
 764
 765 /*
 766  * i7core_inject_inject.eccmask_store
 767  *
 768  * The type of error (UE/CE) will depend on the inject.eccmask value:
 769  *   Any bits set to a 1 will flip the corresponding ECC bit
 770  *   Correctable errors can be injected by flipping 1 bit or the bits within
 771  *   a symbol pair (2 consecutive aligned 8-bit pairs - i.e. 7:0 and 15:8 or
 772  *   23:16 and 31:24). Flipping bits in two symbol pairs will cause an
 773  *   uncorrectable error to be injected.
 774  */
 775 static ssize_t i7core_inject_eccmask_store(struct device *dev,
 776                                            struct device_attribute *mattr,
 777                                            const char *data, size_t count)
 778 {
 779         struct mem_ctl_info *mci = to_mci(dev);
 780         struct i7core_pvt *pvt = mci->pvt_info;
 781         unsigned long value;
 782         int rc;
 783
 784         if (pvt->inject.enable)
 785                 disable_inject(mci);
 786
 787         rc = strict_strtoul(data, 10, &value);
 788         if (rc < 0)
 789                 return -EIO;
 790
 791         pvt->inject.eccmask = (u32) value;
 792         return count;
 793 }
 794
 795 static ssize_t i7core_inject_eccmask_show(struct device *dev,
 796                                           struct device_attribute *mattr,
 797                                           char *data)
 798 {
 799         struct mem_ctl_info *mci = to_mci(dev);
 800         struct i7core_pvt *pvt = mci->pvt_info;
 801
 802         return sprintf(data, "0x%08x\n", pvt->inject.eccmask);
 803 }
 804
 805 /*
 806  * i7core_addrmatch
 807  *
 808  * The type of error (UE/CE) will depend on the inject.eccmask value:
 809  *   Any bits set to a 1 will flip the corresponding ECC bit
 810  *   Correctable errors can be injected by flipping 1 bit or the bits within
 811  *   a symbol pair (2 consecutive aligned 8-bit pairs - i.e. 7:0 and 15:8 or
 812  *   23:16 and 31:24). Flipping bits in two symbol pairs will cause an
 813  *   uncorrectable error to be injected.
 814  */
 815
 816 #define DECLARE_ADDR_MATCH(param, limit)                        \
 817 static ssize_t i7core_inject_store_##param(                     \
 818         struct device *dev,                                     \
 819         struct device_attribute *mattr,                         \
 820         const char *data, size_t count)                         \
 821 {                                                               \
 822         struct mem_ctl_info *mci = to_mci(dev);                 \
 823         struct i7core_pvt *pvt;                                 \
 824         long value;                                             \
 825         int rc;                                                 \
 826                                                                 \
 827         debugf1("%s()\n", __func__);                            \
 828         pvt = mci->pvt_info;                                    \
 829                                                                 \
 830         if (pvt->inject.enable)                                 \
 831                 disable_inject(mci);                            \
 832                                                                 \
 833         if (!strcasecmp(data, "any") || !strcasecmp(data, "any\n"))\
 834                 value = -1;                                     \
 835         else {                                                  \
 836                 rc = strict_strtoul(data, 10, &value);          \
 837                 if ((rc < 0) || (value >= limit))               \
 838                         return -EIO;                            \
 839         }                                                       \
 840                                                                 \
 841         pvt->inject.param = value;                              \
 842                                                                 \
 843         return count;                                           \
 844 }                                                               \
 845                                                                 \
 846 static ssize_t i7core_inject_show_##param(                      \
 847         struct device *dev,                                     \
 848         struct device_attribute *mattr,                         \
 849         char *data)                                             \
 850 {                                                               \
 851         struct mem_ctl_info *mci = to_mci(dev);                 \
 852         struct i7core_pvt *pvt;                                 \
 853                                                                 \
 854         pvt = mci->pvt_info;                                    \
 855         debugf1("%s() pvt=%p\n", __func__, pvt);                \
 856         if (pvt->inject.param < 0)                              \
 857                 return sprintf(data, "any\n");                  \
 858         else                                                    \
 859                 return sprintf(data, "%d\n", pvt->inject.param);\
 860 }
 861
 862 #define ATTR_ADDR_MATCH(param)                                  \
 863         static DEVICE_ATTR(param, S_IRUGO | S_IWUSR,            \
 864                     i7core_inject_show_##param,                 \
 865                     i7core_inject_store_##param)
 866
 867 DECLARE_ADDR_MATCH(channel, 3);
 868 DECLARE_ADDR_MATCH(dimm, 3);
 869 DECLARE_ADDR_MATCH(rank, 4);
 870 DECLARE_ADDR_MATCH(bank, 32);
 871 DECLARE_ADDR_MATCH(page, 0x10000);
 872 DECLARE_ADDR_MATCH(col, 0x4000);
 873
 874 ATTR_ADDR_MATCH(channel);
 875 ATTR_ADDR_MATCH(dimm);
 876 ATTR_ADDR_MATCH(rank);
 877 ATTR_ADDR_MATCH(bank);
 878 ATTR_ADDR_MATCH(page);
 879 ATTR_ADDR_MATCH(col);
 880
 881 static int write_and_test(struct pci_dev *dev, const int where, const u32 val)
 882 {
 883         u32 read;
 884         int count;
 885
 886         debugf0("setting pci %02x:%02x.%x reg=%02x value=%08x\n",
 887                 dev->bus->number, PCI_SLOT(dev->devfn), PCI_FUNC(dev->devfn),
 888                 where, val);
 889
 890         for (count = 0; count < 10; count++) {
 891                 if (count)
 892                         msleep(100);
 893                 pci_write_config_dword(dev, where, val);
 894                 pci_read_config_dword(dev, where, &read);
 895
 896                 if (read == val)
 897                         return 0;
 898         }
 899
 900         i7core_printk(KERN_ERR, "Error during set pci %02x:%02x.%x reg=%02x "
 901                 "write=%08x. Read=%08x\n",
 902                 dev->bus->number, PCI_SLOT(dev->devfn), PCI_FUNC(dev->devfn),
 903                 where, val, read);
 904
 905         return -EINVAL;
 906 }
 907
 908 /*
 909  * This routine prepares the Memory Controller for error injection.
 910  * The error will be injected when some process tries to write to the
 911  * memory that matches the given criteria.
 912  * The criteria can be set in terms of a mask where dimm, rank, bank, page
 913  * and col can be specified.
 914  * A -1 value for any of the mask items will make the MCU to ignore
 915  * that matching criteria for error injection.
 916  *
 917  * It should be noticed that the error will only happen after a write operation
 918  * on a memory that matches the condition. if REPEAT_EN is not enabled at
 919  * inject mask, then it will produce just one error. Otherwise, it will repeat
 920  * until the injectmask would be cleaned.
 921  *
 922  * FIXME: This routine assumes that MAXNUMDIMMS value of MC_MAX_DOD
 923  *    is reliable enough to check if the MC is using the
 924  *    three channels. However, this is not clear at the datasheet.
 925  */
 926 static ssize_t i7core_inject_enable_store(struct device *dev,
 927                                           struct device_attribute *mattr,
 928                                           const char *data, size_t count)
 929 {
 930         struct mem_ctl_info *mci = to_mci(dev);
 931         struct i7core_pvt *pvt = mci->pvt_info;
 932         u32 injectmask;
 933         u64 mask = 0;
 934         int  rc;
 935         long enable;
 936
 937         if (!pvt->pci_ch[pvt->inject.channel][0])
 938                 return 0;
 939
 940         rc = strict_strtoul(data, 10, &enable);
 941         if ((rc < 0))
 942                 return 0;
 943
 944         if (enable) {
 945                 pvt->inject.enable = 1;
 946         } else {
 947                 disable_inject(mci);
 948                 return count;
 949         }
 950
 951         /* Sets pvt->inject.dimm mask */
 952         if (pvt->inject.dimm < 0)
 953                 mask |= 1LL << 41;
 954         else {
 955                 if (pvt->channel[pvt->inject.channel].dimms > 2)
 956                         mask |= (pvt->inject.dimm & 0x3LL) << 35;
 957                 else
 958                         mask |= (pvt->inject.dimm & 0x1LL) << 36;
 959         }
 960
 961         /* Sets pvt->inject.rank mask */
 962         if (pvt->inject.rank < 0)
 963                 mask |= 1LL << 40;
 964         else {
 965                 if (pvt->channel[pvt->inject.channel].dimms > 2)
 966                         mask |= (pvt->inject.rank & 0x1LL) << 34;
 967                 else
 968                         mask |= (pvt->inject.rank & 0x3LL) << 34;
 969         }
 970
 971         /* Sets pvt->inject.bank mask */
 972         if (pvt->inject.bank < 0)
 973                 mask |= 1LL << 39;
 974         else
 975                 mask |= (pvt->inject.bank & 0x15LL) << 30;
 976
 977         /* Sets pvt->inject.page mask */
 978         if (pvt->inject.page < 0)
 979                 mask |= 1LL << 38;
 980         else
 981                 mask |= (pvt->inject.page & 0xffff) << 14;
 982
 983         /* Sets pvt->inject.column mask */
 984         if (pvt->inject.col < 0)
 985                 mask |= 1LL << 37;
 986         else
 987                 mask |= (pvt->inject.col & 0x3fff);
 988
 989         /*
 990          * bit    0: REPEAT_EN
 991          * bits 1-2: MASK_HALF_CACHELINE
 992          * bit    3: INJECT_ECC
 993          * bit    4: INJECT_ADDR_PARITY
 994          */
 995
 996         injectmask = (pvt->inject.type & 1) |
 997                      (pvt->inject.section & 0x3) << 1 |
 998                      (pvt->inject.type & 0x6) << (3 - 1);
 999
1000         /* Unlock writes to registers - this register is write only */
1001         pci_write_config_dword(pvt->pci_noncore,
1002                                MC_CFG_CONTROL, 0x2);
1003
1004         write_and_test(pvt->pci_ch[pvt->inject.channel][0],
1005                                MC_CHANNEL_ADDR_MATCH, mask);
1006         write_and_test(pvt->pci_ch[pvt->inject.channel][0],
1007                                MC_CHANNEL_ADDR_MATCH + 4, mask >> 32L);
1008
1009         write_and_test(pvt->pci_ch[pvt->inject.channel][0],
1010                                MC_CHANNEL_ERROR_MASK, pvt->inject.eccmask);
1011
1012         write_and_test(pvt->pci_ch[pvt->inject.channel][0],
1013                                MC_CHANNEL_ERROR_INJECT, injectmask);
1014
1015         /*
1016          * This is something undocumented, based on my tests
1017          * Without writing 8 to this register, errors aren't injected. Not sure
1018          * why.
1019          */
1020         pci_write_config_dword(pvt->pci_noncore,
1021                                MC_CFG_CONTROL, 8);
1022
1023         debugf0("Error inject addr match 0x%016llx, ecc 0x%08x,"
1024                 " inject 0x%08x\n",
1025                 mask, pvt->inject.eccmask, injectmask);
1026
1027
1028         return count;
1029 }
1030
1031 static ssize_t i7core_inject_enable_show(struct device *dev,
1032                                          struct device_attribute *mattr,
1033                                          char *data)
1034 {
1035         struct mem_ctl_info *mci = to_mci(dev);
1036         struct i7core_pvt *pvt = mci->pvt_info;
1037         u32 injectmask;
1038
1039         if (!pvt->pci_ch[pvt->inject.channel][0])
1040                 return 0;
1041
1042         pci_read_config_dword(pvt->pci_ch[pvt->inject.channel][0],
1043                                MC_CHANNEL_ERROR_INJECT, &injectmask);
1044
1045         debugf0("Inject error read: 0x%018x\n", injectmask);
1046
1047         if (injectmask & 0x0c)
1048                 pvt->inject.enable = 1;
1049
1050         return sprintf(data, "%d\n", pvt->inject.enable);
1051 }
1052
1053 #define DECLARE_COUNTER(param)                                  \
1054 static ssize_t i7core_show_counter_##param(                     \
1055         struct device *dev,                                     \
1056         struct device_attribute *mattr,                         \
1057         char *data)                                             \
1058 {                                                               \
1059         struct mem_ctl_info *mci = to_mci(dev);                 \
1060         struct i7core_pvt *pvt = mci->pvt_info;                 \
1061                                                                 \
1062         debugf1("%s()\n", __func__);                            \
1063         if (!pvt->ce_count_available || (pvt->is_registered))   \
1064                 return sprintf(data, "data unavailable\n");     \
1065         return sprintf(data, "%lu\n",                           \
1066                         pvt->udimm_ce_count[param]);            \
1067 }
1068
1069 #define ATTR_COUNTER(param)                                     \
1070         static DEVICE_ATTR(udimm##param, S_IRUGO | S_IWUSR,     \
1071                     i7core_show_counter_##param,                \
1072                     NULL)
1073
1074 DECLARE_COUNTER(0);
1075 DECLARE_COUNTER(1);
1076 DECLARE_COUNTER(2);
1077
1078 ATTR_COUNTER(0);
1079 ATTR_COUNTER(1);
1080 ATTR_COUNTER(2);
1081
1082 /*
1083  * inject_addrmatch device sysfs struct
1084  */
1085
1086 static struct attribute *i7core_addrmatch_attrs[] = {
1087         &dev_attr_channel.attr,
1088         &dev_attr_dimm.attr,
1089         &dev_attr_rank.attr,
1090         &dev_attr_bank.attr,
1091         &dev_attr_page.attr,
1092         &dev_attr_col.attr,
1093         NULL
1094 };
1095
1096 static struct attribute_group addrmatch_grp = {
1097         .attrs  = i7core_addrmatch_attrs,
1098 };
1099
1100 static const struct attribute_group *addrmatch_groups[] = {
1101         &addrmatch_grp,
1102         NULL
1103 };
1104
1105 static void addrmatch_release(struct device *device)
1106 {
1107         debugf1("Releasing device %s\n", dev_name(device));
1108         kfree(device);
1109 }
1110
1111 static struct device_type addrmatch_type = {
1112         .groups         = addrmatch_groups,
1113         .release        = addrmatch_release,
1114 };
1115
1116 /*
1117  * all_channel_counts sysfs struct
1118  */
1119
1120 static struct attribute *i7core_udimm_counters_attrs[] = {
1121         &dev_attr_udimm0.attr,
1122         &dev_attr_udimm1.attr,
1123         &dev_attr_udimm2.attr,
1124         NULL
1125 };
1126
1127 static struct attribute_group all_channel_counts_grp = {
1128         .attrs  = i7core_udimm_counters_attrs,
1129 };
1130
1131 static const struct attribute_group *all_channel_counts_groups[] = {
1132         &all_channel_counts_grp,
1133         NULL
1134 };
1135
1136 static void all_channel_counts_release(struct device *device)
1137 {
1138         debugf1("Releasing device %s\n", dev_name(device));
1139         kfree(device);
1140 }
1141
1142 static struct device_type all_channel_counts_type = {
1143         .groups         = all_channel_counts_groups,
1144         .release        = all_channel_counts_release,
1145 };
1146
1147 /*
1148  * inject sysfs attributes
1149  */
1150
1151 static DEVICE_ATTR(inject_section, S_IRUGO | S_IWUSR,
1152                    i7core_inject_section_show, i7core_inject_section_store);
1153
1154 static DEVICE_ATTR(inject_type, S_IRUGO | S_IWUSR,
1155                    i7core_inject_type_show, i7core_inject_type_store);
1156
1157
1158 static DEVICE_ATTR(inject_eccmask, S_IRUGO | S_IWUSR,
1159                    i7core_inject_eccmask_show, i7core_inject_eccmask_store);
1160
1161 static DEVICE_ATTR(inject_enable, S_IRUGO | S_IWUSR,
1162                    i7core_inject_enable_show, i7core_inject_enable_store);
1163
1164 static int i7core_create_sysfs_devices(struct mem_ctl_info *mci)
1165 {
1166         struct i7core_pvt *pvt = mci->pvt_info;
1167         int rc;
1168
1169         rc = device_create_file(&mci->dev, &dev_attr_inject_section);
1170         if (rc < 0)
1171                 return rc;
1172         rc = device_create_file(&mci->dev, &dev_attr_inject_type);
1173         if (rc < 0)
1174                 return rc;
1175         rc = device_create_file(&mci->dev, &dev_attr_inject_eccmask);
1176         if (rc < 0)
1177                 return rc;
1178         rc = device_create_file(&mci->dev, &dev_attr_inject_enable);
1179         if (rc < 0)
1180                 return rc;
1181
1182         pvt->addrmatch_dev = kzalloc(sizeof(*pvt->addrmatch_dev), GFP_KERNEL);
1183         if (!pvt->addrmatch_dev)
1184                 return rc;
1185
1186         pvt->addrmatch_dev->type = &addrmatch_type;
1187         pvt->addrmatch_dev->bus = mci->dev.bus;
1188         device_initialize(pvt->addrmatch_dev);
1189         pvt->addrmatch_dev->parent = &mci->dev;
1190         dev_set_name(pvt->addrmatch_dev, "inject_addrmatch");
1191         dev_set_drvdata(pvt->addrmatch_dev, mci);
1192
1193         debugf1("%s(): creating %s\n", __func__,
1194                 dev_name(pvt->addrmatch_dev));
1195
1196         rc = device_add(pvt->addrmatch_dev);
1197         if (rc < 0)
1198                 return rc;
1199
1200         if (!pvt->is_registered) {
1201                 pvt->chancounts_dev = kzalloc(sizeof(*pvt->chancounts_dev),
1202                                               GFP_KERNEL);
1203                 if (!pvt->chancounts_dev) {
1204                         put_device(pvt->addrmatch_dev);
1205                         device_del(pvt->addrmatch_dev);
1206                         return rc;
1207                 }
1208
1209                 pvt->chancounts_dev->type = &all_channel_counts_type;
1210                 pvt->chancounts_dev->bus = mci->dev.bus;
1211                 device_initialize(pvt->chancounts_dev);
1212                 pvt->chancounts_dev->parent = &mci->dev;
1213                 dev_set_name(pvt->chancounts_dev, "all_channel_counts");
1214                 dev_set_drvdata(pvt->chancounts_dev, mci);
1215
1216                 debugf1("%s(): creating %s\n", __func__,
1217                         dev_name(pvt->chancounts_dev));
1218
1219                 rc = device_add(pvt->chancounts_dev);
1220                 if (rc < 0)
1221                         return rc;
1222         }
1223         return 0;
1224 }
1225
1226 static void i7core_delete_sysfs_devices(struct mem_ctl_info *mci)
1227 {
1228         struct i7core_pvt *pvt = mci->pvt_info;
1229
1230         debugf1("\n");
1231
1232         device_remove_file(&mci->dev, &dev_attr_inject_section);
1233         device_remove_file(&mci->dev, &dev_attr_inject_type);
1234         device_remove_file(&mci->dev, &dev_attr_inject_eccmask);
1235         device_remove_file(&mci->dev, &dev_attr_inject_enable);
1236
1237         if (!pvt->is_registered) {
1238                 put_device(pvt->chancounts_dev);
1239                 device_del(pvt->chancounts_dev);
1240         }
1241         put_device(pvt->addrmatch_dev);
1242         device_del(pvt->addrmatch_dev);
1243 }
1244
1245 /****************************************************************************
1246         Device initialization routines: put/get, init/exit
1247  ****************************************************************************/
1248
1249 /*
1250  *      i7core_put_all_devices  'put' all the devices that we have
1251  *                              reserved via 'get'
1252  */
1253 static void i7core_put_devices(struct i7core_dev *i7core_dev)
1254 {
1255         int i;
1256
1257         debugf0(__FILE__ ": %s()\n", __func__);
1258         for (i = 0; i < i7core_dev->n_devs; i++) {
1259                 struct pci_dev *pdev = i7core_dev->pdev[i];
1260                 if (!pdev)
1261                         continue;
1262                 debugf0("Removing dev %02x:%02x.%d\n",
1263                         pdev->bus->number,
1264                         PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
1265                 pci_dev_put(pdev);
1266         }
1267 }
1268
1269 static void i7core_put_all_devices(void)
1270 {
1271         struct i7core_dev *i7core_dev, *tmp;
1272
1273         list_for_each_entry_safe(i7core_dev, tmp, &i7core_edac_list, list) {
1274                 i7core_put_devices(i7core_dev);
1275                 free_i7core_dev(i7core_dev);
1276         }
1277 }
1278
1279 static void __init i7core_xeon_pci_fixup(const struct pci_id_table *table)
1280 {
1281         struct pci_dev *pdev = NULL;
1282         int i;
1283
1284         /*
1285          * On Xeon 55xx, the Intel Quick Path Arch Generic Non-core pci buses
1286          * aren't announced by acpi. So, we need to use a legacy scan probing
1287          * to detect them
1288          */
1289         while (table && table->descr) {
1290                 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, table->descr[0].dev_id, NULL);
1291                 if (unlikely(!pdev)) {
1292                         for (i = 0; i < MAX_SOCKET_BUSES; i++)
1293                                 pcibios_scan_specific_bus(255-i);
1294                 }
1295                 pci_dev_put(pdev);
1296                 table++;
1297         }
1298 }
1299
1300 static unsigned i7core_pci_lastbus(void)
1301 {
1302         int last_bus = 0, bus;
1303         struct pci_bus *b = NULL;
1304
1305         while ((b = pci_find_next_bus(b)) != NULL) {
1306                 bus = b->number;
1307                 debugf0("Found bus %d\n", bus);
1308                 if (bus > last_bus)
1309                         last_bus = bus;
1310         }
1311
1312         debugf0("Last bus %d\n", last_bus);
1313
1314         return last_bus;
1315 }
1316
1317 /*
1318  *      i7core_get_all_devices  Find and perform 'get' operation on the MCH's
1319  *                      device/functions we want to reference for this driver
1320  *
1321  *                      Need to 'get' device 16 func 1 and func 2
1322  */
1323 static int i7core_get_onedevice(struct pci_dev **prev,
1324                                 const struct pci_id_table *table,
1325                                 const unsigned devno,
1326                                 const unsigned last_bus)
1327 {
1328         struct i7core_dev *i7core_dev;
1329         const struct pci_id_descr *dev_descr = &table->descr[devno];
1330
1331         struct pci_dev *pdev = NULL;
1332         u8 bus = 0;
1333         u8 socket = 0;
1334
1335         pdev = pci_get_device(PCI_VENDOR_ID_INTEL,
1336                               dev_descr->dev_id, *prev);
1337
1338         /*
1339          * On Xeon 55xx, the Intel Quckpath Arch Generic Non-core regs
1340          * is at addr 8086:2c40, instead of 8086:2c41. So, we need
1341          * to probe for the alternate address in case of failure
1342          */
1343         if (dev_descr->dev_id == PCI_DEVICE_ID_INTEL_I7_NONCORE && !pdev)
1344                 pdev = pci_get_device(PCI_VENDOR_ID_INTEL,
1345                                       PCI_DEVICE_ID_INTEL_I7_NONCORE_ALT, *prev);
1346
1347         if (dev_descr->dev_id == PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE && !pdev)
1348                 pdev = pci_get_device(PCI_VENDOR_ID_INTEL,
1349                                       PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE_ALT,
1350                                       *prev);
1351
1352         if (!pdev) {
1353                 if (*prev) {
1354                         *prev = pdev;
1355                         return 0;
1356                 }
1357
1358                 if (dev_descr->optional)
1359                         return 0;
1360
1361                 if (devno == 0)
1362                         return -ENODEV;
1363
1364                 i7core_printk(KERN_INFO,
1365                         "Device not found: dev %02x.%d PCI ID %04x:%04x\n",
1366                         dev_descr->dev, dev_descr->func,
1367                         PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
1368
1369                 /* End of list, leave */
1370                 return -ENODEV;
1371         }
1372         bus = pdev->bus->number;
1373
1374         socket = last_bus - bus;
1375
1376         i7core_dev = get_i7core_dev(socket);
1377         if (!i7core_dev) {
1378                 i7core_dev = alloc_i7core_dev(socket, table);
1379                 if (!i7core_dev) {
1380                         pci_dev_put(pdev);
1381                         return -ENOMEM;
1382                 }
1383         }
1384
1385         if (i7core_dev->pdev[devno]) {
1386                 i7core_printk(KERN_ERR,
1387                         "Duplicated device for "
1388                         "dev %02x:%02x.%d PCI ID %04x:%04x\n",
1389                         bus, dev_descr->dev, dev_descr->func,
1390                         PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
1391                 pci_dev_put(pdev);
1392                 return -ENODEV;
1393         }
1394
1395         i7core_dev->pdev[devno] = pdev;
1396
1397         /* Sanity check */
1398         if (unlikely(PCI_SLOT(pdev->devfn) != dev_descr->dev ||
1399                         PCI_FUNC(pdev->devfn) != dev_descr->func)) {
1400                 i7core_printk(KERN_ERR,
1401                         "Device PCI ID %04x:%04x "
1402                         "has dev %02x:%02x.%d instead of dev %02x:%02x.%d\n",
1403                         PCI_VENDOR_ID_INTEL, dev_descr->dev_id,
1404                         bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn),
1405                         bus, dev_descr->dev, dev_descr->func);
1406                 return -ENODEV;
1407         }
1408
1409         /* Be sure that the device is enabled */
1410         if (unlikely(pci_enable_device(pdev) < 0)) {
1411                 i7core_printk(KERN_ERR,
1412                         "Couldn't enable "
1413                         "dev %02x:%02x.%d PCI ID %04x:%04x\n",
1414                         bus, dev_descr->dev, dev_descr->func,
1415                         PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
1416                 return -ENODEV;
1417         }
1418
1419         debugf0("Detected socket %d dev %02x:%02x.%d PCI ID %04x:%04x\n",
1420                 socket, bus, dev_descr->dev,
1421                 dev_descr->func,
1422                 PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
1423
1424         /*
1425          * As stated on drivers/pci/search.c, the reference count for
1426          * @from is always decremented if it is not %NULL. So, as we need
1427          * to get all devices up to null, we need to do a get for the device
1428          */
1429         pci_dev_get(pdev);
1430
1431         *prev = pdev;
1432
1433         return 0;
1434 }
1435
1436 static int i7core_get_all_devices(void)
1437 {
1438         int i, rc, last_bus;
1439         struct pci_dev *pdev = NULL;
1440         const struct pci_id_table *table = pci_dev_table;
1441
1442         last_bus = i7core_pci_lastbus();
1443
1444         while (table && table->descr) {
1445                 for (i = 0; i < table->n_devs; i++) {
1446                         pdev = NULL;
1447                         do {
1448                                 rc = i7core_get_onedevice(&pdev, table, i,
1449                                                           last_bus);
1450                                 if (rc < 0) {
1451                                         if (i == 0) {
1452                                                 i = table->n_devs;
1453                                                 break;
1454                                         }
1455                                         i7core_put_all_devices();
1456                                         return -ENODEV;
1457                                 }
1458                         } while (pdev);
1459                 }
1460                 table++;
1461         }
1462
1463         return 0;
1464 }
1465
1466 static int mci_bind_devs(struct mem_ctl_info *mci,
1467                          struct i7core_dev *i7core_dev)
1468 {
1469         struct i7core_pvt *pvt = mci->pvt_info;
1470         struct pci_dev *pdev;
1471         int i, func, slot;
1472         char *family;
1473
1474         pvt->is_registered = false;
1475         pvt->enable_scrub  = false;
1476         for (i = 0; i < i7core_dev->n_devs; i++) {
1477                 pdev = i7core_dev->pdev[i];
1478                 if (!pdev)
1479                         continue;
1480
1481                 func = PCI_FUNC(pdev->devfn);
1482                 slot = PCI_SLOT(pdev->devfn);
1483                 if (slot == 3) {
1484                         if (unlikely(func > MAX_MCR_FUNC))
1485                                 goto error;
1486                         pvt->pci_mcr[func] = pdev;
1487                 } else if (likely(slot >= 4 && slot < 4 + NUM_CHANS)) {
1488                         if (unlikely(func > MAX_CHAN_FUNC))
1489                                 goto error;
1490                         pvt->pci_ch[slot - 4][func] = pdev;
1491                 } else if (!slot && !func) {
1492                         pvt->pci_noncore = pdev;
1493
1494                         /* Detect the processor family */
1495                         switch (pdev->device) {
1496                         case PCI_DEVICE_ID_INTEL_I7_NONCORE:
1497                                 family = "Xeon 35xx/ i7core";
1498                                 pvt->enable_scrub = false;
1499                                 break;
1500                         case PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE_ALT:
1501                                 family = "i7-800/i5-700";
1502                                 pvt->enable_scrub = false;
1503                                 break;
1504                         case PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE:
1505                                 family = "Xeon 34xx";
1506                                 pvt->enable_scrub = false;
1507                                 break;
1508                         case PCI_DEVICE_ID_INTEL_I7_NONCORE_ALT:
1509                                 family = "Xeon 55xx";
1510                                 pvt->enable_scrub = true;
1511                                 break;
1512                         case PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE_REV2:
1513                                 family = "Xeon 56xx / i7-900";
1514                                 pvt->enable_scrub = true;
1515                                 break;
1516                         default:
1517                                 family = "unknown";
1518                                 pvt->enable_scrub = false;
1519                         }
1520                         debugf0("Detected a processor type %s\n", family);
1521                 } else
1522                         goto error;
1523
1524                 debugf0("Associated fn %d.%d, dev = %p, socket %d\n",
1525                         PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn),
1526                         pdev, i7core_dev->socket);
1527
1528                 if (PCI_SLOT(pdev->devfn) == 3 &&
1529                         PCI_FUNC(pdev->devfn) == 2)
1530                         pvt->is_registered = true;
1531         }
1532
1533         return 0;
1534
1535 error:
1536         i7core_printk(KERN_ERR, "Device %d, function %d "
1537                       "is out of the expected range\n",
1538                       slot, func);
1539         return -EINVAL;
1540 }
1541
1542 /****************************************************************************
1543                         Error check routines
1544  ****************************************************************************/
1545 static void i7core_rdimm_update_errcount(struct mem_ctl_info *mci,
1546                                       const int chan,
1547                                       const int dimm,
1548                                       const int add)
1549 {
1550         int i;
1551
1552         for (i = 0; i < add; i++) {
1553                 edac_mc_handle_error(HW_EVENT_ERR_CORRECTED, mci, 0, 0, 0,
1554                                      chan, dimm, -1, "error", "", NULL);
1555         }
1556 }
1557
1558 static void i7core_rdimm_update_ce_count(struct mem_ctl_info *mci,
1559                                          const int chan,
1560                                          const int new0,
1561                                          const int new1,
1562                                          const int new2)
1563 {
1564         struct i7core_pvt *pvt = mci->pvt_info;
1565         int add0 = 0, add1 = 0, add2 = 0;
1566         /* Updates CE counters if it is not the first time here */
1567         if (pvt->ce_count_available) {
1568                 /* Updates CE counters */
1569
1570                 add2 = new2 - pvt->rdimm_last_ce_count[chan][2];
1571                 add1 = new1 - pvt->rdimm_last_ce_count[chan][1];
1572                 add0 = new0 - pvt->rdimm_last_ce_count[chan][0];
1573
1574                 if (add2 < 0)
1575                         add2 += 0x7fff;
1576                 pvt->rdimm_ce_count[chan][2] += add2;
1577
1578                 if (add1 < 0)
1579                         add1 += 0x7fff;
1580                 pvt->rdimm_ce_count[chan][1] += add1;
1581
1582                 if (add0 < 0)
1583                         add0 += 0x7fff;
1584                 pvt->rdimm_ce_count[chan][0] += add0;
1585         } else
1586                 pvt->ce_count_available = 1;
1587
1588         /* Store the new values */
1589         pvt->rdimm_last_ce_count[chan][2] = new2;
1590         pvt->rdimm_last_ce_count[chan][1] = new1;
1591         pvt->rdimm_last_ce_count[chan][0] = new0;
1592
1593         /*updated the edac core */
1594         if (add0 != 0)
1595                 i7core_rdimm_update_errcount(mci, chan, 0, add0);
1596         if (add1 != 0)
1597                 i7core_rdimm_update_errcount(mci, chan, 1, add1);
1598         if (add2 != 0)
1599                 i7core_rdimm_update_errcount(mci, chan, 2, add2);
1600
1601 }
1602
1603 static void i7core_rdimm_check_mc_ecc_err(struct mem_ctl_info *mci)
1604 {
1605         struct i7core_pvt *pvt = mci->pvt_info;
1606         u32 rcv[3][2];
1607         int i, new0, new1, new2;
1608
1609         /*Read DEV 3: FUN 2:  MC_COR_ECC_CNT regs directly*/
1610         pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_0,
1611                                                                 &rcv[0][0]);
1612         pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_1,
1613                                                                 &rcv[0][1]);
1614         pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_2,
1615                                                                 &rcv[1][0]);
1616         pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_3,
1617                                                                 &rcv[1][1]);
1618         pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_4,
1619                                                                 &rcv[2][0]);
1620         pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_5,
1621                                                                 &rcv[2][1]);
1622         for (i = 0 ; i < 3; i++) {
1623                 debugf3("MC_COR_ECC_CNT%d = 0x%x; MC_COR_ECC_CNT%d = 0x%x\n",
1624                         (i * 2), rcv[i][0], (i * 2) + 1, rcv[i][1]);
1625                 /*if the channel has 3 dimms*/
1626                 if (pvt->channel[i].dimms > 2) {
1627                         new0 = DIMM_BOT_COR_ERR(rcv[i][0]);
1628                         new1 = DIMM_TOP_COR_ERR(rcv[i][0]);
1629                         new2 = DIMM_BOT_COR_ERR(rcv[i][1]);
1630                 } else {
1631                         new0 = DIMM_TOP_COR_ERR(rcv[i][0]) +
1632                                         DIMM_BOT_COR_ERR(rcv[i][0]);
1633                         new1 = DIMM_TOP_COR_ERR(rcv[i][1]) +
1634                                         DIMM_BOT_COR_ERR(rcv[i][1]);
1635                         new2 = 0;
1636                 }
1637
1638                 i7core_rdimm_update_ce_count(mci, i, new0, new1, new2);
1639         }
1640 }
1641
1642 /* This function is based on the device 3 function 4 registers as described on:
1643  * Intel Xeon Processor 5500 Series Datasheet Volume 2
1644  *      http://www.intel.com/Assets/PDF/datasheet/321322.pdf
1645  * also available at:
1646  *      http://www.arrownac.com/manufacturers/intel/s/nehalem/5500-datasheet-v2.pdf
1647  */
1648 static void i7core_udimm_check_mc_ecc_err(struct mem_ctl_info *mci)
1649 {
1650         struct i7core_pvt *pvt = mci->pvt_info;
1651         u32 rcv1, rcv0;
1652         int new0, new1, new2;
1653
1654         if (!pvt->pci_mcr[4]) {
1655                 debugf0("%s MCR registers not found\n", __func__);
1656                 return;
1657         }
1658
1659         /* Corrected test errors */
1660         pci_read_config_dword(pvt->pci_mcr[4], MC_TEST_ERR_RCV1, &rcv1);
1661         pci_read_config_dword(pvt->pci_mcr[4], MC_TEST_ERR_RCV0, &rcv0);
1662
1663         /* Store the new values */
1664         new2 = DIMM2_COR_ERR(rcv1);
1665         new1 = DIMM1_COR_ERR(rcv0);
1666         new0 = DIMM0_COR_ERR(rcv0);
1667
1668         /* Updates CE counters if it is not the first time here */
1669         if (pvt->ce_count_available) {
1670                 /* Updates CE counters */
1671                 int add0, add1, add2;
1672
1673                 add2 = new2 - pvt->udimm_last_ce_count[2];
1674                 add1 = new1 - pvt->udimm_last_ce_count[1];
1675                 add0 = new0 - pvt->udimm_last_ce_count[0];
1676
1677                 if (add2 < 0)
1678                         add2 += 0x7fff;
1679                 pvt->udimm_ce_count[2] += add2;
1680
1681                 if (add1 < 0)
1682                         add1 += 0x7fff;
1683                 pvt->udimm_ce_count[1] += add1;
1684
1685                 if (add0 < 0)
1686                         add0 += 0x7fff;
1687                 pvt->udimm_ce_count[0] += add0;
1688
1689                 if (add0 | add1 | add2)
1690                         i7core_printk(KERN_ERR, "New Corrected error(s): "
1691                                       "dimm0: +%d, dimm1: +%d, dimm2 +%d\n",
1692                                       add0, add1, add2);
1693         } else
1694                 pvt->ce_count_available = 1;
1695
1696         /* Store the new values */
1697         pvt->udimm_last_ce_count[2] = new2;
1698         pvt->udimm_last_ce_count[1] = new1;
1699         pvt->udimm_last_ce_count[0] = new0;
1700 }
1701
1702 /*
1703  * According with tables E-11 and E-12 of chapter E.3.3 of Intel 64 and IA-32
1704  * Architectures Software Developer’s Manual Volume 3B.
1705  * Nehalem are defined as family 0x06, model 0x1a
1706  *
1707  * The MCA registers used here are the following ones:
1708  *     struct mce field MCA Register
1709  *     m->status        MSR_IA32_MC8_STATUS
1710  *     m->addr          MSR_IA32_MC8_ADDR
1711  *     m->misc          MSR_IA32_MC8_MISC
1712  * In the case of Nehalem, the error information is masked at .status and .misc
1713  * fields
1714  */
1715 static void i7core_mce_output_error(struct mem_ctl_info *mci,
1716                                     const struct mce *m)
1717 {
1718         struct i7core_pvt *pvt = mci->pvt_info;
1719         char *type, *optype, *err, msg[80];
1720         enum hw_event_mc_err_type tp_event;
1721         unsigned long error = m->status & 0x1ff0000l;
1722         bool uncorrected_error = m->mcgstatus & 1ll << 61;
1723         bool ripv = m->mcgstatus & 1;
1724         u32 optypenum = (m->status >> 4) & 0x07;
1725         u32 core_err_cnt = (m->status >> 38) & 0x7fff;
1726         u32 dimm = (m->misc >> 16) & 0x3;
1727         u32 channel = (m->misc >> 18) & 0x3;
1728         u32 syndrome = m->misc >> 32;
1729         u32 errnum = find_first_bit(&error, 32);
1730
1731         if (uncorrected_error) {
1732                 if (ripv) {
1733                         type = "FATAL";
1734                         tp_event = HW_EVENT_ERR_FATAL;
1735                 } else {
1736                         type = "NON_FATAL";
1737                         tp_event = HW_EVENT_ERR_UNCORRECTED;
1738                 }
1739         } else {
1740                 type = "CORRECTED";
1741                 tp_event = HW_EVENT_ERR_CORRECTED;
1742         }
1743
1744         switch (optypenum) {
1745         case 0:
1746                 optype = "generic undef request";
1747                 break;
1748         case 1:
1749                 optype = "read error";
1750                 break;
1751         case 2:
1752                 optype = "write error";
1753                 break;
1754         case 3:
1755                 optype = "addr/cmd error";
1756                 break;
1757         case 4:
1758                 optype = "scrubbing error";
1759                 break;
1760         default:
1761                 optype = "reserved";
1762                 break;
1763         }
1764
1765         switch (errnum) {
1766         case 16:
1767                 err = "read ECC error";
1768                 break;
1769         case 17:
1770                 err = "RAS ECC error";
1771                 break;
1772         case 18:
1773                 err = "write parity error";
1774                 break;
1775         case 19:
1776                 err = "redundacy loss";
1777                 break;
1778         case 20:
1779                 err = "reserved";
1780                 break;
1781         case 21:
1782                 err = "memory range error";
1783                 break;
1784         case 22:
1785                 err = "RTID out of range";
1786                 break;
1787         case 23:
1788                 err = "address parity error";
1789                 break;
1790         case 24:
1791                 err = "byte enable parity error";
1792                 break;
1793         default:
1794                 err = "unknown";
1795         }
1796
1797         snprintf(msg, sizeof(msg), "count=%d %s", core_err_cnt, optype);
1798
1799         /*
1800          * Call the helper to output message
1801          * FIXME: what to do if core_err_cnt > 1? Currently, it generates
1802          * only one event
1803          */
1804         if (uncorrected_error || !pvt->is_registered)
1805                 edac_mc_handle_error(tp_event, mci,
1806                                      m->addr >> PAGE_SHIFT,
1807                                      m->addr & ~PAGE_MASK,
1808                                      syndrome,
1809                                      channel, dimm, -1,
1810                                      err, msg, m);
1811 }
1812
1813 /*
1814  *      i7core_check_error      Retrieve and process errors reported by the
1815  *                              hardware. Called by the Core module.
1816  */
1817 static void i7core_check_error(struct mem_ctl_info *mci)
1818 {
1819         struct i7core_pvt *pvt = mci->pvt_info;
1820         int i;
1821         unsigned count = 0;
1822         struct mce *m;
1823
1824         /*
1825          * MCE first step: Copy all mce errors into a temporary buffer
1826          * We use a double buffering here, to reduce the risk of
1827          * losing an error.
1828          */
1829         smp_rmb();
1830         count = (pvt->mce_out + MCE_LOG_LEN - pvt->mce_in)
1831                 % MCE_LOG_LEN;
1832         if (!count)
1833                 goto check_ce_error;
1834
1835         m = pvt->mce_outentry;
1836         if (pvt->mce_in + count > MCE_LOG_LEN) {
1837                 unsigned l = MCE_LOG_LEN - pvt->mce_in;
1838
1839                 memcpy(m, &pvt->mce_entry[pvt->mce_in], sizeof(*m) * l);
1840                 smp_wmb();
1841                 pvt->mce_in = 0;
1842                 count -= l;
1843                 m += l;
1844         }
1845         memcpy(m, &pvt->mce_entry[pvt->mce_in], sizeof(*m) * count);
1846         smp_wmb();
1847         pvt->mce_in += count;
1848
1849         smp_rmb();
1850         if (pvt->mce_overrun) {
1851                 i7core_printk(KERN_ERR, "Lost %d memory errors\n",
1852                               pvt->mce_overrun);
1853                 smp_wmb();
1854                 pvt->mce_overrun = 0;
1855         }
1856
1857         /*
1858          * MCE second step: parse errors and display
1859          */
1860         for (i = 0; i < count; i++)
1861                 i7core_mce_output_error(mci, &pvt->mce_outentry[i]);
1862
1863         /*
1864          * Now, let's increment CE error counts
1865          */
1866 check_ce_error:
1867         if (!pvt->is_registered)
1868                 i7core_udimm_check_mc_ecc_err(mci);
1869         else
1870                 i7core_rdimm_check_mc_ecc_err(mci);
1871 }
1872
1873 /*
1874  * i7core_mce_check_error       Replicates mcelog routine to get errors
1875  *                              This routine simply queues mcelog errors, and
1876  *                              return. The error itself should be handled later
1877  *                              by i7core_check_error.
1878  * WARNING: As this routine should be called at NMI time, extra care should
1879  * be taken to avoid deadlocks, and to be as fast as possible.
1880  */
1881 static int i7core_mce_check_error(struct notifier_block *nb, unsigned long val,
1882                                   void *data)
1883 {
1884         struct mce *mce = (struct mce *)data;
1885         struct i7core_dev *i7_dev;
1886         struct mem_ctl_info *mci;
1887         struct i7core_pvt *pvt;
1888
1889         i7_dev = get_i7core_dev(mce->socketid);
1890         if (!i7_dev)
1891                 return NOTIFY_BAD;
1892
1893         mci = i7_dev->mci;
1894         pvt = mci->pvt_info;
1895
1896         /*
1897          * Just let mcelog handle it if the error is
1898          * outside the memory controller
1899          */
1900         if (((mce->status & 0xffff) >> 7) != 1)
1901                 return NOTIFY_DONE;
1902
1903         /* Bank 8 registers are the only ones that we know how to handle */
1904         if (mce->bank != 8)
1905                 return NOTIFY_DONE;
1906
1907 #ifdef CONFIG_SMP
1908         /* Only handle if it is the right mc controller */
1909         if (mce->socketid != pvt->i7core_dev->socket)
1910                 return NOTIFY_DONE;
1911 #endif
1912
1913         smp_rmb();
1914         if ((pvt->mce_out + 1) % MCE_LOG_LEN == pvt->mce_in) {
1915                 smp_wmb();
1916                 pvt->mce_overrun++;
1917                 return NOTIFY_DONE;
1918         }
1919
1920         /* Copy memory error at the ringbuffer */
1921         memcpy(&pvt->mce_entry[pvt->mce_out], mce, sizeof(*mce));
1922         smp_wmb();
1923         pvt->mce_out = (pvt->mce_out + 1) % MCE_LOG_LEN;
1924
1925         /* Handle fatal errors immediately */
1926         if (mce->mcgstatus & 1)
1927                 i7core_check_error(mci);
1928
1929         /* Advise mcelog that the errors were handled */
1930         return NOTIFY_STOP;
1931 }
1932
1933 static struct notifier_block i7_mce_dec = {
1934         .notifier_call  = i7core_mce_check_error,
1935 };
1936
1937 struct memdev_dmi_entry {
1938         u8 type;
1939         u8 length;
1940         u16 handle;
1941         u16 phys_mem_array_handle;
1942         u16 mem_err_info_handle;
1943         u16 total_width;
1944         u16 data_width;
1945         u16 size;
1946         u8 form;
1947         u8 device_set;
1948         u8 device_locator;
1949         u8 bank_locator;
1950         u8 memory_type;
1951         u16 type_detail;
1952         u16 speed;
1953         u8 manufacturer;
1954         u8 serial_number;
1955         u8 asset_tag;
1956         u8 part_number;
1957         u8 attributes;
1958         u32 extended_size;
1959         u16 conf_mem_clk_speed;
1960 } __attribute__((__packed__));
1961
1962
1963 /*
1964  * Decode the DRAM Clock Frequency, be paranoid, make sure that all
1965  * memory devices show the same speed, and if they don't then consider
1966  * all speeds to be invalid.
1967  */
1968 static void decode_dclk(const struct dmi_header *dh, void *_dclk_freq)
1969 {
1970         int *dclk_freq = _dclk_freq;
1971         u16 dmi_mem_clk_speed;
1972
1973         if (*dclk_freq == -1)
1974                 return;
1975
1976         if (dh->type == DMI_ENTRY_MEM_DEVICE) {
1977                 struct memdev_dmi_entry *memdev_dmi_entry =
1978                         (struct memdev_dmi_entry *)dh;
1979                 unsigned long conf_mem_clk_speed_offset =
1980                         (unsigned long)&memdev_dmi_entry->conf_mem_clk_speed -
1981                         (unsigned long)&memdev_dmi_entry->type;
1982                 unsigned long speed_offset =
1983                         (unsigned long)&memdev_dmi_entry->speed -
1984                         (unsigned long)&memdev_dmi_entry->type;
1985
1986                 /* Check that a DIMM is present */
1987                 if (memdev_dmi_entry->size == 0)
1988                         return;
1989
1990                 /*
1991                  * Pick the configured speed if it's available, otherwise
1992                  * pick the DIMM speed, or we don't have a speed.
1993                  */
1994                 if (memdev_dmi_entry->length > conf_mem_clk_speed_offset) {
1995                         dmi_mem_clk_speed =
1996                                 memdev_dmi_entry->conf_mem_clk_speed;
1997                 } else if (memdev_dmi_entry->length > speed_offset) {
1998                         dmi_mem_clk_speed = memdev_dmi_entry->speed;
1999                 } else {
2000                         *dclk_freq = -1;
2001                         return;
2002                 }
2003
2004                 if (*dclk_freq == 0) {
2005                         /* First pass, speed was 0 */
2006                         if (dmi_mem_clk_speed > 0) {
2007                                 /* Set speed if a valid speed is read */
2008                                 *dclk_freq = dmi_mem_clk_speed;
2009                         } else {
2010                                 /* Otherwise we don't have a valid speed */
2011                                 *dclk_freq = -1;
2012                         }
2013                 } else if (*dclk_freq > 0 &&
2014                            *dclk_freq != dmi_mem_clk_speed) {
2015                         /*
2016                          * If we have a speed, check that all DIMMS are the same
2017                          * speed, otherwise set the speed as invalid.
2018                          */
2019                         *dclk_freq = -1;
2020                 }
2021         }
2022 }
2023
2024 /*
2025  * The default DCLK frequency is used as a fallback if we
2026  * fail to find anything reliable in the DMI. The value
2027  * is taken straight from the datasheet.
2028  */
2029 #define DEFAULT_DCLK_FREQ 800
2030
2031 static int get_dclk_freq(void)
2032 {
2033         int dclk_freq = 0;
2034
2035         dmi_walk(decode_dclk, (void *)&dclk_freq);
2036
2037         if (dclk_freq < 1)
2038                 return DEFAULT_DCLK_FREQ;
2039
2040         return dclk_freq;
2041 }
2042
2043 /*
2044  * set_sdram_scrub_rate         This routine sets byte/sec bandwidth scrub rate
2045  *                              to hardware according to SCRUBINTERVAL formula
2046  *                              found in datasheet.
2047  */
2048 static int set_sdram_scrub_rate(struct mem_ctl_info *mci, u32 new_bw)
2049 {
2050         struct i7core_pvt *pvt = mci->pvt_info;
2051         struct pci_dev *pdev;
2052         u32 dw_scrub;
2053         u32 dw_ssr;
2054
2055         /* Get data from the MC register, function 2 */
2056         pdev = pvt->pci_mcr[2];
2057         if (!pdev)
2058                 return -ENODEV;
2059
2060         pci_read_config_dword(pdev, MC_SCRUB_CONTROL, &dw_scrub);
2061
2062         if (new_bw == 0) {
2063                 /* Prepare to disable petrol scrub */
2064                 dw_scrub &= ~STARTSCRUB;
2065                 /* Stop the patrol scrub engine */
2066                 write_and_test(pdev, MC_SCRUB_CONTROL,
2067                                dw_scrub & ~SCRUBINTERVAL_MASK);
2068
2069                 /* Get current status of scrub rate and set bit to disable */
2070                 pci_read_config_dword(pdev, MC_SSRCONTROL, &dw_ssr);
2071                 dw_ssr &= ~SSR_MODE_MASK;
2072                 dw_ssr |= SSR_MODE_DISABLE;
2073         } else {
2074                 const int cache_line_size = 64;
2075                 const u32 freq_dclk_mhz = pvt->dclk_freq;
2076                 unsigned long long scrub_interval;
2077                 /*
2078                  * Translate the desired scrub rate to a register value and
2079                  * program the corresponding register value.
2080                  */
2081                 scrub_interval = (unsigned long long)freq_dclk_mhz *
2082                         cache_line_size * 1000000;
2083                 do_div(scrub_interval, new_bw);
2084
2085                 if (!scrub_interval || scrub_interval > SCRUBINTERVAL_MASK)
2086                         return -EINVAL;
2087
2088                 dw_scrub = SCRUBINTERVAL_MASK & scrub_interval;
2089
2090                 /* Start the patrol scrub engine */
2091                 pci_write_config_dword(pdev, MC_SCRUB_CONTROL,
2092                                        STARTSCRUB | dw_scrub);
2093
2094                 /* Get current status of scrub rate and set bit to enable */
2095                 pci_read_config_dword(pdev, MC_SSRCONTROL, &dw_ssr);
2096                 dw_ssr &= ~SSR_MODE_MASK;
2097                 dw_ssr |= SSR_MODE_ENABLE;
2098         }
2099         /* Disable or enable scrubbing */
2100         pci_write_config_dword(pdev, MC_SSRCONTROL, dw_ssr);
2101
2102         return new_bw;
2103 }
2104
2105 /*
2106  * get_sdram_scrub_rate         This routine convert current scrub rate value
2107  *                              into byte/sec bandwidth accourding to
2108  *                              SCRUBINTERVAL formula found in datasheet.
2109  */
2110 static int get_sdram_scrub_rate(struct mem_ctl_info *mci)
2111 {
2112         struct i7core_pvt *pvt = mci->pvt_info;
2113         struct pci_dev *pdev;
2114         const u32 cache_line_size = 64;
2115         const u32 freq_dclk_mhz = pvt->dclk_freq;
2116         unsigned long long scrub_rate;
2117         u32 scrubval;
2118
2119         /* Get data from the MC register, function 2 */
2120         pdev = pvt->pci_mcr[2];
2121         if (!pdev)
2122                 return -ENODEV;
2123
2124         /* Get current scrub control data */
2125         pci_read_config_dword(pdev, MC_SCRUB_CONTROL, &scrubval);
2126
2127         /* Mask highest 8-bits to 0 */
2128         scrubval &=  SCRUBINTERVAL_MASK;
2129         if (!scrubval)
2130                 return 0;
2131
2132         /* Calculate scrub rate value into byte/sec bandwidth */
2133         scrub_rate =  (unsigned long long)freq_dclk_mhz *
2134                 1000000 * cache_line_size;
2135         do_div(scrub_rate, scrubval);
2136         return (int)scrub_rate;
2137 }
2138
2139 static void enable_sdram_scrub_setting(struct mem_ctl_info *mci)
2140 {
2141         struct i7core_pvt *pvt = mci->pvt_info;
2142         u32 pci_lock;
2143
2144         /* Unlock writes to pci registers */
2145         pci_read_config_dword(pvt->pci_noncore, MC_CFG_CONTROL, &pci_lock);
2146         pci_lock &= ~0x3;
2147         pci_write_config_dword(pvt->pci_noncore, MC_CFG_CONTROL,
2148                                pci_lock | MC_CFG_UNLOCK);
2149
2150         mci->set_sdram_scrub_rate = set_sdram_scrub_rate;
2151         mci->get_sdram_scrub_rate = get_sdram_scrub_rate;
2152 }
2153
2154 static void disable_sdram_scrub_setting(struct mem_ctl_info *mci)
2155 {
2156         struct i7core_pvt *pvt = mci->pvt_info;
2157         u32 pci_lock;
2158
2159         /* Lock writes to pci registers */
2160         pci_read_config_dword(pvt->pci_noncore, MC_CFG_CONTROL, &pci_lock);
2161         pci_lock &= ~0x3;
2162         pci_write_config_dword(pvt->pci_noncore, MC_CFG_CONTROL,
2163                                pci_lock | MC_CFG_LOCK);
2164 }
2165
2166 static void i7core_pci_ctl_create(struct i7core_pvt *pvt)
2167 {
2168         pvt->i7core_pci = edac_pci_create_generic_ctl(
2169                                                 &pvt->i7core_dev->pdev[0]->dev,
2170                                                 EDAC_MOD_STR);
2171         if (unlikely(!pvt->i7core_pci))
2172                 i7core_printk(KERN_WARNING,
2173                               "Unable to setup PCI error report via EDAC\n");
2174 }
2175
2176 static void i7core_pci_ctl_release(struct i7core_pvt *pvt)
2177 {
2178         if (likely(pvt->i7core_pci))
2179                 edac_pci_release_generic_ctl(pvt->i7core_pci);
2180         else
2181                 i7core_printk(KERN_ERR,
2182                                 "Couldn't find mem_ctl_info for socket %d\n",
2183                                 pvt->i7core_dev->socket);
2184         pvt->i7core_pci = NULL;
2185 }
2186
2187 static void i7core_unregister_mci(struct i7core_dev *i7core_dev)
2188 {
2189         struct mem_ctl_info *mci = i7core_dev->mci;
2190         struct i7core_pvt *pvt;
2191
2192         if (unlikely(!mci || !mci->pvt_info)) {
2193                 debugf0("MC: " __FILE__ ": %s(): dev = %p\n",
2194                         __func__, &i7core_dev->pdev[0]->dev);
2195
2196                 i7core_printk(KERN_ERR, "Couldn't find mci handler\n");
2197                 return;
2198         }
2199
2200         pvt = mci->pvt_info;
2201
2202         debugf0("MC: " __FILE__ ": %s(): mci = %p, dev = %p\n",
2203                 __func__, mci, &i7core_dev->pdev[0]->dev);
2204
2205         /* Disable scrubrate setting */
2206         if (pvt->enable_scrub)
2207                 disable_sdram_scrub_setting(mci);
2208
2209         mce_unregister_decode_chain(&i7_mce_dec);
2210
2211         /* Disable EDAC polling */
2212         i7core_pci_ctl_release(pvt);
2213
2214         /* Remove MC sysfs nodes */
2215         i7core_delete_sysfs_devices(mci);
2216         edac_mc_del_mc(mci->pdev);
2217
2218         debugf1("%s: free mci struct\n", mci->ctl_name);
2219         kfree(mci->ctl_name);
2220         edac_mc_free(mci);
2221         i7core_dev->mci = NULL;
2222 }
2223
2224 static int i7core_register_mci(struct i7core_dev *i7core_dev)
2225 {
2226         struct mem_ctl_info *mci;
2227         struct i7core_pvt *pvt;
2228         int rc;
2229         struct edac_mc_layer layers[2];
2230
2231         /* allocate a new MC control structure */
2232
2233         layers[0].type = EDAC_MC_LAYER_CHANNEL;
2234         layers[0].size = NUM_CHANS;
2235         layers[0].is_virt_csrow = false;
2236         layers[1].type = EDAC_MC_LAYER_SLOT;
2237         layers[1].size = MAX_DIMMS;
2238         layers[1].is_virt_csrow = true;
2239         mci = edac_mc_alloc(i7core_dev->socket, ARRAY_SIZE(layers), layers,
2240                             sizeof(*pvt));
2241         if (unlikely(!mci))
2242                 return -ENOMEM;
2243
2244         debugf0("MC: " __FILE__ ": %s(): mci = %p, dev = %p\n",
2245                 __func__, mci, &i7core_dev->pdev[0]->dev);
2246
2247         pvt = mci->pvt_info;
2248         memset(pvt, 0, sizeof(*pvt));
2249
2250         /* Associates i7core_dev and mci for future usage */
2251         pvt->i7core_dev = i7core_dev;
2252         i7core_dev->mci = mci;
2253
2254         /*
2255          * FIXME: how to handle RDDR3 at MCI level? It is possible to have
2256          * Mixed RDDR3/UDDR3 with Nehalem, provided that they are on different
2257          * memory channels
2258          */
2259         mci->mtype_cap = MEM_FLAG_DDR3;
2260         mci->edac_ctl_cap = EDAC_FLAG_NONE;
2261         mci->edac_cap = EDAC_FLAG_NONE;
2262         mci->mod_name = "i7core_edac.c";
2263         mci->mod_ver = I7CORE_REVISION;
2264         mci->ctl_name = kasprintf(GFP_KERNEL, "i7 core #%d",
2265                                   i7core_dev->socket);
2266         mci->dev_name = pci_name(i7core_dev->pdev[0]);
2267         mci->ctl_page_to_phys = NULL;
2268
2269         /* Store pci devices at mci for faster access */
2270         rc = mci_bind_devs(mci, i7core_dev);
2271         if (unlikely(rc < 0))
2272                 goto fail0;
2273
2274
2275         /* Get dimm basic config */
2276         get_dimm_config(mci);
2277         /* record ptr to the generic device */
2278         mci->pdev = &i7core_dev->pdev[0]->dev;
2279         /* Set the function pointer to an actual operation function */
2280         mci->edac_check = i7core_check_error;
2281
2282         /* Enable scrubrate setting */
2283         if (pvt->enable_scrub)
2284                 enable_sdram_scrub_setting(mci);
2285
2286         /* add this new MC control structure to EDAC's list of MCs */
2287         if (unlikely(edac_mc_add_mc(mci))) {
2288                 debugf0("MC: " __FILE__
2289                         ": %s(): failed edac_mc_add_mc()\n", __func__);
2290                 /* FIXME: perhaps some code should go here that disables error
2291                  * reporting if we just enabled it
2292                  */
2293
2294                 rc = -EINVAL;
2295                 goto fail0;
2296         }
2297         if (i7core_create_sysfs_devices(mci)) {
2298                 debugf0("MC: " __FILE__
2299                         ": %s(): failed to create sysfs nodes\n", __func__);
2300                 edac_mc_del_mc(mci->pdev);
2301                 rc = -EINVAL;
2302                 goto fail0;
2303         }
2304
2305         /* Default error mask is any memory */
2306         pvt->inject.channel = 0;
2307         pvt->inject.dimm = -1;
2308         pvt->inject.rank = -1;
2309         pvt->inject.bank = -1;
2310         pvt->inject.page = -1;
2311         pvt->inject.col = -1;
2312
2313         /* allocating generic PCI control info */
2314         i7core_pci_ctl_create(pvt);
2315
2316         /* DCLK for scrub rate setting */
2317         pvt->dclk_freq = get_dclk_freq();
2318
2319         mce_register_decode_chain(&i7_mce_dec);
2320
2321         return 0;
2322
2323 fail0:
2324         kfree(mci->ctl_name);
2325         edac_mc_free(mci);
2326         i7core_dev->mci = NULL;
2327         return rc;
2328 }
2329
2330 /*
2331  *      i7core_probe    Probe for ONE instance of device to see if it is
2332  *                      present.
2333  *      return:
2334  *              0 for FOUND a device
2335  *              < 0 for error code
2336  */
2337
2338 static int __devinit i7core_probe(struct pci_dev *pdev,
2339                                   const struct pci_device_id *id)
2340 {
2341         int rc, count = 0;
2342         struct i7core_dev *i7core_dev;
2343
2344         /* get the pci devices we want to reserve for our use */
2345         mutex_lock(&i7core_edac_lock);
2346
2347         /*
2348          * All memory controllers are allocated at the first pass.
2349          */
2350         if (unlikely(probed >= 1)) {
2351                 mutex_unlock(&i7core_edac_lock);
2352                 return -ENODEV;
2353         }
2354         probed++;
2355
2356         rc = i7core_get_all_devices();
2357         if (unlikely(rc < 0))
2358                 goto fail0;
2359
2360         list_for_each_entry(i7core_dev, &i7core_edac_list, list) {
2361                 count++;
2362                 rc = i7core_register_mci(i7core_dev);
2363                 if (unlikely(rc < 0))
2364                         goto fail1;
2365         }
2366
2367         /*
2368          * Nehalem-EX uses a different memory controller. However, as the
2369          * memory controller is not visible on some Nehalem/Nehalem-EP, we
2370          * need to indirectly probe via a X58 PCI device. The same devices
2371          * are found on (some) Nehalem-EX. So, on those machines, the
2372          * probe routine needs to return -ENODEV, as the actual Memory
2373          * Controller registers won't be detected.
2374          */
2375         if (!count) {
2376                 rc = -ENODEV;
2377                 goto fail1;
2378         }
2379
2380         i7core_printk(KERN_INFO,
2381                       "Driver loaded, %d memory controller(s) found.\n",
2382                       count);
2383
2384         mutex_unlock(&i7core_edac_lock);
2385         return 0;
2386
2387 fail1:
2388         list_for_each_entry(i7core_dev, &i7core_edac_list, list)
2389                 i7core_unregister_mci(i7core_dev);
2390
2391         i7core_put_all_devices();
2392 fail0:
2393         mutex_unlock(&i7core_edac_lock);
2394         return rc;
2395 }
2396
2397 /*
2398  *      i7core_remove   destructor for one instance of device
2399  *
2400  */
2401 static void __devexit i7core_remove(struct pci_dev *pdev)
2402 {
2403         struct i7core_dev *i7core_dev;
2404
2405         debugf0(__FILE__ ": %s()\n", __func__);
2406
2407         /*
2408          * we have a trouble here: pdev value for removal will be wrong, since
2409          * it will point to the X58 register used to detect that the machine
2410          * is a Nehalem or upper design. However, due to the way several PCI
2411          * devices are grouped together to provide MC functionality, we need
2412          * to use a different method for releasing the devices
2413          */
2414
2415         mutex_lock(&i7core_edac_lock);
2416
2417         if (unlikely(!probed)) {
2418                 mutex_unlock(&i7core_edac_lock);
2419                 return;
2420         }
2421
2422         list_for_each_entry(i7core_dev, &i7core_edac_list, list)
2423                 i7core_unregister_mci(i7core_dev);
2424
2425         /* Release PCI resources */
2426         i7core_put_all_devices();
2427
2428         probed--;
2429
2430         mutex_unlock(&i7core_edac_lock);
2431 }
2432
2433 MODULE_DEVICE_TABLE(pci, i7core_pci_tbl);
2434
2435 /*
2436  *      i7core_driver   pci_driver structure for this module
2437  *
2438  */
2439 static struct pci_driver i7core_driver = {
2440         .name     = "i7core_edac",
2441         .probe    = i7core_probe,
2442         .remove   = __devexit_p(i7core_remove),
2443         .id_table = i7core_pci_tbl,
2444 };
2445
2446 /*
2447  *      i7core_init             Module entry function
2448  *                      Try to initialize this module for its devices
2449  */
2450 static int __init i7core_init(void)
2451 {
2452         int pci_rc;
2453
2454         debugf2("MC: " __FILE__ ": %s()\n", __func__);
2455
2456         /* Ensure that the OPSTATE is set correctly for POLL or NMI */
2457         opstate_init();
2458
2459         if (use_pci_fixup)
2460                 i7core_xeon_pci_fixup(pci_dev_table);
2461
2462         pci_rc = pci_register_driver(&i7core_driver);
2463
2464         if (pci_rc >= 0)
2465                 return 0;
2466
2467         i7core_printk(KERN_ERR, "Failed to register device with error %d.\n",
2468                       pci_rc);
2469
2470         return pci_rc;
2471 }
2472
2473 /*
2474  *      i7core_exit()   Module exit function
2475  *                      Unregister the driver
2476  */
2477 static void __exit i7core_exit(void)
2478 {
2479         debugf2("MC: " __FILE__ ": %s()\n", __func__);
2480         pci_unregister_driver(&i7core_driver);
2481 }
2482
2483 module_init(i7core_init);
2484 module_exit(i7core_exit);
2485
2486 MODULE_LICENSE("GPL");
2487 MODULE_AUTHOR("Mauro Carvalho Chehab <mchehab@redhat.com>");
2488 MODULE_AUTHOR("Red Hat Inc. (http://www.redhat.com)");
2489 MODULE_DESCRIPTION("MC Driver for Intel i7 Core memory controllers - "
2490                    I7CORE_REVISION);
2491
2492 module_param(edac_op_state, int, 0444);
2493 MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI");