[linux-2.6-block.git] / arch / x86 / kernel / cpu / mcheck / mce_intel.c

/*
 * Intel specific MCE features.
 * Copyright 2004 Zwane Mwaikambo <zwane@linuxpower.ca>
 * Copyright (C) 2008, 2009 Intel Corporation
 * Author: Andi Kleen
 */

#include <linux/gfp.h>
#include <linux/interrupt.h>
#include <linux/percpu.h>
#include <linux/sched.h>
#include <asm/apic.h>
#include <asm/processor.h>
#include <asm/msr.h>
#include <asm/mce.h>

#include "mce-internal.h"

/*
 * Support for Intel Correct Machine Check Interrupts. This allows
 * the CPU to raise an interrupt when a corrected machine check happened.
 * Normally we pick those up using a regular polling timer.
 * Also supports reliable discovery of shared banks.
 */

/*
 * CMCI can be delivered to multiple cpus that share a machine check bank
 * so we need to designate a single cpu to process errors logged in each bank
 * in the interrupt handler (otherwise we would have many races and potential
 * double reporting of the same error).
 * Note that this can change when a cpu is offlined or brought online since
 * some MCA banks are shared across cpus. When a cpu is offlined, cmci_clear()
 * disables CMCI on all banks owned by the cpu and clears this bitfield. At
 * this point, cmci_rediscover() kicks in and a different cpu may end up
 * taking ownership of some of the shared MCA banks that were previously
 * owned by the offlined cpu.
 */
static DEFINE_PER_CPU(mce_banks_t, mce_banks_owned);

/*
 * cmci_discover_lock protects against parallel discovery attempts
 * which could race against each other.
 */
static DEFINE_RAW_SPINLOCK(cmci_discover_lock);

#define CMCI_THRESHOLD		1
#define CMCI_POLL_INTERVAL	(30 * HZ)
#define CMCI_STORM_INTERVAL	(1 * HZ)
#define CMCI_STORM_THRESHOLD	15

static DEFINE_PER_CPU(unsigned long, cmci_time_stamp);
static DEFINE_PER_CPU(unsigned int, cmci_storm_cnt);
static DEFINE_PER_CPU(unsigned int, cmci_storm_state);

enum {
	CMCI_STORM_NONE,
	CMCI_STORM_ACTIVE,
	CMCI_STORM_SUBSIDED,
};

static atomic_t cmci_storm_on_cpus;

static int cmci_supported(int *banks)
{
	u64 cap;

	if (mca_cfg.cmci_disabled || mca_cfg.ignore_ce)
		return 0;

	/*
	 * Vendor check is not strictly needed, but the initial
	 * initialization is vendor keyed and this
	 * makes sure none of the backdoors are entered otherwise.
	 */
	if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
		return 0;
	if (!cpu_has_apic || lapic_get_maxlvt() < 6)
		return 0;
	rdmsrl(MSR_IA32_MCG_CAP, cap);
	*banks = min_t(unsigned, MAX_NR_BANKS, cap & 0xff);
	return !!(cap & MCG_CMCI_P);
}

void mce_intel_cmci_poll(void)
{
	if (__this_cpu_read(cmci_storm_state) == CMCI_STORM_NONE)
		return;
	machine_check_poll(MCP_TIMESTAMP, &__get_cpu_var(mce_banks_owned));
}

void mce_intel_hcpu_update(unsigned long cpu)
{
	if (per_cpu(cmci_storm_state, cpu) == CMCI_STORM_ACTIVE)
		atomic_dec(&cmci_storm_on_cpus);

	per_cpu(cmci_storm_state, cpu) = CMCI_STORM_NONE;
}

unsigned long mce_intel_adjust_timer(unsigned long interval)
{
	int r;

	if (interval < CMCI_POLL_INTERVAL)
		return interval;

	switch (__this_cpu_read(cmci_storm_state)) {
	case CMCI_STORM_ACTIVE:
		/*
		 * We switch back to interrupt mode once the poll timer has
		 * silenced itself. That means no events recorded and the
		 * timer interval is back to our poll interval.
		 */
		__this_cpu_write(cmci_storm_state, CMCI_STORM_SUBSIDED);
		r = atomic_sub_return(1, &cmci_storm_on_cpus);
		if (r == 0)
			pr_notice("CMCI storm subsided: switching to interrupt mode\n");
		/* FALLTHROUGH */

	case CMCI_STORM_SUBSIDED:
		/*
		 * We wait for all cpus to go back to SUBSIDED
		 * state. When that happens we switch back to
		 * interrupt mode.
		 */
		if (!atomic_read(&cmci_storm_on_cpus)) {
			__this_cpu_write(cmci_storm_state, CMCI_STORM_NONE);
			cmci_reenable();
			cmci_recheck();
		}
		return CMCI_POLL_INTERVAL;
	default:
		/*
		 * We have shiny weather. Let the poll do whatever it
		 * thinks.
		 */
		return interval;
	}
}

static bool cmci_storm_detect(void)
{
	unsigned int cnt = __this_cpu_read(cmci_storm_cnt);
	unsigned long ts = __this_cpu_read(cmci_time_stamp);
	unsigned long now = jiffies;
	int r;

	if (__this_cpu_read(cmci_storm_state) != CMCI_STORM_NONE)
		return true;

	if (time_before_eq(now, ts + CMCI_STORM_INTERVAL)) {
		cnt++;
	} else {
		cnt = 1;
		__this_cpu_write(cmci_time_stamp, now);
	}
	__this_cpu_write(cmci_storm_cnt, cnt);

	if (cnt <= CMCI_STORM_THRESHOLD)
		return false;

	cmci_clear();
	__this_cpu_write(cmci_storm_state, CMCI_STORM_ACTIVE);
	r = atomic_add_return(1, &cmci_storm_on_cpus);
	mce_timer_kick(CMCI_POLL_INTERVAL);

	if (r == 1)
		pr_notice("CMCI storm detected: switching to poll mode\n");
	return true;
}

/*
 * The interrupt handler. This is called on every event.
 * Just call the poller directly to log any events.
 * This could in theory increase the threshold under high load,
 * but doesn't for now.
 */
static void intel_threshold_interrupt(void)
{
	if (cmci_storm_detect())
		return;
	machine_check_poll(MCP_TIMESTAMP, &__get_cpu_var(mce_banks_owned));
	mce_notify_irq();
}

/*
 * Enable CMCI (Corrected Machine Check Interrupt) for available MCE banks
 * on this CPU. Use the algorithm recommended in the SDM to discover shared
 * banks.
 */
static void cmci_discover(int banks)
{
	unsigned long *owned = (void *)&__get_cpu_var(mce_banks_owned);
	unsigned long flags;
	int i;
	int bios_wrong_thresh = 0;

	raw_spin_lock_irqsave(&cmci_discover_lock, flags);
	for (i = 0; i < banks; i++) {
		u64 val;
		int bios_zero_thresh = 0;

		if (test_bit(i, owned))
			continue;

		/* Skip banks in firmware first mode */
		if (test_bit(i, mce_banks_ce_disabled))
			continue;

		rdmsrl(MSR_IA32_MCx_CTL2(i), val);

		/* Already owned by someone else? */
		if (val & MCI_CTL2_CMCI_EN) {
			clear_bit(i, owned);
			__clear_bit(i, __get_cpu_var(mce_poll_banks));
			continue;
		}

		if (!mca_cfg.bios_cmci_threshold) {
			val &= ~MCI_CTL2_CMCI_THRESHOLD_MASK;
			val |= CMCI_THRESHOLD;
		} else if (!(val & MCI_CTL2_CMCI_THRESHOLD_MASK)) {
			/*
			 * If bios_cmci_threshold boot option was specified
			 * but the threshold is zero, we'll try to initialize
			 * it to 1.
			 */
			bios_zero_thresh = 1;
			val |= CMCI_THRESHOLD;
		}

		val |= MCI_CTL2_CMCI_EN;
		wrmsrl(MSR_IA32_MCx_CTL2(i), val);
		rdmsrl(MSR_IA32_MCx_CTL2(i), val);

		/* Did the enable bit stick? -- the bank supports CMCI */
		if (val & MCI_CTL2_CMCI_EN) {
			set_bit(i, owned);
			__clear_bit(i, __get_cpu_var(mce_poll_banks));
			/*
			 * We are able to set thresholds for some banks that
			 * had a threshold of 0. This means the BIOS has not
			 * set the thresholds properly or does not work with
			 * this boot option. Note down now and report later.
			 */
			if (mca_cfg.bios_cmci_threshold && bios_zero_thresh &&
					(val & MCI_CTL2_CMCI_THRESHOLD_MASK))
				bios_wrong_thresh = 1;
		} else {
			WARN_ON(!test_bit(i, __get_cpu_var(mce_poll_banks)));
		}
	}
	raw_spin_unlock_irqrestore(&cmci_discover_lock, flags);
	if (mca_cfg.bios_cmci_threshold && bios_wrong_thresh) {
		pr_info_once(
			"bios_cmci_threshold: Some banks do not have valid thresholds set\n");
		pr_info_once(
			"bios_cmci_threshold: Make sure your BIOS supports this boot option\n");
	}
}

/*
 * Just in case we missed an event during initialization check
 * all the CMCI owned banks.
 */
void cmci_recheck(void)
{
	unsigned long flags;
	int banks;

	if (!mce_available(__this_cpu_ptr(&cpu_info)) || !cmci_supported(&banks))
		return;
	local_irq_save(flags);
	machine_check_poll(MCP_TIMESTAMP, &__get_cpu_var(mce_banks_owned));
	local_irq_restore(flags);
}

/* Caller must hold the lock on cmci_discover_lock */
static void __cmci_disable_bank(int bank)
{
	u64 val;

	if (!test_bit(bank, __get_cpu_var(mce_banks_owned)))
		return;
	rdmsrl(MSR_IA32_MCx_CTL2(bank), val);
	val &= ~MCI_CTL2_CMCI_EN;
	wrmsrl(MSR_IA32_MCx_CTL2(bank), val);
	__clear_bit(bank, __get_cpu_var(mce_banks_owned));
}

/*
 * Disable CMCI on this CPU for all banks it owns when it goes down.
 * This allows other CPUs to claim the banks on rediscovery.
 */
void cmci_clear(void)
{
	unsigned long flags;
	int i;
	int banks;

	if (!cmci_supported(&banks))
		return;
	raw_spin_lock_irqsave(&cmci_discover_lock, flags);
	for (i = 0; i < banks; i++)
		__cmci_disable_bank(i);
	raw_spin_unlock_irqrestore(&cmci_discover_lock, flags);
}

static void cmci_rediscover_work_func(void *arg)
{
	int banks;

	/* Recheck banks in case CPUs don't all have the same */
	if (cmci_supported(&banks))
		cmci_discover(banks);
}

/* After a CPU went down cycle through all the others and rediscover */
void cmci_rediscover(void)
{
	int banks;

	if (!cmci_supported(&banks))
		return;

	on_each_cpu(cmci_rediscover_work_func, NULL, 1);
}

/*
 * Reenable CMCI on this CPU in case a CPU down failed.
 */
void cmci_reenable(void)
{
	int banks;
	if (cmci_supported(&banks))
		cmci_discover(banks);
}

void cmci_disable_bank(int bank)
{
	int banks;
	unsigned long flags;

	if (!cmci_supported(&banks))
		return;

	raw_spin_lock_irqsave(&cmci_discover_lock, flags);
	__cmci_disable_bank(bank);
	raw_spin_unlock_irqrestore(&cmci_discover_lock, flags);
}

static void intel_init_cmci(void)
{
	int banks;

	if (!cmci_supported(&banks))
		return;

	mce_threshold_vector = intel_threshold_interrupt;
	cmci_discover(banks);
	/*
	 * For CPU #0 this runs with still disabled APIC, but that's
	 * ok because only the vector is set up. We still do another
	 * check for the banks later for CPU #0 just to make sure
	 * to not miss any events.
	 */
	apic_write(APIC_LVTCMCI, THRESHOLD_APIC_VECTOR|APIC_DM_FIXED);
	cmci_recheck();
}

void mce_intel_feature_init(struct cpuinfo_x86 *c)
{
	intel_init_thermal(c);
	intel_init_cmci();
}
Commit	Line	Data
1da177e4 LT	1	/*
	2	* Intel specific MCE features.
	3	* Copyright 2004 Zwane Mwaikambo <zwane@linuxpower.ca>
88ccbedd AK	4	* Copyright (C) 2008, 2009 Intel Corporation
88ccbedd AK	5	* Author: Andi Kleen
1da177e4 LT	6	*/
1da177e4 LT	7
5a0e3ad6	8	#include <linux/gfp.h>
1da177e4 LT	9	#include <linux/interrupt.h>
1da177e4 LT	10	#include <linux/percpu.h>
d43c36dc	11	#include <linux/sched.h>
1bf7b31e	12	#include <asm/apic.h>
1da177e4 LT	13	#include <asm/processor.h>
	14	#include <asm/msr.h>
	15	#include <asm/mce.h>
1da177e4	16
55babd8f CG	17	#include "mce-internal.h"
55babd8f CG	18
88ccbedd AK	19	/*
	20	* Support for Intel Correct Machine Check Interrupts. This allows
	21	* the CPU to raise an interrupt when a corrected machine check happened.
	22	* Normally we pick those up using a regular polling timer.
	23	* Also supports reliable discovery of shared banks.
	24	*/
	25
0644414e NR	26	/*
	27	* CMCI can be delivered to multiple cpus that share a machine check bank
	28	* so we need to designate a single cpu to process errors logged in each bank
	29	* in the interrupt handler (otherwise we would have many races and potential
	30	* double reporting of the same error).
	31	* Note that this can change when a cpu is offlined or brought online since
	32	* some MCA banks are shared across cpus. When a cpu is offlined, cmci_clear()
	33	* disables CMCI on all banks owned by the cpu and clears this bitfield. At
	34	* this point, cmci_rediscover() kicks in and a different cpu may end up
	35	* taking ownership of some of the shared MCA banks that were previously
	36	* owned by the offlined cpu.
	37	*/
88ccbedd AK	38	static DEFINE_PER_CPU(mce_banks_t, mce_banks_owned);
	39
	40	/*
	41	* cmci_discover_lock protects against parallel discovery attempts
	42	* which could race against each other.
	43	*/
59d958d2	44	static DEFINE_RAW_SPINLOCK(cmci_discover_lock);
88ccbedd	45
55babd8f CG	46	#define CMCI_THRESHOLD 1
	47	#define CMCI_POLL_INTERVAL (30 * HZ)
	48	#define CMCI_STORM_INTERVAL (1 * HZ)
	49	#define CMCI_STORM_THRESHOLD 15
	50
	51	static DEFINE_PER_CPU(unsigned long, cmci_time_stamp);
	52	static DEFINE_PER_CPU(unsigned int, cmci_storm_cnt);
	53	static DEFINE_PER_CPU(unsigned int, cmci_storm_state);
	54
	55	enum {
	56	CMCI_STORM_NONE,
	57	CMCI_STORM_ACTIVE,
	58	CMCI_STORM_SUBSIDED,
	59	};
	60
	61	static atomic_t cmci_storm_on_cpus;
88ccbedd	62
df20e2eb	63	static int cmci_supported(int *banks)
88ccbedd AK	64	{
	65	u64 cap;
	66
7af19e4a	67	if (mca_cfg.cmci_disabled \|\| mca_cfg.ignore_ce)
62fdac59 HS	68	return 0;
62fdac59 HS	69
88ccbedd AK	70	/*
	71	* Vendor check is not strictly needed, but the initial
	72	* initialization is vendor keyed and this
	73	* makes sure none of the backdoors are entered otherwise.
	74	*/
	75	if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
	76	return 0;
	77	if (!cpu_has_apic \|\| lapic_get_maxlvt() < 6)
	78	return 0;
	79	rdmsrl(MSR_IA32_MCG_CAP, cap);
	80	*banks = min_t(unsigned, MAX_NR_BANKS, cap & 0xff);
	81	return !!(cap & MCG_CMCI_P);
	82	}
	83
55babd8f CG	84	void mce_intel_cmci_poll(void)
	85	{
	86	if (__this_cpu_read(cmci_storm_state) == CMCI_STORM_NONE)
	87	return;
	88	machine_check_poll(MCP_TIMESTAMP, &__get_cpu_var(mce_banks_owned));
	89	}
	90
	91	void mce_intel_hcpu_update(unsigned long cpu)
	92	{
	93	if (per_cpu(cmci_storm_state, cpu) == CMCI_STORM_ACTIVE)
	94	atomic_dec(&cmci_storm_on_cpus);
	95
	96	per_cpu(cmci_storm_state, cpu) = CMCI_STORM_NONE;
	97	}
	98
	99	unsigned long mce_intel_adjust_timer(unsigned long interval)
	100	{
	101	int r;
	102
	103	if (interval < CMCI_POLL_INTERVAL)
	104	return interval;
	105
	106	switch (__this_cpu_read(cmci_storm_state)) {
	107	case CMCI_STORM_ACTIVE:
	108	/*
	109	* We switch back to interrupt mode once the poll timer has
	110	* silenced itself. That means no events recorded and the
	111	* timer interval is back to our poll interval.
	112	*/
	113	__this_cpu_write(cmci_storm_state, CMCI_STORM_SUBSIDED);
	114	r = atomic_sub_return(1, &cmci_storm_on_cpus);
	115	if (r == 0)
	116	pr_notice("CMCI storm subsided: switching to interrupt mode\n");
	117	/* FALLTHROUGH */
	118
	119	case CMCI_STORM_SUBSIDED:
	120	/*
	121	* We wait for all cpus to go back to SUBSIDED
	122	* state. When that happens we switch back to
	123	* interrupt mode.
	124	*/
	125	if (!atomic_read(&cmci_storm_on_cpus)) {
	126	__this_cpu_write(cmci_storm_state, CMCI_STORM_NONE);
	127	cmci_reenable();
	128	cmci_recheck();
	129	}
	130	return CMCI_POLL_INTERVAL;
	131	default:
	132	/*
	133	* We have shiny weather. Let the poll do whatever it
	134	* thinks.
	135	*/
	136	return interval;
	137	}
	138	}
	139
	140	static bool cmci_storm_detect(void)
	141	{
	142	unsigned int cnt = __this_cpu_read(cmci_storm_cnt);
	143	unsigned long ts = __this_cpu_read(cmci_time_stamp);
	144	unsigned long now = jiffies;
	145	int r;
	146
	147	if (__this_cpu_read(cmci_storm_state) != CMCI_STORM_NONE)
148	return true;
149
150	if (time_before_eq(now, ts + CMCI_STORM_INTERVAL)) {
151	cnt++;
152	} else {
153	cnt = 1;
154	__this_cpu_write(cmci_time_stamp, now);
155	}
156	__this_cpu_write(cmci_storm_cnt, cnt);
157
158	if (cnt <= CMCI_STORM_THRESHOLD)
159	return false;
160
161	cmci_clear();
162	__this_cpu_write(cmci_storm_state, CMCI_STORM_ACTIVE);
163	r = atomic_add_return(1, &cmci_storm_on_cpus);
164	mce_timer_kick(CMCI_POLL_INTERVAL);
165
166	if (r == 1)
167	pr_notice("CMCI storm detected: switching to poll mode\n");
168	return true;
169	}
170
88ccbedd AK	171	/*
	172	* The interrupt handler. This is called on every event.
	173	* Just call the poller directly to log any events.
	174	* This could in theory increase the threshold under high load,
	175	* but doesn't for now.
	176	*/
	177	static void intel_threshold_interrupt(void)
	178	{
55babd8f CG	179	if (cmci_storm_detect())
55babd8f CG	180	return;
88ccbedd	181	machine_check_poll(MCP_TIMESTAMP, &__get_cpu_var(mce_banks_owned));
9ff36ee9	182	mce_notify_irq();
88ccbedd AK	183	}
88ccbedd AK	184
88ccbedd AK	185	/*
	186	* Enable CMCI (Corrected Machine Check Interrupt) for available MCE banks
	187	* on this CPU. Use the algorithm recommended in the SDM to discover shared
	188	* banks.
	189	*/
4670a300	190	static void cmci_discover(int banks)
88ccbedd AK	191	{
88ccbedd AK	192	unsigned long owned = (void )&__get_cpu_var(mce_banks_owned);
e5299926	193	unsigned long flags;
88ccbedd	194	int i;
450cc201	195	int bios_wrong_thresh = 0;
88ccbedd	196
59d958d2	197	raw_spin_lock_irqsave(&cmci_discover_lock, flags);
88ccbedd AK	198	for (i = 0; i < banks; i++) {
88ccbedd AK	199	u64 val;
450cc201	200	int bios_zero_thresh = 0;
88ccbedd AK	201
	202	if (test_bit(i, owned))
	203	continue;
	204
c3d1fb56 NR	205	/* Skip banks in firmware first mode */
	206	if (test_bit(i, mce_banks_ce_disabled))
	207	continue;
	208
a2d32bcb	209	rdmsrl(MSR_IA32_MCx_CTL2(i), val);
88ccbedd AK	210
88ccbedd AK	211	/* Already owned by someone else? */
1f9a0bd4	212	if (val & MCI_CTL2_CMCI_EN) {
4670a300	213	clear_bit(i, owned);
88ccbedd AK	214	__clear_bit(i, __get_cpu_var(mce_poll_banks));
	215	continue;
	216	}
	217
1462594b	218	if (!mca_cfg.bios_cmci_threshold) {
450cc201 NR	219	val &= ~MCI_CTL2_CMCI_THRESHOLD_MASK;
	220	val \|= CMCI_THRESHOLD;
	221	} else if (!(val & MCI_CTL2_CMCI_THRESHOLD_MASK)) {
	222	/*
	223	* If bios_cmci_threshold boot option was specified
	224	* but the threshold is zero, we'll try to initialize
	225	* it to 1.
	226	*/
	227	bios_zero_thresh = 1;
	228	val \|= CMCI_THRESHOLD;
	229	}
	230
	231	val \|= MCI_CTL2_CMCI_EN;
a2d32bcb AK	232	wrmsrl(MSR_IA32_MCx_CTL2(i), val);
a2d32bcb AK	233	rdmsrl(MSR_IA32_MCx_CTL2(i), val);
88ccbedd AK	234
88ccbedd AK	235	/* Did the enable bit stick? -- the bank supports CMCI */
1f9a0bd4	236	if (val & MCI_CTL2_CMCI_EN) {
4670a300	237	set_bit(i, owned);
88ccbedd	238	__clear_bit(i, __get_cpu_var(mce_poll_banks));
450cc201 NR	239	/*
	240	* We are able to set thresholds for some banks that
	241	* had a threshold of 0. This means the BIOS has not
	242	* set the thresholds properly or does not work with
	243	* this boot option. Note down now and report later.
	244	*/
1462594b	245	if (mca_cfg.bios_cmci_threshold && bios_zero_thresh &&
450cc201 NR	246	(val & MCI_CTL2_CMCI_THRESHOLD_MASK))
450cc201 NR	247	bios_wrong_thresh = 1;
88ccbedd AK	248	} else {
	249	WARN_ON(!test_bit(i, __get_cpu_var(mce_poll_banks)));
	250	}
	251	}
59d958d2	252	raw_spin_unlock_irqrestore(&cmci_discover_lock, flags);
1462594b	253	if (mca_cfg.bios_cmci_threshold && bios_wrong_thresh) {
450cc201 NR	254	pr_info_once(
	255	"bios_cmci_threshold: Some banks do not have valid thresholds set\n");
	256	pr_info_once(
	257	"bios_cmci_threshold: Make sure your BIOS supports this boot option\n");
	258	}
88ccbedd AK	259	}
	260
	261	/*
	262	* Just in case we missed an event during initialization check
	263	* all the CMCI owned banks.
	264	*/
df20e2eb	265	void cmci_recheck(void)
88ccbedd AK	266	{
	267	unsigned long flags;
	268	int banks;
	269
7b543a53	270	if (!mce_available(__this_cpu_ptr(&cpu_info)) \|\| !cmci_supported(&banks))
88ccbedd AK	271	return;
	272	local_irq_save(flags);
	273	machine_check_poll(MCP_TIMESTAMP, &__get_cpu_var(mce_banks_owned));
	274	local_irq_restore(flags);
	275	}
	276
c3d1fb56 NR	277	/* Caller must hold the lock on cmci_discover_lock */
	278	static void __cmci_disable_bank(int bank)
	279	{
	280	u64 val;
	281
	282	if (!test_bit(bank, __get_cpu_var(mce_banks_owned)))
	283	return;
	284	rdmsrl(MSR_IA32_MCx_CTL2(bank), val);
	285	val &= ~MCI_CTL2_CMCI_EN;
	286	wrmsrl(MSR_IA32_MCx_CTL2(bank), val);
	287	__clear_bit(bank, __get_cpu_var(mce_banks_owned));
	288	}
	289
88ccbedd AK	290	/*
	291	* Disable CMCI on this CPU for all banks it owns when it goes down.
	292	* This allows other CPUs to claim the banks on rediscovery.
	293	*/
df20e2eb	294	void cmci_clear(void)
88ccbedd	295	{
e5299926	296	unsigned long flags;
88ccbedd AK	297	int i;
88ccbedd AK	298	int banks;
88ccbedd AK	299
	300	if (!cmci_supported(&banks))
	301	return;
59d958d2	302	raw_spin_lock_irqsave(&cmci_discover_lock, flags);
c3d1fb56 NR	303	for (i = 0; i < banks; i++)
c3d1fb56 NR	304	__cmci_disable_bank(i);
59d958d2	305	raw_spin_unlock_irqrestore(&cmci_discover_lock, flags);
88ccbedd AK	306	}
88ccbedd AK	307
7a0c819d	308	static void cmci_rediscover_work_func(void *arg)
85b97637 TC	309	{
	310	int banks;
	311
	312	/* Recheck banks in case CPUs don't all have the same */
	313	if (cmci_supported(&banks))
	314	cmci_discover(banks);
85b97637 TC	315	}
85b97637 TC	316
7a0c819d SB	317	/* After a CPU went down cycle through all the others and rediscover */
7a0c819d SB	318	void cmci_rediscover(void)
88ccbedd	319	{
7a0c819d	320	int banks;
88ccbedd AK	321
	322	if (!cmci_supported(&banks))
	323	return;
88ccbedd	324
7a0c819d	325	on_each_cpu(cmci_rediscover_work_func, NULL, 1);
88ccbedd AK	326	}
	327
	328	/*
	329	* Reenable CMCI on this CPU in case a CPU down failed.
	330	*/
	331	void cmci_reenable(void)
	332	{
	333	int banks;
	334	if (cmci_supported(&banks))
4670a300	335	cmci_discover(banks);
88ccbedd AK	336	}
88ccbedd AK	337
c3d1fb56 NR	338	void cmci_disable_bank(int bank)
	339	{
	340	int banks;
	341	unsigned long flags;
	342
	343	if (!cmci_supported(&banks))
	344	return;
	345
	346	raw_spin_lock_irqsave(&cmci_discover_lock, flags);
	347	__cmci_disable_bank(bank);
	348	raw_spin_unlock_irqrestore(&cmci_discover_lock, flags);
	349	}
	350
514ec49a	351	static void intel_init_cmci(void)
88ccbedd AK	352	{
	353	int banks;
	354
	355	if (!cmci_supported(&banks))
	356	return;
	357
	358	mce_threshold_vector = intel_threshold_interrupt;
4670a300	359	cmci_discover(banks);
88ccbedd AK	360	/*
	361	* For CPU #0 this runs with still disabled APIC, but that's
	362	* ok because only the vector is set up. We still do another
	363	* check for the banks later for CPU #0 just to make sure
	364	* to not miss any events.
	365	*/
	366	apic_write(APIC_LVTCMCI, THRESHOLD_APIC_VECTOR\|APIC_DM_FIXED);
	367	cmci_recheck();
	368	}
	369
cc3ca220	370	void mce_intel_feature_init(struct cpuinfo_x86 *c)
1da177e4 LT	371	{
1da177e4 LT	372	intel_init_thermal(c);
88ccbedd	373	intel_init_cmci();
1da177e4	374	}