[linux-2.6-block.git] / arch / x86 / power / cpu.c

/*
 * Suspend support specific for i386/x86-64.
 *
 * Distribute under GPLv2
 *
 * Copyright (c) 2007 Rafael J. Wysocki <rjw@sisk.pl>
 * Copyright (c) 2002 Pavel Machek <pavel@ucw.cz>
 * Copyright (c) 2001 Patrick Mochel <mochel@osdl.org>
 */

#include <linux/suspend.h>
#include <linux/export.h>
#include <linux/smp.h>
#include <linux/perf_event.h>
#include <linux/tboot.h>

#include <asm/pgtable.h>
#include <asm/proto.h>
#include <asm/mtrr.h>
#include <asm/page.h>
#include <asm/mce.h>
#include <asm/suspend.h>
#include <asm/fpu/internal.h>
#include <asm/debugreg.h>
#include <asm/cpu.h>
#include <asm/mmu_context.h>
#include <linux/dmi.h>

#ifdef CONFIG_X86_32
__visible unsigned long saved_context_ebx;
__visible unsigned long saved_context_esp, saved_context_ebp;
__visible unsigned long saved_context_esi, saved_context_edi;
__visible unsigned long saved_context_eflags;
#endif
struct saved_context saved_context;

static void msr_save_context(struct saved_context *ctxt)
{
	struct saved_msr *msr = ctxt->saved_msrs.array;
	struct saved_msr *end = msr + ctxt->saved_msrs.num;

	while (msr < end) {
		msr->valid = !rdmsrl_safe(msr->info.msr_no, &msr->info.reg.q);
		msr++;
	}
}

static void msr_restore_context(struct saved_context *ctxt)
{
	struct saved_msr *msr = ctxt->saved_msrs.array;
	struct saved_msr *end = msr + ctxt->saved_msrs.num;

	while (msr < end) {
		if (msr->valid)
			wrmsrl(msr->info.msr_no, msr->info.reg.q);
		msr++;
	}
}

/**
 *	__save_processor_state - save CPU registers before creating a
 *		hibernation image and before restoring the memory state from it
 *	@ctxt - structure to store the registers contents in
 *
 *	NOTE: If there is a CPU register the modification of which by the
 *	boot kernel (ie. the kernel used for loading the hibernation image)
 *	might affect the operations of the restored target kernel (ie. the one
 *	saved in the hibernation image), then its contents must be saved by this
 *	function.  In other words, if kernel A is hibernated and different
 *	kernel B is used for loading the hibernation image into memory, the
 *	kernel A's __save_processor_state() function must save all registers
 *	needed by kernel A, so that it can operate correctly after the resume
 *	regardless of what kernel B does in the meantime.
 */
static void __save_processor_state(struct saved_context *ctxt)
{
#ifdef CONFIG_X86_32
	mtrr_save_fixed_ranges(NULL);
#endif
	kernel_fpu_begin();

	/*
	 * descriptor tables
	 */
#ifdef CONFIG_X86_32
	store_idt(&ctxt->idt);
#else
/* CONFIG_X86_64 */
	store_idt((struct desc_ptr *)&ctxt->idt_limit);
#endif
	/*
	 * We save it here, but restore it only in the hibernate case.
	 * For ACPI S3 resume, this is loaded via 'early_gdt_desc' in 64-bit
	 * mode in "secondary_startup_64". In 32-bit mode it is done via
	 * 'pmode_gdt' in wakeup_start.
	 */
	ctxt->gdt_desc.size = GDT_SIZE - 1;
	ctxt->gdt_desc.address = (unsigned long)get_cpu_gdt_rw(smp_processor_id());

	store_tr(ctxt->tr);

	/* XMM0..XMM15 should be handled by kernel_fpu_begin(). */
	/*
	 * segment registers
	 */
#ifdef CONFIG_X86_32
	savesegment(es, ctxt->es);
	savesegment(fs, ctxt->fs);
	savesegment(gs, ctxt->gs);
	savesegment(ss, ctxt->ss);
#else
/* CONFIG_X86_64 */
	asm volatile ("movw %%ds, %0" : "=m" (ctxt->ds));
	asm volatile ("movw %%es, %0" : "=m" (ctxt->es));
	asm volatile ("movw %%fs, %0" : "=m" (ctxt->fs));
	asm volatile ("movw %%gs, %0" : "=m" (ctxt->gs));
	asm volatile ("movw %%ss, %0" : "=m" (ctxt->ss));

	rdmsrl(MSR_FS_BASE, ctxt->fs_base);
	rdmsrl(MSR_GS_BASE, ctxt->gs_base);
	rdmsrl(MSR_KERNEL_GS_BASE, ctxt->gs_kernel_base);
	mtrr_save_fixed_ranges(NULL);

	rdmsrl(MSR_EFER, ctxt->efer);
#endif

	/*
	 * control registers
	 */
	ctxt->cr0 = read_cr0();
	ctxt->cr2 = read_cr2();
	ctxt->cr3 = __read_cr3();
	ctxt->cr4 = __read_cr4();
#ifdef CONFIG_X86_64
	ctxt->cr8 = read_cr8();
#endif
	ctxt->misc_enable_saved = !rdmsrl_safe(MSR_IA32_MISC_ENABLE,
					       &ctxt->misc_enable);
	msr_save_context(ctxt);
}

/* Needed by apm.c */
void save_processor_state(void)
{
	__save_processor_state(&saved_context);
	x86_platform.save_sched_clock_state();
}
#ifdef CONFIG_X86_32
EXPORT_SYMBOL(save_processor_state);
#endif

static void do_fpu_end(void)
{
	/*
	 * Restore FPU regs if necessary.
	 */
	kernel_fpu_end();
}

static void fix_processor_context(void)
{
	int cpu = smp_processor_id();
	struct tss_struct *t = &per_cpu(cpu_tss, cpu);
#ifdef CONFIG_X86_64
	struct desc_struct *desc = get_cpu_gdt_rw(cpu);
	tss_desc tss;
#endif
	set_tss_desc(cpu, t);	/*
				 * This just modifies memory; should not be
				 * necessary. But... This is necessary, because
				 * 386 hardware has concept of busy TSS or some
				 * similar stupidity.
				 */

#ifdef CONFIG_X86_64
	memcpy(&tss, &desc[GDT_ENTRY_TSS], sizeof(tss_desc));
	tss.type = 0x9; /* The available 64-bit TSS (see AMD vol 2, pg 91 */
	write_gdt_entry(desc, GDT_ENTRY_TSS, &tss, DESC_TSS);

	syscall_init();				/* This sets MSR_*STAR and related */
#endif
	load_TR_desc();				/* This does ltr */
	load_mm_ldt(current->active_mm);	/* This does lldt */
	initialize_tlbstate_and_flush();

	fpu__resume_cpu();

	/* The processor is back on the direct GDT, load back the fixmap */
	load_fixmap_gdt(cpu);
}

/**
 *	__restore_processor_state - restore the contents of CPU registers saved
 *		by __save_processor_state()
 *	@ctxt - structure to load the registers contents from
 */
static void notrace __restore_processor_state(struct saved_context *ctxt)
{
	if (ctxt->misc_enable_saved)
		wrmsrl(MSR_IA32_MISC_ENABLE, ctxt->misc_enable);
	/*
	 * control registers
	 */
	/* cr4 was introduced in the Pentium CPU */
#ifdef CONFIG_X86_32
	if (ctxt->cr4)
		__write_cr4(ctxt->cr4);
#else
/* CONFIG X86_64 */
	wrmsrl(MSR_EFER, ctxt->efer);
	write_cr8(ctxt->cr8);
	__write_cr4(ctxt->cr4);
#endif
	write_cr3(ctxt->cr3);
	write_cr2(ctxt->cr2);
	write_cr0(ctxt->cr0);

	/*
	 * now restore the descriptor tables to their proper values
	 * ltr is done i fix_processor_context().
	 */
#ifdef CONFIG_X86_32
	load_idt(&ctxt->idt);
#else
/* CONFIG_X86_64 */
	load_idt((const struct desc_ptr *)&ctxt->idt_limit);
#endif

#ifdef CONFIG_X86_64
	/*
	 * We need GSBASE restored before percpu access can work.
	 * percpu access can happen in exception handlers or in complicated
	 * helpers like load_gs_index().
	 */
	wrmsrl(MSR_GS_BASE, ctxt->gs_base);
#endif

	fix_processor_context();

	/*
	 * Restore segment registers.  This happens after restoring the GDT
	 * and LDT, which happen in fix_processor_context().
	 */
#ifdef CONFIG_X86_32
	loadsegment(es, ctxt->es);
	loadsegment(fs, ctxt->fs);
	loadsegment(gs, ctxt->gs);
	loadsegment(ss, ctxt->ss);

	/*
	 * sysenter MSRs
	 */
	if (boot_cpu_has(X86_FEATURE_SEP))
		enable_sep_cpu();
#else
/* CONFIG_X86_64 */
	asm volatile ("movw %0, %%ds" :: "r" (ctxt->ds));
	asm volatile ("movw %0, %%es" :: "r" (ctxt->es));
	asm volatile ("movw %0, %%fs" :: "r" (ctxt->fs));
	load_gs_index(ctxt->gs);
	asm volatile ("movw %0, %%ss" :: "r" (ctxt->ss));

	/*
	 * Restore FSBASE and user GSBASE after reloading the respective
	 * segment selectors.
	 */
	wrmsrl(MSR_FS_BASE, ctxt->fs_base);
	wrmsrl(MSR_KERNEL_GS_BASE, ctxt->gs_kernel_base);
#endif

	do_fpu_end();
	tsc_verify_tsc_adjust(true);
	x86_platform.restore_sched_clock_state();
	mtrr_bp_restore();
	perf_restore_debug_store();
	msr_restore_context(ctxt);
}

/* Needed by apm.c */
void notrace restore_processor_state(void)
{
	__restore_processor_state(&saved_context);
}
#ifdef CONFIG_X86_32
EXPORT_SYMBOL(restore_processor_state);
#endif

#if defined(CONFIG_HIBERNATION) && defined(CONFIG_HOTPLUG_CPU)
static void resume_play_dead(void)
{
	play_dead_common();
	tboot_shutdown(TB_SHUTDOWN_WFS);
	hlt_play_dead();
}

int hibernate_resume_nonboot_cpu_disable(void)
{
	void (*play_dead)(void) = smp_ops.play_dead;
	int ret;

	/*
	 * Ensure that MONITOR/MWAIT will not be used in the "play dead" loop
	 * during hibernate image restoration, because it is likely that the
	 * monitored address will be actually written to at that time and then
	 * the "dead" CPU will attempt to execute instructions again, but the
	 * address in its instruction pointer may not be possible to resolve
	 * any more at that point (the page tables used by it previously may
	 * have been overwritten by hibernate image data).
	 */
	smp_ops.play_dead = resume_play_dead;
	ret = disable_nonboot_cpus();
	smp_ops.play_dead = play_dead;
	return ret;
}
#endif

/*
 * When bsp_check() is called in hibernate and suspend, cpu hotplug
 * is disabled already. So it's unnessary to handle race condition between
 * cpumask query and cpu hotplug.
 */
static int bsp_check(void)
{
	if (cpumask_first(cpu_online_mask) != 0) {
		pr_warn("CPU0 is offline.\n");
		return -ENODEV;
	}

	return 0;
}

static int bsp_pm_callback(struct notifier_block *nb, unsigned long action,
			   void *ptr)
{
	int ret = 0;

	switch (action) {
	case PM_SUSPEND_PREPARE:
	case PM_HIBERNATION_PREPARE:
		ret = bsp_check();
		break;
#ifdef CONFIG_DEBUG_HOTPLUG_CPU0
	case PM_RESTORE_PREPARE:
		/*
		 * When system resumes from hibernation, online CPU0 because
		 * 1. it's required for resume and
		 * 2. the CPU was online before hibernation
		 */
		if (!cpu_online(0))
			_debug_hotplug_cpu(0, 1);
		break;
	case PM_POST_RESTORE:
		/*
		 * When a resume really happens, this code won't be called.
		 *
		 * This code is called only when user space hibernation software
		 * prepares for snapshot device during boot time. So we just
		 * call _debug_hotplug_cpu() to restore to CPU0's state prior to
		 * preparing the snapshot device.
		 *
		 * This works for normal boot case in our CPU0 hotplug debug
		 * mode, i.e. CPU0 is offline and user mode hibernation
		 * software initializes during boot time.
		 *
		 * If CPU0 is online and user application accesses snapshot
		 * device after boot time, this will offline CPU0 and user may
		 * see different CPU0 state before and after accessing
		 * the snapshot device. But hopefully this is not a case when
		 * user debugging CPU0 hotplug. Even if users hit this case,
		 * they can easily online CPU0 back.
		 *
		 * To simplify this debug code, we only consider normal boot
		 * case. Otherwise we need to remember CPU0's state and restore
		 * to that state and resolve racy conditions etc.
		 */
		_debug_hotplug_cpu(0, 0);
		break;
#endif
	default:
		break;
	}
	return notifier_from_errno(ret);
}

static int __init bsp_pm_check_init(void)
{
	/*
	 * Set this bsp_pm_callback as lower priority than
	 * cpu_hotplug_pm_callback. So cpu_hotplug_pm_callback will be called
	 * earlier to disable cpu hotplug before bsp online check.
	 */
	pm_notifier(bsp_pm_callback, -INT_MAX);
	return 0;
}

core_initcall(bsp_pm_check_init);

static int msr_init_context(const u32 *msr_id, const int total_num)
{
	int i = 0;
	struct saved_msr *msr_array;

	if (saved_context.saved_msrs.array || saved_context.saved_msrs.num > 0) {
		pr_err("x86/pm: MSR quirk already applied, please check your DMI match table.\n");
		return -EINVAL;
	}

	msr_array = kmalloc_array(total_num, sizeof(struct saved_msr), GFP_KERNEL);
	if (!msr_array) {
		pr_err("x86/pm: Can not allocate memory to save/restore MSRs during suspend.\n");
		return -ENOMEM;
	}

	for (i = 0; i < total_num; i++) {
		msr_array[i].info.msr_no	= msr_id[i];
		msr_array[i].valid		= false;
		msr_array[i].info.reg.q		= 0;
	}
	saved_context.saved_msrs.num	= total_num;
	saved_context.saved_msrs.array	= msr_array;

	return 0;
}

/*
 * The following section is a quirk framework for problematic BIOSen:
 * Sometimes MSRs are modified by the BIOSen after suspended to
 * RAM, this might cause unexpected behavior after wakeup.
 * Thus we save/restore these specified MSRs across suspend/resume
 * in order to work around it.
 *
 * For any further problematic BIOSen/platforms,
 * please add your own function similar to msr_initialize_bdw.
 */
static int msr_initialize_bdw(const struct dmi_system_id *d)
{
	/* Add any extra MSR ids into this array. */
	u32 bdw_msr_id[] = { MSR_IA32_THERM_CONTROL };

	pr_info("x86/pm: %s detected, MSR saving is needed during suspending.\n", d->ident);
	return msr_init_context(bdw_msr_id, ARRAY_SIZE(bdw_msr_id));
}

static const struct dmi_system_id msr_save_dmi_table[] = {
	{
	 .callback = msr_initialize_bdw,
	 .ident = "BROADWELL BDX_EP",
	 .matches = {
		DMI_MATCH(DMI_PRODUCT_NAME, "GRANTLEY"),
		DMI_MATCH(DMI_PRODUCT_VERSION, "E63448-400"),
		},
	},
	{}
};

static int pm_check_save_msr(void)
{
	dmi_check_system(msr_save_dmi_table);
	return 0;
}

device_initcall(pm_check_save_msr);
Commit	Line	Data
1da177e4	1	/*
6d48becd	2	* Suspend support specific for i386/x86-64.
1da177e4 LT	3	*
	4	* Distribute under GPLv2
	5	*
cf7700fe	6	* Copyright (c) 2007 Rafael J. Wysocki <rjw@sisk.pl>
a2531293	7	* Copyright (c) 2002 Pavel Machek <pavel@ucw.cz>
1da177e4 LT	8	* Copyright (c) 2001 Patrick Mochel <mochel@osdl.org>
	9	*/
	10
1da177e4	11	#include <linux/suspend.h>
69c60c88	12	#include <linux/export.h>
f6783d20	13	#include <linux/smp.h>
1d9d8639	14	#include <linux/perf_event.h>
406f992e	15	#include <linux/tboot.h>
f6783d20	16
3dd08325	17	#include <asm/pgtable.h>
f6783d20	18	#include <asm/proto.h>
3ebad590	19	#include <asm/mtrr.h>
f6783d20 SL	20	#include <asm/page.h>
f6783d20 SL	21	#include <asm/mce.h>
a8af7898	22	#include <asm/suspend.h>
952f07ec	23	#include <asm/fpu/internal.h>
1e350066	24	#include <asm/debugreg.h>
a71c8bc5	25	#include <asm/cpu.h>
37868fe1	26	#include <asm/mmu_context.h>
7a9c2dd0	27	#include <linux/dmi.h>
1da177e4	28
833b2ca0	29	#ifdef CONFIG_X86_32
d6efc2f7 AK	30	__visible unsigned long saved_context_ebx;
	31	__visible unsigned long saved_context_esp, saved_context_ebp;
	32	__visible unsigned long saved_context_esi, saved_context_edi;
	33	__visible unsigned long saved_context_eflags;
833b2ca0	34	#endif
cc456c4e	35	struct saved_context saved_context;
1da177e4	36
7a9c2dd0 CY	37	static void msr_save_context(struct saved_context *ctxt)
	38	{
	39	struct saved_msr *msr = ctxt->saved_msrs.array;
	40	struct saved_msr *end = msr + ctxt->saved_msrs.num;
	41
	42	while (msr < end) {
	43	msr->valid = !rdmsrl_safe(msr->info.msr_no, &msr->info.reg.q);
	44	msr++;
	45	}
	46	}
	47
	48	static void msr_restore_context(struct saved_context *ctxt)
	49	{
	50	struct saved_msr *msr = ctxt->saved_msrs.array;
	51	struct saved_msr *end = msr + ctxt->saved_msrs.num;
	52
	53	while (msr < end) {
	54	if (msr->valid)
	55	wrmsrl(msr->info.msr_no, msr->info.reg.q);
	56	msr++;
	57	}
	58	}
	59
5c9c9bec RW	60	/**
	61	* __save_processor_state - save CPU registers before creating a
	62	* hibernation image and before restoring the memory state from it
	63	* @ctxt - structure to store the registers contents in
	64	*
	65	* NOTE: If there is a CPU register the modification of which by the
	66	* boot kernel (ie. the kernel used for loading the hibernation image)
	67	* might affect the operations of the restored target kernel (ie. the one
	68	* saved in the hibernation image), then its contents must be saved by this
	69	* function. In other words, if kernel A is hibernated and different
	70	* kernel B is used for loading the hibernation image into memory, the
	71	* kernel A's __save_processor_state() function must save all registers
	72	* needed by kernel A, so that it can operate correctly after the resume
	73	* regardless of what kernel B does in the meantime.
	74	*/
cae45957	75	static void __save_processor_state(struct saved_context *ctxt)
1da177e4	76	{
f9ebbe53 SL	77	#ifdef CONFIG_X86_32
	78	mtrr_save_fixed_ranges(NULL);
	79	#endif
1da177e4 LT	80	kernel_fpu_begin();
	81
	82	/*
	83	* descriptor tables
	84	*/
f9ebbe53	85	#ifdef CONFIG_X86_32
f9ebbe53 SL	86	store_idt(&ctxt->idt);
	87	#else
	88	/* CONFIG_X86_64 */
9d1c6e7c	89	store_idt((struct desc_ptr *)&ctxt->idt_limit);
f9ebbe53	90	#endif
cc456c4e KRW	91	/*
	92	* We save it here, but restore it only in the hibernate case.
	93	* For ACPI S3 resume, this is loaded via 'early_gdt_desc' in 64-bit
	94	* mode in "secondary_startup_64". In 32-bit mode it is done via
	95	* 'pmode_gdt' in wakeup_start.
	96	*/
	97	ctxt->gdt_desc.size = GDT_SIZE - 1;
69218e47	98	ctxt->gdt_desc.address = (unsigned long)get_cpu_gdt_rw(smp_processor_id());
cc456c4e	99
9d1c6e7c	100	store_tr(ctxt->tr);
1da177e4 LT	101
1da177e4 LT	102	/* XMM0..XMM15 should be handled by kernel_fpu_begin(). */
1da177e4 LT	103	/*
	104	* segment registers
	105	*/
f9ebbe53 SL	106	#ifdef CONFIG_X86_32
	107	savesegment(es, ctxt->es);
	108	savesegment(fs, ctxt->fs);
	109	savesegment(gs, ctxt->gs);
	110	savesegment(ss, ctxt->ss);
	111	#else
	112	/* CONFIG_X86_64 */
1da177e4 LT	113	asm volatile ("movw %%ds, %0" : "=m" (ctxt->ds));
	114	asm volatile ("movw %%es, %0" : "=m" (ctxt->es));
	115	asm volatile ("movw %%fs, %0" : "=m" (ctxt->fs));
	116	asm volatile ("movw %%gs, %0" : "=m" (ctxt->gs));
	117	asm volatile ("movw %%ss, %0" : "=m" (ctxt->ss));
	118
	119	rdmsrl(MSR_FS_BASE, ctxt->fs_base);
	120	rdmsrl(MSR_GS_BASE, ctxt->gs_base);
	121	rdmsrl(MSR_KERNEL_GS_BASE, ctxt->gs_kernel_base);
3ebad590	122	mtrr_save_fixed_ranges(NULL);
1da177e4	123
f9ebbe53 SL	124	rdmsrl(MSR_EFER, ctxt->efer);
	125	#endif
	126
1da177e4	127	/*
cf7700fe	128	* control registers
1da177e4	129	*/
f51c9452 GOC	130	ctxt->cr0 = read_cr0();
f51c9452 GOC	131	ctxt->cr2 = read_cr2();
6c690ee1	132	ctxt->cr3 = __read_cr3();
1ef55be1	133	ctxt->cr4 = __read_cr4();
1e02ce4c	134	#ifdef CONFIG_X86_64
f51c9452	135	ctxt->cr8 = read_cr8();
f9ebbe53	136	#endif
85a0e753 OZ	137	ctxt->misc_enable_saved = !rdmsrl_safe(MSR_IA32_MISC_ENABLE,
85a0e753 OZ	138	&ctxt->misc_enable);
7a9c2dd0	139	msr_save_context(ctxt);
1da177e4 LT	140	}
1da177e4 LT	141
f9ebbe53	142	/* Needed by apm.c */
1da177e4 LT	143	void save_processor_state(void)
	144	{
	145	__save_processor_state(&saved_context);
b74f05d6	146	x86_platform.save_sched_clock_state();
1da177e4	147	}
f9ebbe53 SL	148	#ifdef CONFIG_X86_32
	149	EXPORT_SYMBOL(save_processor_state);
	150	#endif
1da177e4	151
08967f94	152	static void do_fpu_end(void)
1da177e4	153	{
08967f94	154	/*
3134d04b	155	* Restore FPU regs if necessary.
08967f94 SL	156	*/
08967f94 SL	157	kernel_fpu_end();
1da177e4 LT	158	}
1da177e4 LT	159
3134d04b SL	160	static void fix_processor_context(void)
	161	{
	162	int cpu = smp_processor_id();
24933b82	163	struct tss_struct *t = &per_cpu(cpu_tss, cpu);
4d681be3	164	#ifdef CONFIG_X86_64
69218e47	165	struct desc_struct *desc = get_cpu_gdt_rw(cpu);
4d681be3	166	tss_desc tss;
4d681be3	167	#endif
3134d04b SL	168	set_tss_desc(cpu, t); /*
	169	* This just modifies memory; should not be
	170	* necessary. But... This is necessary, because
	171	* 386 hardware has concept of busy TSS or some
	172	* similar stupidity.
	173	*/
	174
	175	#ifdef CONFIG_X86_64
4d681be3	176	memcpy(&tss, &desc[GDT_ENTRY_TSS], sizeof(tss_desc));
	177	tss.type = 0x9; /* The available 64-bit TSS (see AMD vol 2, pg 91 */
	178	write_gdt_entry(desc, GDT_ENTRY_TSS, &tss, DESC_TSS);
3134d04b SL	179
	180	syscall_init(); /* This sets MSR_STAR and related /
	181	#endif
	182	load_TR_desc(); /* This does ltr */
37868fe1	183	load_mm_ldt(current->active_mm); /* This does lldt */
72c0098d	184	initialize_tlbstate_and_flush();
9254aaa0 IM	185
9254aaa0 IM	186	fpu__resume_cpu();
69218e47 TG	187
	188	/* The processor is back on the direct GDT, load back the fixmap */
	189	load_fixmap_gdt(cpu);
3134d04b SL	190	}
3134d04b SL	191
5c9c9bec RW	192	/**
	193	* __restore_processor_state - restore the contents of CPU registers saved
	194	* by __save_processor_state()
	195	* @ctxt - structure to load the registers contents from
	196	*/
b8f99b3e	197	static void notrace __restore_processor_state(struct saved_context *ctxt)
1da177e4	198	{
85a0e753 OZ	199	if (ctxt->misc_enable_saved)
85a0e753 OZ	200	wrmsrl(MSR_IA32_MISC_ENABLE, ctxt->misc_enable);
1da177e4 LT	201	/*
	202	* control registers
	203	*/
3134d04b SL	204	/* cr4 was introduced in the Pentium CPU */
	205	#ifdef CONFIG_X86_32
	206	if (ctxt->cr4)
1e02ce4c	207	__write_cr4(ctxt->cr4);
3134d04b SL	208	#else
3134d04b SL	209	/* CONFIG X86_64 */
3c321bce	210	wrmsrl(MSR_EFER, ctxt->efer);
f51c9452	211	write_cr8(ctxt->cr8);
1e02ce4c	212	__write_cr4(ctxt->cr4);
3134d04b	213	#endif
f51c9452 GOC	214	write_cr3(ctxt->cr3);
	215	write_cr2(ctxt->cr2);
	216	write_cr0(ctxt->cr0);
1da177e4	217
8d783b3e PM	218	/*
	219	* now restore the descriptor tables to their proper values
	220	* ltr is done i fix_processor_context().
	221	*/
3134d04b	222	#ifdef CONFIG_X86_32
3134d04b SL	223	load_idt(&ctxt->idt);
	224	#else
	225	/* CONFIG_X86_64 */
9d1c6e7c	226	load_idt((const struct desc_ptr *)&ctxt->idt_limit);
3134d04b	227	#endif
8d783b3e	228
5b06bbcf	229	#ifdef CONFIG_X86_64
1da177e4	230	/*
5b06bbcf AL	231	* We need GSBASE restored before percpu access can work.
	232	* percpu access can happen in exception handlers or in complicated
	233	* helpers like load_gs_index().
	234	*/
	235	wrmsrl(MSR_GS_BASE, ctxt->gs_base);
	236	#endif
	237
	238	fix_processor_context();
	239
	240	/*
	241	* Restore segment registers. This happens after restoring the GDT
	242	* and LDT, which happen in fix_processor_context().
1da177e4	243	*/
3134d04b SL	244	#ifdef CONFIG_X86_32
	245	loadsegment(es, ctxt->es);
	246	loadsegment(fs, ctxt->fs);
	247	loadsegment(gs, ctxt->gs);
	248	loadsegment(ss, ctxt->ss);
	249
	250	/*
	251	* sysenter MSRs
	252	*/
	253	if (boot_cpu_has(X86_FEATURE_SEP))
	254	enable_sep_cpu();
	255	#else
	256	/* CONFIG_X86_64 */
1da177e4 LT	257	asm volatile ("movw %0, %%ds" :: "r" (ctxt->ds));
	258	asm volatile ("movw %0, %%es" :: "r" (ctxt->es));
	259	asm volatile ("movw %0, %%fs" :: "r" (ctxt->fs));
	260	load_gs_index(ctxt->gs);
	261	asm volatile ("movw %0, %%ss" :: "r" (ctxt->ss));
	262
5b06bbcf AL	263	/*
	264	* Restore FSBASE and user GSBASE after reloading the respective
	265	* segment selectors.
	266	*/
1da177e4	267	wrmsrl(MSR_FS_BASE, ctxt->fs_base);
1da177e4	268	wrmsrl(MSR_KERNEL_GS_BASE, ctxt->gs_kernel_base);
3134d04b	269	#endif
1da177e4	270
1da177e4	271	do_fpu_end();
6a369583	272	tsc_verify_tsc_adjust(true);
dba69d10	273	x86_platform.restore_sched_clock_state();
d0af9eed	274	mtrr_bp_restore();
1d9d8639	275	perf_restore_debug_store();
7a9c2dd0	276	msr_restore_context(ctxt);
1da177e4 LT	277	}
1da177e4 LT	278
3134d04b	279	/* Needed by apm.c */
b8f99b3e	280	void notrace restore_processor_state(void)
1da177e4 LT	281	{
	282	__restore_processor_state(&saved_context);
	283	}
3134d04b SL	284	#ifdef CONFIG_X86_32
	285	EXPORT_SYMBOL(restore_processor_state);
	286	#endif
209efae1	287
406f992e RW	288	#if defined(CONFIG_HIBERNATION) && defined(CONFIG_HOTPLUG_CPU)
	289	static void resume_play_dead(void)
	290	{
	291	play_dead_common();
	292	tboot_shutdown(TB_SHUTDOWN_WFS);
	293	hlt_play_dead();
	294	}
	295
	296	int hibernate_resume_nonboot_cpu_disable(void)
	297	{
	298	void (*play_dead)(void) = smp_ops.play_dead;
	299	int ret;
	300
	301	/*
	302	* Ensure that MONITOR/MWAIT will not be used in the "play dead" loop
	303	* during hibernate image restoration, because it is likely that the
	304	* monitored address will be actually written to at that time and then
	305	* the "dead" CPU will attempt to execute instructions again, but the
	306	* address in its instruction pointer may not be possible to resolve
	307	* any more at that point (the page tables used by it previously may
	308	* have been overwritten by hibernate image data).
	309	*/
	310	smp_ops.play_dead = resume_play_dead;
	311	ret = disable_nonboot_cpus();
	312	smp_ops.play_dead = play_dead;
	313	return ret;
	314	}
	315	#endif
	316
209efae1 FY	317	/*
	318	* When bsp_check() is called in hibernate and suspend, cpu hotplug
	319	* is disabled already. So it's unnessary to handle race condition between
	320	* cpumask query and cpu hotplug.
	321	*/
	322	static int bsp_check(void)
	323	{
	324	if (cpumask_first(cpu_online_mask) != 0) {
	325	pr_warn("CPU0 is offline.\n");
	326	return -ENODEV;
	327	}
	328
	329	return 0;
	330	}
	331
	332	static int bsp_pm_callback(struct notifier_block *nb, unsigned long action,
	333	void *ptr)
	334	{
	335	int ret = 0;
	336
	337	switch (action) {
	338	case PM_SUSPEND_PREPARE:
	339	case PM_HIBERNATION_PREPARE:
	340	ret = bsp_check();
	341	break;
a71c8bc5 FY	342	#ifdef CONFIG_DEBUG_HOTPLUG_CPU0
	343	case PM_RESTORE_PREPARE:
	344	/*
	345	* When system resumes from hibernation, online CPU0 because
	346	* 1. it's required for resume and
	347	* 2. the CPU was online before hibernation
	348	*/
	349	if (!cpu_online(0))
	350	_debug_hotplug_cpu(0, 1);
	351	break;
	352	case PM_POST_RESTORE:
	353	/*
	354	* When a resume really happens, this code won't be called.
	355	*
	356	* This code is called only when user space hibernation software
	357	* prepares for snapshot device during boot time. So we just
	358	* call _debug_hotplug_cpu() to restore to CPU0's state prior to
	359	* preparing the snapshot device.
	360	*
	361	* This works for normal boot case in our CPU0 hotplug debug
	362	* mode, i.e. CPU0 is offline and user mode hibernation
	363	* software initializes during boot time.
	364	*
	365	* If CPU0 is online and user application accesses snapshot
	366	* device after boot time, this will offline CPU0 and user may
	367	* see different CPU0 state before and after accessing
	368	* the snapshot device. But hopefully this is not a case when
	369	* user debugging CPU0 hotplug. Even if users hit this case,
	370	* they can easily online CPU0 back.
	371	*
	372	* To simplify this debug code, we only consider normal boot
	373	* case. Otherwise we need to remember CPU0's state and restore
	374	* to that state and resolve racy conditions etc.
	375	*/
	376	_debug_hotplug_cpu(0, 0);
	377	break;
	378	#endif
209efae1 FY	379	default:
	380	break;
	381	}
	382	return notifier_from_errno(ret);
	383	}
	384
	385	static int __init bsp_pm_check_init(void)
	386	{
	387	/*
	388	* Set this bsp_pm_callback as lower priority than
	389	* cpu_hotplug_pm_callback. So cpu_hotplug_pm_callback will be called
	390	* earlier to disable cpu hotplug before bsp online check.
	391	*/
	392	pm_notifier(bsp_pm_callback, -INT_MAX);
	393	return 0;
	394	}
	395
	396	core_initcall(bsp_pm_check_init);
7a9c2dd0 CY	397
	398	static int msr_init_context(const u32 *msr_id, const int total_num)
	399	{
	400	int i = 0;
	401	struct saved_msr *msr_array;
	402
	403	if (saved_context.saved_msrs.array \|\| saved_context.saved_msrs.num > 0) {
	404	pr_err("x86/pm: MSR quirk already applied, please check your DMI match table.\n");
	405	return -EINVAL;
	406	}
	407
	408	msr_array = kmalloc_array(total_num, sizeof(struct saved_msr), GFP_KERNEL);
	409	if (!msr_array) {
	410	pr_err("x86/pm: Can not allocate memory to save/restore MSRs during suspend.\n");
	411	return -ENOMEM;
	412	}
	413
	414	for (i = 0; i < total_num; i++) {
	415	msr_array[i].info.msr_no = msr_id[i];
	416	msr_array[i].valid = false;
	417	msr_array[i].info.reg.q = 0;
	418	}
	419	saved_context.saved_msrs.num = total_num;
	420	saved_context.saved_msrs.array = msr_array;
	421
	422	return 0;
	423	}
	424
	425	/*
	426	* The following section is a quirk framework for problematic BIOSen:
	427	* Sometimes MSRs are modified by the BIOSen after suspended to
	428	* RAM, this might cause unexpected behavior after wakeup.
	429	* Thus we save/restore these specified MSRs across suspend/resume
	430	* in order to work around it.
	431	*
	432	* For any further problematic BIOSen/platforms,
	433	* please add your own function similar to msr_initialize_bdw.
	434	*/
	435	static int msr_initialize_bdw(const struct dmi_system_id *d)
	436	{
	437	/* Add any extra MSR ids into this array. */
	438	u32 bdw_msr_id[] = { MSR_IA32_THERM_CONTROL };
	439
	440	pr_info("x86/pm: %s detected, MSR saving is needed during suspending.\n", d->ident);
	441	return msr_init_context(bdw_msr_id, ARRAY_SIZE(bdw_msr_id));
	442	}
	443
6faadbbb	444	static const struct dmi_system_id msr_save_dmi_table[] = {
7a9c2dd0 CY	445	{
	446	.callback = msr_initialize_bdw,
	447	.ident = "BROADWELL BDX_EP",
	448	.matches = {
	449	DMI_MATCH(DMI_PRODUCT_NAME, "GRANTLEY"),
	450	DMI_MATCH(DMI_PRODUCT_VERSION, "E63448-400"),
	451	},
	452	},
	453	{}
	454	};
	455
	456	static int pm_check_save_msr(void)
	457	{
	458	dmi_check_system(msr_save_dmi_table);
	459	return 0;
	460	}
	461
	462	device_initcall(pm_check_save_msr);