[linux-2.6-block.git] / arch / powerpc / kernel / kvm.c

/*
 * Copyright (C) 2010 SUSE Linux Products GmbH. All rights reserved.
 *
 * Authors:
 *     Alexander Graf <agraf@suse.de>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License, version 2, as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 */

#include <linux/kvm_host.h>
#include <linux/init.h>
#include <linux/kvm_para.h>
#include <linux/slab.h>
#include <linux/of.h>

#include <asm/reg.h>
#include <asm/sections.h>
#include <asm/cacheflush.h>
#include <asm/disassemble.h>

#define KVM_MAGIC_PAGE		(-4096L)
#define magic_var(x) KVM_MAGIC_PAGE + offsetof(struct kvm_vcpu_arch_shared, x)

#define KVM_INST_LWZ		0x80000000
#define KVM_INST_STW		0x90000000
#define KVM_INST_LD		0xe8000000
#define KVM_INST_STD		0xf8000000
#define KVM_INST_NOP		0x60000000
#define KVM_INST_B		0x48000000
#define KVM_INST_B_MASK		0x03ffffff
#define KVM_INST_B_MAX		0x01ffffff

#define KVM_MASK_RT		0x03e00000
#define KVM_INST_MFMSR		0x7c0000a6
#define KVM_INST_MFSPR_SPRG0	0x7c1042a6
#define KVM_INST_MFSPR_SPRG1	0x7c1142a6
#define KVM_INST_MFSPR_SPRG2	0x7c1242a6
#define KVM_INST_MFSPR_SPRG3	0x7c1342a6
#define KVM_INST_MFSPR_SRR0	0x7c1a02a6
#define KVM_INST_MFSPR_SRR1	0x7c1b02a6
#define KVM_INST_MFSPR_DAR	0x7c1302a6
#define KVM_INST_MFSPR_DSISR	0x7c1202a6

#define KVM_INST_MTSPR_SPRG0	0x7c1043a6
#define KVM_INST_MTSPR_SPRG1	0x7c1143a6
#define KVM_INST_MTSPR_SPRG2	0x7c1243a6
#define KVM_INST_MTSPR_SPRG3	0x7c1343a6
#define KVM_INST_MTSPR_SRR0	0x7c1a03a6
#define KVM_INST_MTSPR_SRR1	0x7c1b03a6
#define KVM_INST_MTSPR_DAR	0x7c1303a6
#define KVM_INST_MTSPR_DSISR	0x7c1203a6

#define KVM_INST_TLBSYNC	0x7c00046c
#define KVM_INST_MTMSRD_L0	0x7c000164
#define KVM_INST_MTMSRD_L1	0x7c010164
#define KVM_INST_MTMSR		0x7c000124

#define KVM_INST_WRTEEI_0	0x7c000146
#define KVM_INST_WRTEEI_1	0x7c008146

static bool kvm_patching_worked = true;
static char kvm_tmp[1024 * 1024];
static int kvm_tmp_index;

static inline void kvm_patch_ins(u32 *inst, u32 new_inst)
{
	*inst = new_inst;
	flush_icache_range((ulong)inst, (ulong)inst + 4);
}

static void kvm_patch_ins_ld(u32 *inst, long addr, u32 rt)
{
#ifdef CONFIG_64BIT
	kvm_patch_ins(inst, KVM_INST_LD | rt | (addr & 0x0000fffc));
#else
	kvm_patch_ins(inst, KVM_INST_LWZ | rt | ((addr + 4) & 0x0000fffc));
#endif
}

static void kvm_patch_ins_lwz(u32 *inst, long addr, u32 rt)
{
	kvm_patch_ins(inst, KVM_INST_LWZ | rt | (addr & 0x0000ffff));
}

static void kvm_patch_ins_std(u32 *inst, long addr, u32 rt)
{
#ifdef CONFIG_64BIT
	kvm_patch_ins(inst, KVM_INST_STD | rt | (addr & 0x0000fffc));
#else
	kvm_patch_ins(inst, KVM_INST_STW | rt | ((addr + 4) & 0x0000fffc));
#endif
}

static void kvm_patch_ins_stw(u32 *inst, long addr, u32 rt)
{
	kvm_patch_ins(inst, KVM_INST_STW | rt | (addr & 0x0000fffc));
}

static void kvm_patch_ins_nop(u32 *inst)
{
	kvm_patch_ins(inst, KVM_INST_NOP);
}

static void kvm_patch_ins_b(u32 *inst, int addr)
{
#ifdef CONFIG_RELOCATABLE
	/* On relocatable kernels interrupts handlers and our code
	   can be in different regions, so we don't patch them */

	extern u32 __end_interrupts;
	if ((ulong)inst < (ulong)&__end_interrupts)
		return;
#endif

	kvm_patch_ins(inst, KVM_INST_B | (addr & KVM_INST_B_MASK));
}

static u32 *kvm_alloc(int len)
{
	u32 *p;

	if ((kvm_tmp_index + len) > ARRAY_SIZE(kvm_tmp)) {
		printk(KERN_ERR "KVM: No more space (%d + %d)\n",
				kvm_tmp_index, len);
		kvm_patching_worked = false;
		return NULL;
	}

	p = (void*)&kvm_tmp[kvm_tmp_index];
	kvm_tmp_index += len;

	return p;
}

extern u32 kvm_emulate_mtmsrd_branch_offs;
extern u32 kvm_emulate_mtmsrd_reg_offs;
extern u32 kvm_emulate_mtmsrd_len;
extern u32 kvm_emulate_mtmsrd[];

static void kvm_patch_ins_mtmsrd(u32 *inst, u32 rt)
{
	u32 *p;
	int distance_start;
	int distance_end;
	ulong next_inst;

	p = kvm_alloc(kvm_emulate_mtmsrd_len * 4);
	if (!p)
		return;

	/* Find out where we are and put everything there */
	distance_start = (ulong)p - (ulong)inst;
	next_inst = ((ulong)inst + 4);
	distance_end = next_inst - (ulong)&p[kvm_emulate_mtmsrd_branch_offs];

	/* Make sure we only write valid b instructions */
	if (distance_start > KVM_INST_B_MAX) {
		kvm_patching_worked = false;
		return;
	}

	/* Modify the chunk to fit the invocation */
	memcpy(p, kvm_emulate_mtmsrd, kvm_emulate_mtmsrd_len * 4);
	p[kvm_emulate_mtmsrd_branch_offs] |= distance_end & KVM_INST_B_MASK;
	p[kvm_emulate_mtmsrd_reg_offs] |= rt;
	flush_icache_range((ulong)p, (ulong)p + kvm_emulate_mtmsrd_len * 4);

	/* Patch the invocation */
	kvm_patch_ins_b(inst, distance_start);
}

extern u32 kvm_emulate_mtmsr_branch_offs;
extern u32 kvm_emulate_mtmsr_reg1_offs;
extern u32 kvm_emulate_mtmsr_reg2_offs;
extern u32 kvm_emulate_mtmsr_reg3_offs;
extern u32 kvm_emulate_mtmsr_orig_ins_offs;
extern u32 kvm_emulate_mtmsr_len;
extern u32 kvm_emulate_mtmsr[];

static void kvm_patch_ins_mtmsr(u32 *inst, u32 rt)
{
	u32 *p;
	int distance_start;
	int distance_end;
	ulong next_inst;

	p = kvm_alloc(kvm_emulate_mtmsr_len * 4);
	if (!p)
		return;

	/* Find out where we are and put everything there */
	distance_start = (ulong)p - (ulong)inst;
	next_inst = ((ulong)inst + 4);
	distance_end = next_inst - (ulong)&p[kvm_emulate_mtmsr_branch_offs];

	/* Make sure we only write valid b instructions */
	if (distance_start > KVM_INST_B_MAX) {
		kvm_patching_worked = false;
		return;
	}

	/* Modify the chunk to fit the invocation */
	memcpy(p, kvm_emulate_mtmsr, kvm_emulate_mtmsr_len * 4);
	p[kvm_emulate_mtmsr_branch_offs] |= distance_end & KVM_INST_B_MASK;
	p[kvm_emulate_mtmsr_reg1_offs] |= rt;
	p[kvm_emulate_mtmsr_reg2_offs] |= rt;
	p[kvm_emulate_mtmsr_reg3_offs] |= rt;
	p[kvm_emulate_mtmsr_orig_ins_offs] = *inst;
	flush_icache_range((ulong)p, (ulong)p + kvm_emulate_mtmsr_len * 4);

	/* Patch the invocation */
	kvm_patch_ins_b(inst, distance_start);
}

#ifdef CONFIG_BOOKE

extern u32 kvm_emulate_wrteei_branch_offs;
extern u32 kvm_emulate_wrteei_ee_offs;
extern u32 kvm_emulate_wrteei_len;
extern u32 kvm_emulate_wrteei[];

static void kvm_patch_ins_wrteei(u32 *inst)
{
	u32 *p;
	int distance_start;
	int distance_end;
	ulong next_inst;

	p = kvm_alloc(kvm_emulate_wrteei_len * 4);
	if (!p)
		return;

	/* Find out where we are and put everything there */
	distance_start = (ulong)p - (ulong)inst;
	next_inst = ((ulong)inst + 4);
	distance_end = next_inst - (ulong)&p[kvm_emulate_wrteei_branch_offs];

	/* Make sure we only write valid b instructions */
	if (distance_start > KVM_INST_B_MAX) {
		kvm_patching_worked = false;
		return;
	}

	/* Modify the chunk to fit the invocation */
	memcpy(p, kvm_emulate_wrteei, kvm_emulate_wrteei_len * 4);
	p[kvm_emulate_wrteei_branch_offs] |= distance_end & KVM_INST_B_MASK;
	p[kvm_emulate_wrteei_ee_offs] |= (*inst & MSR_EE);
	flush_icache_range((ulong)p, (ulong)p + kvm_emulate_wrteei_len * 4);

	/* Patch the invocation */
	kvm_patch_ins_b(inst, distance_start);
}

#endif

static void kvm_map_magic_page(void *data)
{
	kvm_hypercall2(KVM_HC_PPC_MAP_MAGIC_PAGE,
		       KVM_MAGIC_PAGE,  /* Physical Address */
		       KVM_MAGIC_PAGE); /* Effective Address */
}

static void kvm_check_ins(u32 *inst)
{
	u32 _inst = *inst;
	u32 inst_no_rt = _inst & ~KVM_MASK_RT;
	u32 inst_rt = _inst & KVM_MASK_RT;

	switch (inst_no_rt) {
	/* Loads */
	case KVM_INST_MFMSR:
		kvm_patch_ins_ld(inst, magic_var(msr), inst_rt);
		break;
	case KVM_INST_MFSPR_SPRG0:
		kvm_patch_ins_ld(inst, magic_var(sprg0), inst_rt);
		break;
	case KVM_INST_MFSPR_SPRG1:
		kvm_patch_ins_ld(inst, magic_var(sprg1), inst_rt);
		break;
	case KVM_INST_MFSPR_SPRG2:
		kvm_patch_ins_ld(inst, magic_var(sprg2), inst_rt);
		break;
	case KVM_INST_MFSPR_SPRG3:
		kvm_patch_ins_ld(inst, magic_var(sprg3), inst_rt);
		break;
	case KVM_INST_MFSPR_SRR0:
		kvm_patch_ins_ld(inst, magic_var(srr0), inst_rt);
		break;
	case KVM_INST_MFSPR_SRR1:
		kvm_patch_ins_ld(inst, magic_var(srr1), inst_rt);
		break;
	case KVM_INST_MFSPR_DAR:
		kvm_patch_ins_ld(inst, magic_var(dar), inst_rt);
		break;
	case KVM_INST_MFSPR_DSISR:
		kvm_patch_ins_lwz(inst, magic_var(dsisr), inst_rt);
		break;

	/* Stores */
	case KVM_INST_MTSPR_SPRG0:
		kvm_patch_ins_std(inst, magic_var(sprg0), inst_rt);
		break;
	case KVM_INST_MTSPR_SPRG1:
		kvm_patch_ins_std(inst, magic_var(sprg1), inst_rt);
		break;
	case KVM_INST_MTSPR_SPRG2:
		kvm_patch_ins_std(inst, magic_var(sprg2), inst_rt);
		break;
	case KVM_INST_MTSPR_SPRG3:
		kvm_patch_ins_std(inst, magic_var(sprg3), inst_rt);
		break;
	case KVM_INST_MTSPR_SRR0:
		kvm_patch_ins_std(inst, magic_var(srr0), inst_rt);
		break;
	case KVM_INST_MTSPR_SRR1:
		kvm_patch_ins_std(inst, magic_var(srr1), inst_rt);
		break;
	case KVM_INST_MTSPR_DAR:
		kvm_patch_ins_std(inst, magic_var(dar), inst_rt);
		break;
	case KVM_INST_MTSPR_DSISR:
		kvm_patch_ins_stw(inst, magic_var(dsisr), inst_rt);
		break;

	/* Nops */
	case KVM_INST_TLBSYNC:
		kvm_patch_ins_nop(inst);
		break;

	/* Rewrites */
	case KVM_INST_MTMSRD_L1:
		/* We use r30 and r31 during the hook */
		if (get_rt(inst_rt) < 30)
			kvm_patch_ins_mtmsrd(inst, inst_rt);
		break;
	case KVM_INST_MTMSR:
	case KVM_INST_MTMSRD_L0:
		/* We use r30 and r31 during the hook */
		if (get_rt(inst_rt) < 30)
			kvm_patch_ins_mtmsr(inst, inst_rt);
		break;
	}

	switch (_inst) {
#ifdef CONFIG_BOOKE
	case KVM_INST_WRTEEI_0:
	case KVM_INST_WRTEEI_1:
		kvm_patch_ins_wrteei(inst);
		break;
#endif
	}
}

static void kvm_use_magic_page(void)
{
	u32 *p;
	u32 *start, *end;
	u32 tmp;

	/* Tell the host to map the magic page to -4096 on all CPUs */
	on_each_cpu(kvm_map_magic_page, NULL, 1);

	/* Quick self-test to see if the mapping works */
	if (__get_user(tmp, (u32*)KVM_MAGIC_PAGE)) {
		kvm_patching_worked = false;
		return;
	}

	/* Now loop through all code and find instructions */
	start = (void*)_stext;
	end = (void*)_etext;

	for (p = start; p < end; p++)
		kvm_check_ins(p);

	printk(KERN_INFO "KVM: Live patching for a fast VM %s\n",
			 kvm_patching_worked ? "worked" : "failed");
}

unsigned long kvm_hypercall(unsigned long *in,
			    unsigned long *out,
			    unsigned long nr)
{
	unsigned long register r0 asm("r0");
	unsigned long register r3 asm("r3") = in[0];
	unsigned long register r4 asm("r4") = in[1];
	unsigned long register r5 asm("r5") = in[2];
	unsigned long register r6 asm("r6") = in[3];
	unsigned long register r7 asm("r7") = in[4];
	unsigned long register r8 asm("r8") = in[5];
	unsigned long register r9 asm("r9") = in[6];
	unsigned long register r10 asm("r10") = in[7];
	unsigned long register r11 asm("r11") = nr;
	unsigned long register r12 asm("r12");

	asm volatile("bl	kvm_hypercall_start"
		     : "=r"(r0), "=r"(r3), "=r"(r4), "=r"(r5), "=r"(r6),
		       "=r"(r7), "=r"(r8), "=r"(r9), "=r"(r10), "=r"(r11),
		       "=r"(r12)
		     : "r"(r3), "r"(r4), "r"(r5), "r"(r6), "r"(r7), "r"(r8),
		       "r"(r9), "r"(r10), "r"(r11)
		     : "memory", "cc", "xer", "ctr", "lr");

	out[0] = r4;
	out[1] = r5;
	out[2] = r6;
	out[3] = r7;
	out[4] = r8;
	out[5] = r9;
	out[6] = r10;
	out[7] = r11;

	return r3;
}
EXPORT_SYMBOL_GPL(kvm_hypercall);

static int kvm_para_setup(void)
{
	extern u32 kvm_hypercall_start;
	struct device_node *hyper_node;
	u32 *insts;
	int len, i;

	hyper_node = of_find_node_by_path("/hypervisor");
	if (!hyper_node)
		return -1;

	insts = (u32*)of_get_property(hyper_node, "hcall-instructions", &len);
	if (len % 4)
		return -1;
	if (len > (4 * 4))
		return -1;

	for (i = 0; i < (len / 4); i++)
		kvm_patch_ins(&(&kvm_hypercall_start)[i], insts[i]);

	return 0;
}

static __init void kvm_free_tmp(void)
{
	unsigned long start, end;

	start = (ulong)&kvm_tmp[kvm_tmp_index + (PAGE_SIZE - 1)] & PAGE_MASK;
	end = (ulong)&kvm_tmp[ARRAY_SIZE(kvm_tmp)] & PAGE_MASK;

	/* Free the tmp space we don't need */
	for (; start < end; start += PAGE_SIZE) {
		ClearPageReserved(virt_to_page(start));
		init_page_count(virt_to_page(start));
		free_page(start);
		totalram_pages++;
	}
}

static int __init kvm_guest_init(void)
{
	if (!kvm_para_available())
		goto free_tmp;

	if (kvm_para_setup())
		goto free_tmp;

	if (kvm_para_has_feature(KVM_FEATURE_MAGIC_PAGE))
		kvm_use_magic_page();

free_tmp:
	kvm_free_tmp();

	return 0;
}

postcore_initcall(kvm_guest_init);
Commit	Line	Data
2a342ed5 AG	1	/*
	2	* Copyright (C) 2010 SUSE Linux Products GmbH. All rights reserved.
	3	*
	4	* Authors:
	5	* Alexander Graf <agraf@suse.de>
	6	*
	7	* This program is free software; you can redistribute it and/or modify
	8	* it under the terms of the GNU General Public License, version 2, as
	9	* published by the Free Software Foundation.
	10	*
	11	* This program is distributed in the hope that it will be useful,
	12	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	13	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	14	* GNU General Public License for more details.
	15	*
	16	* You should have received a copy of the GNU General Public License
	17	* along with this program; if not, write to the Free Software
	18	* Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
	19	*/
	20
	21	#include <linux/kvm_host.h>
	22	#include <linux/init.h>
	23	#include <linux/kvm_para.h>
	24	#include <linux/slab.h>
	25	#include <linux/of.h>
	26
	27	#include <asm/reg.h>
2a342ed5 AG	28	#include <asm/sections.h>
	29	#include <asm/cacheflush.h>
	30	#include <asm/disassemble.h>
	31
d17051cb AG	32	#define KVM_MAGIC_PAGE (-4096L)
	33	#define magic_var(x) KVM_MAGIC_PAGE + offsetof(struct kvm_vcpu_arch_shared, x)
	34
d1293c92 AG	35	#define KVM_INST_LWZ 0x80000000
	36	#define KVM_INST_STW 0x90000000
	37	#define KVM_INST_LD 0xe8000000
	38	#define KVM_INST_STD 0xf8000000
	39	#define KVM_INST_NOP 0x60000000
	40	#define KVM_INST_B 0x48000000
	41	#define KVM_INST_B_MASK 0x03ffffff
	42	#define KVM_INST_B_MAX 0x01ffffff
	43
73a18109	44	#define KVM_MASK_RT 0x03e00000
d1293c92 AG	45	#define KVM_INST_MFMSR 0x7c0000a6
	46	#define KVM_INST_MFSPR_SPRG0 0x7c1042a6
	47	#define KVM_INST_MFSPR_SPRG1 0x7c1142a6
	48	#define KVM_INST_MFSPR_SPRG2 0x7c1242a6
	49	#define KVM_INST_MFSPR_SPRG3 0x7c1342a6
	50	#define KVM_INST_MFSPR_SRR0 0x7c1a02a6
	51	#define KVM_INST_MFSPR_SRR1 0x7c1b02a6
	52	#define KVM_INST_MFSPR_DAR 0x7c1302a6
	53	#define KVM_INST_MFSPR_DSISR 0x7c1202a6
	54
	55	#define KVM_INST_MTSPR_SPRG0 0x7c1043a6
	56	#define KVM_INST_MTSPR_SPRG1 0x7c1143a6
	57	#define KVM_INST_MTSPR_SPRG2 0x7c1243a6
	58	#define KVM_INST_MTSPR_SPRG3 0x7c1343a6
	59	#define KVM_INST_MTSPR_SRR0 0x7c1a03a6
	60	#define KVM_INST_MTSPR_SRR1 0x7c1b03a6
	61	#define KVM_INST_MTSPR_DAR 0x7c1303a6
	62	#define KVM_INST_MTSPR_DSISR 0x7c1203a6
73a18109	63
d1290b15	64	#define KVM_INST_TLBSYNC 0x7c00046c
78109277	65	#define KVM_INST_MTMSRD_L0 0x7c000164
819a63dc	66	#define KVM_INST_MTMSRD_L1 0x7c010164
78109277	67	#define KVM_INST_MTMSR 0x7c000124
d1290b15	68
644bfa01 AG	69	#define KVM_INST_WRTEEI_0 0x7c000146
	70	#define KVM_INST_WRTEEI_1 0x7c008146
	71
73a18109	72	static bool kvm_patching_worked = true;
2d4f5671 AG	73	static char kvm_tmp[1024 * 1024];
2d4f5671 AG	74	static int kvm_tmp_index;
73a18109 AG	75
	76	static inline void kvm_patch_ins(u32 *inst, u32 new_inst)
	77	{
	78	*inst = new_inst;
	79	flush_icache_range((ulong)inst, (ulong)inst + 4);
	80	}
	81
d1293c92 AG	82	static void kvm_patch_ins_ld(u32 *inst, long addr, u32 rt)
	83	{
	84	#ifdef CONFIG_64BIT
	85	kvm_patch_ins(inst, KVM_INST_LD \| rt \| (addr & 0x0000fffc));
	86	#else
	87	kvm_patch_ins(inst, KVM_INST_LWZ \| rt \| ((addr + 4) & 0x0000fffc));
	88	#endif
	89	}
	90
	91	static void kvm_patch_ins_lwz(u32 *inst, long addr, u32 rt)
	92	{
	93	kvm_patch_ins(inst, KVM_INST_LWZ \| rt \| (addr & 0x0000ffff));
	94	}
	95
	96	static void kvm_patch_ins_std(u32 *inst, long addr, u32 rt)
	97	{
	98	#ifdef CONFIG_64BIT
	99	kvm_patch_ins(inst, KVM_INST_STD \| rt \| (addr & 0x0000fffc));
	100	#else
	101	kvm_patch_ins(inst, KVM_INST_STW \| rt \| ((addr + 4) & 0x0000fffc));
	102	#endif
	103	}
	104
	105	static void kvm_patch_ins_stw(u32 *inst, long addr, u32 rt)
	106	{
	107	kvm_patch_ins(inst, KVM_INST_STW \| rt \| (addr & 0x0000fffc));
	108	}
	109
d1290b15 AG	110	static void kvm_patch_ins_nop(u32 *inst)
	111	{
	112	kvm_patch_ins(inst, KVM_INST_NOP);
	113	}
	114
71ee8e34 AG	115	static void kvm_patch_ins_b(u32 *inst, int addr)
	116	{
	117	#ifdef CONFIG_RELOCATABLE
	118	/* On relocatable kernels interrupts handlers and our code
	119	can be in different regions, so we don't patch them */
	120
	121	extern u32 __end_interrupts;
	122	if ((ulong)inst < (ulong)&__end_interrupts)
	123	return;
	124	#endif
	125
	126	kvm_patch_ins(inst, KVM_INST_B \| (addr & KVM_INST_B_MASK));
	127	}
	128
2d4f5671 AG	129	static u32 *kvm_alloc(int len)
	130	{
	131	u32 *p;
	132
	133	if ((kvm_tmp_index + len) > ARRAY_SIZE(kvm_tmp)) {
	134	printk(KERN_ERR "KVM: No more space (%d + %d)\n",
	135	kvm_tmp_index, len);
	136	kvm_patching_worked = false;
	137	return NULL;
	138	}
	139
	140	p = (void*)&kvm_tmp[kvm_tmp_index];
	141	kvm_tmp_index += len;
	142
	143	return p;
	144	}
	145
819a63dc AG	146	extern u32 kvm_emulate_mtmsrd_branch_offs;
	147	extern u32 kvm_emulate_mtmsrd_reg_offs;
	148	extern u32 kvm_emulate_mtmsrd_len;
	149	extern u32 kvm_emulate_mtmsrd[];
	150
	151	static void kvm_patch_ins_mtmsrd(u32 *inst, u32 rt)
	152	{
	153	u32 *p;
	154	int distance_start;
	155	int distance_end;
	156	ulong next_inst;
	157
	158	p = kvm_alloc(kvm_emulate_mtmsrd_len * 4);
	159	if (!p)
	160	return;
	161
	162	/* Find out where we are and put everything there */
	163	distance_start = (ulong)p - (ulong)inst;
	164	next_inst = ((ulong)inst + 4);
	165	distance_end = next_inst - (ulong)&p[kvm_emulate_mtmsrd_branch_offs];
	166
	167	/* Make sure we only write valid b instructions */
	168	if (distance_start > KVM_INST_B_MAX) {
	169	kvm_patching_worked = false;
	170	return;
	171	}
	172
	173	/* Modify the chunk to fit the invocation */
	174	memcpy(p, kvm_emulate_mtmsrd, kvm_emulate_mtmsrd_len * 4);
	175	p[kvm_emulate_mtmsrd_branch_offs] \|= distance_end & KVM_INST_B_MASK;
	176	p[kvm_emulate_mtmsrd_reg_offs] \|= rt;
	177	flush_icache_range((ulong)p, (ulong)p + kvm_emulate_mtmsrd_len * 4);
	178
	179	/* Patch the invocation */
	180	kvm_patch_ins_b(inst, distance_start);
	181	}
	182
78109277 AG	183	extern u32 kvm_emulate_mtmsr_branch_offs;
	184	extern u32 kvm_emulate_mtmsr_reg1_offs;
	185	extern u32 kvm_emulate_mtmsr_reg2_offs;
	186	extern u32 kvm_emulate_mtmsr_reg3_offs;
	187	extern u32 kvm_emulate_mtmsr_orig_ins_offs;
	188	extern u32 kvm_emulate_mtmsr_len;
	189	extern u32 kvm_emulate_mtmsr[];
	190
	191	static void kvm_patch_ins_mtmsr(u32 *inst, u32 rt)
	192	{
	193	u32 *p;
	194	int distance_start;
	195	int distance_end;
	196	ulong next_inst;
	197
	198	p = kvm_alloc(kvm_emulate_mtmsr_len * 4);
	199	if (!p)
	200	return;
	201
	202	/* Find out where we are and put everything there */
	203	distance_start = (ulong)p - (ulong)inst;
	204	next_inst = ((ulong)inst + 4);
	205	distance_end = next_inst - (ulong)&p[kvm_emulate_mtmsr_branch_offs];
	206
	207	/* Make sure we only write valid b instructions */
	208	if (distance_start > KVM_INST_B_MAX) {
	209	kvm_patching_worked = false;
	210	return;
	211	}
	212
	213	/* Modify the chunk to fit the invocation */
	214	memcpy(p, kvm_emulate_mtmsr, kvm_emulate_mtmsr_len * 4);
	215	p[kvm_emulate_mtmsr_branch_offs] \|= distance_end & KVM_INST_B_MASK;
	216	p[kvm_emulate_mtmsr_reg1_offs] \|= rt;
	217	p[kvm_emulate_mtmsr_reg2_offs] \|= rt;
	218	p[kvm_emulate_mtmsr_reg3_offs] \|= rt;
	219	p[kvm_emulate_mtmsr_orig_ins_offs] = *inst;
	220	flush_icache_range((ulong)p, (ulong)p + kvm_emulate_mtmsr_len * 4);
	221
	222	/* Patch the invocation */
	223	kvm_patch_ins_b(inst, distance_start);
	224	}
	225
644bfa01 AG	226	#ifdef CONFIG_BOOKE
	227
	228	extern u32 kvm_emulate_wrteei_branch_offs;
	229	extern u32 kvm_emulate_wrteei_ee_offs;
	230	extern u32 kvm_emulate_wrteei_len;
	231	extern u32 kvm_emulate_wrteei[];
	232
	233	static void kvm_patch_ins_wrteei(u32 *inst)
	234	{
	235	u32 *p;
	236	int distance_start;
	237	int distance_end;
	238	ulong next_inst;
	239
	240	p = kvm_alloc(kvm_emulate_wrteei_len * 4);
	241	if (!p)
	242	return;
	243
	244	/* Find out where we are and put everything there */
	245	distance_start = (ulong)p - (ulong)inst;
	246	next_inst = ((ulong)inst + 4);
	247	distance_end = next_inst - (ulong)&p[kvm_emulate_wrteei_branch_offs];
	248
	249	/* Make sure we only write valid b instructions */
	250	if (distance_start > KVM_INST_B_MAX) {
	251	kvm_patching_worked = false;
	252	return;
	253	}
	254
	255	/* Modify the chunk to fit the invocation */
	256	memcpy(p, kvm_emulate_wrteei, kvm_emulate_wrteei_len * 4);
	257	p[kvm_emulate_wrteei_branch_offs] \|= distance_end & KVM_INST_B_MASK;
	258	p[kvm_emulate_wrteei_ee_offs] \|= (*inst & MSR_EE);
	259	flush_icache_range((ulong)p, (ulong)p + kvm_emulate_wrteei_len * 4);
	260
	261	/* Patch the invocation */
	262	kvm_patch_ins_b(inst, distance_start);
	263	}
	264
	265	#endif
	266
73a18109 AG	267	static void kvm_map_magic_page(void *data)
	268	{
	269	kvm_hypercall2(KVM_HC_PPC_MAP_MAGIC_PAGE,
	270	KVM_MAGIC_PAGE, /* Physical Address */
	271	KVM_MAGIC_PAGE); /* Effective Address */
	272	}
	273
	274	static void kvm_check_ins(u32 *inst)
	275	{
	276	u32 _inst = *inst;
	277	u32 inst_no_rt = _inst & ~KVM_MASK_RT;
	278	u32 inst_rt = _inst & KVM_MASK_RT;
	279
	280	switch (inst_no_rt) {
d1293c92 AG	281	/* Loads */
	282	case KVM_INST_MFMSR:
	283	kvm_patch_ins_ld(inst, magic_var(msr), inst_rt);
	284	break;
	285	case KVM_INST_MFSPR_SPRG0:
	286	kvm_patch_ins_ld(inst, magic_var(sprg0), inst_rt);
	287	break;
	288	case KVM_INST_MFSPR_SPRG1:
	289	kvm_patch_ins_ld(inst, magic_var(sprg1), inst_rt);
	290	break;
	291	case KVM_INST_MFSPR_SPRG2:
	292	kvm_patch_ins_ld(inst, magic_var(sprg2), inst_rt);
	293	break;
	294	case KVM_INST_MFSPR_SPRG3:
	295	kvm_patch_ins_ld(inst, magic_var(sprg3), inst_rt);
	296	break;
	297	case KVM_INST_MFSPR_SRR0:
	298	kvm_patch_ins_ld(inst, magic_var(srr0), inst_rt);
	299	break;
	300	case KVM_INST_MFSPR_SRR1:
	301	kvm_patch_ins_ld(inst, magic_var(srr1), inst_rt);
	302	break;
	303	case KVM_INST_MFSPR_DAR:
	304	kvm_patch_ins_ld(inst, magic_var(dar), inst_rt);
	305	break;
	306	case KVM_INST_MFSPR_DSISR:
	307	kvm_patch_ins_lwz(inst, magic_var(dsisr), inst_rt);
	308	break;
	309
	310	/* Stores */
	311	case KVM_INST_MTSPR_SPRG0:
	312	kvm_patch_ins_std(inst, magic_var(sprg0), inst_rt);
	313	break;
	314	case KVM_INST_MTSPR_SPRG1:
	315	kvm_patch_ins_std(inst, magic_var(sprg1), inst_rt);
	316	break;
	317	case KVM_INST_MTSPR_SPRG2:
	318	kvm_patch_ins_std(inst, magic_var(sprg2), inst_rt);
	319	break;
	320	case KVM_INST_MTSPR_SPRG3:
	321	kvm_patch_ins_std(inst, magic_var(sprg3), inst_rt);
	322	break;
	323	case KVM_INST_MTSPR_SRR0:
	324	kvm_patch_ins_std(inst, magic_var(srr0), inst_rt);
	325	break;
	326	case KVM_INST_MTSPR_SRR1:
	327	kvm_patch_ins_std(inst, magic_var(srr1), inst_rt);
	328	break;
	329	case KVM_INST_MTSPR_DAR:
	330	kvm_patch_ins_std(inst, magic_var(dar), inst_rt);
	331	break;
	332	case KVM_INST_MTSPR_DSISR:
	333	kvm_patch_ins_stw(inst, magic_var(dsisr), inst_rt);
	334	break;
d1290b15 AG	335
	336	/* Nops */
	337	case KVM_INST_TLBSYNC:
	338	kvm_patch_ins_nop(inst);
	339	break;
819a63dc AG	340
	341	/* Rewrites */
	342	case KVM_INST_MTMSRD_L1:
	343	/* We use r30 and r31 during the hook */
	344	if (get_rt(inst_rt) < 30)
	345	kvm_patch_ins_mtmsrd(inst, inst_rt);
	346	break;
78109277 AG	347	case KVM_INST_MTMSR:
	348	case KVM_INST_MTMSRD_L0:
	349	/* We use r30 and r31 during the hook */
	350	if (get_rt(inst_rt) < 30)
	351	kvm_patch_ins_mtmsr(inst, inst_rt);
	352	break;
73a18109 AG	353	}
	354
	355	switch (_inst) {
644bfa01 AG	356	#ifdef CONFIG_BOOKE
	357	case KVM_INST_WRTEEI_0:
	358	case KVM_INST_WRTEEI_1:
	359	kvm_patch_ins_wrteei(inst);
	360	break;
	361	#endif
73a18109 AG	362	}
	363	}
	364
	365	static void kvm_use_magic_page(void)
	366	{
	367	u32 *p;
	368	u32 start, end;
	369	u32 tmp;
	370
	371	/* Tell the host to map the magic page to -4096 on all CPUs */
	372	on_each_cpu(kvm_map_magic_page, NULL, 1);
	373
	374	/* Quick self-test to see if the mapping works */
	375	if (__get_user(tmp, (u32*)KVM_MAGIC_PAGE)) {
	376	kvm_patching_worked = false;
	377	return;
	378	}
	379
	380	/* Now loop through all code and find instructions */
	381	start = (void*)_stext;
	382	end = (void*)_etext;
	383
	384	for (p = start; p < end; p++)
	385	kvm_check_ins(p);
	386
	387	printk(KERN_INFO "KVM: Live patching for a fast VM %s\n",
	388	kvm_patching_worked ? "worked" : "failed");
	389	}
	390
2a342ed5 AG	391	unsigned long kvm_hypercall(unsigned long *in,
	392	unsigned long *out,
	393	unsigned long nr)
	394	{
	395	unsigned long register r0 asm("r0");
	396	unsigned long register r3 asm("r3") = in[0];
	397	unsigned long register r4 asm("r4") = in[1];
	398	unsigned long register r5 asm("r5") = in[2];
	399	unsigned long register r6 asm("r6") = in[3];
	400	unsigned long register r7 asm("r7") = in[4];
	401	unsigned long register r8 asm("r8") = in[5];
	402	unsigned long register r9 asm("r9") = in[6];
	403	unsigned long register r10 asm("r10") = in[7];
	404	unsigned long register r11 asm("r11") = nr;
	405	unsigned long register r12 asm("r12");
	406
	407	asm volatile("bl kvm_hypercall_start"
	408	: "=r"(r0), "=r"(r3), "=r"(r4), "=r"(r5), "=r"(r6),
	409	"=r"(r7), "=r"(r8), "=r"(r9), "=r"(r10), "=r"(r11),
	410	"=r"(r12)
	411	: "r"(r3), "r"(r4), "r"(r5), "r"(r6), "r"(r7), "r"(r8),
	412	"r"(r9), "r"(r10), "r"(r11)
	413	: "memory", "cc", "xer", "ctr", "lr");
	414
	415	out[0] = r4;
	416	out[1] = r5;
	417	out[2] = r6;
	418	out[3] = r7;
	419	out[4] = r8;
	420	out[5] = r9;
	421	out[6] = r10;
	422	out[7] = r11;
	423
	424	return r3;
	425	}
	426	EXPORT_SYMBOL_GPL(kvm_hypercall);
73a18109 AG	427
	428	static int kvm_para_setup(void)
	429	{
	430	extern u32 kvm_hypercall_start;
	431	struct device_node *hyper_node;
	432	u32 *insts;
	433	int len, i;
	434
	435	hyper_node = of_find_node_by_path("/hypervisor");
	436	if (!hyper_node)
	437	return -1;
	438
	439	insts = (u32*)of_get_property(hyper_node, "hcall-instructions", &len);
	440	if (len % 4)
	441	return -1;
	442	if (len > (4 * 4))
	443	return -1;
	444
	445	for (i = 0; i < (len / 4); i++)
	446	kvm_patch_ins(&(&kvm_hypercall_start)[i], insts[i]);
	447
	448	return 0;
	449	}
	450
2d4f5671 AG	451	static __init void kvm_free_tmp(void)
	452	{
	453	unsigned long start, end;
	454
	455	start = (ulong)&kvm_tmp[kvm_tmp_index + (PAGE_SIZE - 1)] & PAGE_MASK;
	456	end = (ulong)&kvm_tmp[ARRAY_SIZE(kvm_tmp)] & PAGE_MASK;
	457
	458	/* Free the tmp space we don't need */
	459	for (; start < end; start += PAGE_SIZE) {
	460	ClearPageReserved(virt_to_page(start));
	461	init_page_count(virt_to_page(start));
	462	free_page(start);
	463	totalram_pages++;
	464	}
	465	}
	466
73a18109 AG	467	static int __init kvm_guest_init(void)
	468	{
	469	if (!kvm_para_available())
2d4f5671	470	goto free_tmp;
73a18109 AG	471
73a18109 AG	472	if (kvm_para_setup())
2d4f5671	473	goto free_tmp;
73a18109 AG	474
	475	if (kvm_para_has_feature(KVM_FEATURE_MAGIC_PAGE))
	476	kvm_use_magic_page();
	477
2d4f5671 AG	478	free_tmp:
	479	kvm_free_tmp();
	480
73a18109 AG	481	return 0;
	482	}
	483
	484	postcore_initcall(kvm_guest_init);