[linux-2.6-block.git] / arch / x86 / hyperv / mmu.c

#define pr_fmt(fmt)  "Hyper-V: " fmt

#include <linux/hyperv.h>
#include <linux/log2.h>
#include <linux/slab.h>
#include <linux/types.h>

#include <asm/fpu/api.h>
#include <asm/mshyperv.h>
#include <asm/msr.h>
#include <asm/tlbflush.h>

#define CREATE_TRACE_POINTS
#include <asm/trace/hyperv.h>

/* HvFlushVirtualAddressSpace, HvFlushVirtualAddressList hypercalls */
struct hv_flush_pcpu {
	u64 address_space;
	u64 flags;
	u64 processor_mask;
	u64 gva_list[];
};

/* HvFlushVirtualAddressSpaceEx, HvFlushVirtualAddressListEx hypercalls */
struct hv_flush_pcpu_ex {
	u64 address_space;
	u64 flags;
	struct hv_vpset hv_vp_set;
	u64 gva_list[];
};

/* Each gva in gva_list encodes up to 4096 pages to flush */
#define HV_TLB_FLUSH_UNIT (4096 * PAGE_SIZE)


/*
 * Fills in gva_list starting from offset. Returns the number of items added.
 */
static inline int fill_gva_list(u64 gva_list[], int offset,
				unsigned long start, unsigned long end)
{
	int gva_n = offset;
	unsigned long cur = start, diff;

	do {
		diff = end > cur ? end - cur : 0;

		gva_list[gva_n] = cur & PAGE_MASK;
		/*
		 * Lower 12 bits encode the number of additional
		 * pages to flush (in addition to the 'cur' page).
		 */
		if (diff >= HV_TLB_FLUSH_UNIT)
			gva_list[gva_n] |= ~PAGE_MASK;
		else if (diff)
			gva_list[gva_n] |= (diff - 1) >> PAGE_SHIFT;

		cur += HV_TLB_FLUSH_UNIT;
		gva_n++;

	} while (cur < end);

	return gva_n - offset;
}

static void hyperv_flush_tlb_others(const struct cpumask *cpus,
				    const struct flush_tlb_info *info)
{
	int cpu, vcpu, gva_n, max_gvas;
	struct hv_flush_pcpu **flush_pcpu;
	struct hv_flush_pcpu *flush;
	u64 status = U64_MAX;
	unsigned long flags;

	trace_hyperv_mmu_flush_tlb_others(cpus, info);

	if (!hv_hypercall_pg)
		goto do_native;

	if (cpumask_empty(cpus))
		return;

	local_irq_save(flags);

	flush_pcpu = (struct hv_flush_pcpu **)
		     this_cpu_ptr(hyperv_pcpu_input_arg);

	flush = *flush_pcpu;

	if (unlikely(!flush)) {
		local_irq_restore(flags);
		goto do_native;
	}

	if (info->mm) {
		/*
		 * AddressSpace argument must match the CR3 with PCID bits
		 * stripped out.
		 */
		flush->address_space = virt_to_phys(info->mm->pgd);
		flush->address_space &= CR3_ADDR_MASK;
		flush->flags = 0;
	} else {
		flush->address_space = 0;
		flush->flags = HV_FLUSH_ALL_VIRTUAL_ADDRESS_SPACES;
	}

	flush->processor_mask = 0;
	if (cpumask_equal(cpus, cpu_present_mask)) {
		flush->flags |= HV_FLUSH_ALL_PROCESSORS;
	} else {
		for_each_cpu(cpu, cpus) {
			vcpu = hv_cpu_number_to_vp_number(cpu);
			if (vcpu >= 64)
				goto do_native;

			__set_bit(vcpu, (unsigned long *)
				  &flush->processor_mask);
		}
	}

	/*
	 * We can flush not more than max_gvas with one hypercall. Flush the
	 * whole address space if we were asked to do more.
	 */
	max_gvas = (PAGE_SIZE - sizeof(*flush)) / sizeof(flush->gva_list[0]);

	if (info->end == TLB_FLUSH_ALL) {
		flush->flags |= HV_FLUSH_NON_GLOBAL_MAPPINGS_ONLY;
		status = hv_do_hypercall(HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE,
					 flush, NULL);
	} else if (info->end &&
		   ((info->end - info->start)/HV_TLB_FLUSH_UNIT) > max_gvas) {
		status = hv_do_hypercall(HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE,
					 flush, NULL);
	} else {
		gva_n = fill_gva_list(flush->gva_list, 0,
				      info->start, info->end);
		status = hv_do_rep_hypercall(HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST,
					     gva_n, 0, flush, NULL);
	}

	local_irq_restore(flags);

	if (!(status & HV_HYPERCALL_RESULT_MASK))
		return;
do_native:
	native_flush_tlb_others(cpus, info);
}

static void hyperv_flush_tlb_others_ex(const struct cpumask *cpus,
				       const struct flush_tlb_info *info)
{
	int nr_bank = 0, max_gvas, gva_n;
	struct hv_flush_pcpu_ex **flush_pcpu;
	struct hv_flush_pcpu_ex *flush;
	u64 status = U64_MAX;
	unsigned long flags;

	trace_hyperv_mmu_flush_tlb_others(cpus, info);

	if (!hv_hypercall_pg)
		goto do_native;

	if (cpumask_empty(cpus))
		return;

	local_irq_save(flags);

	flush_pcpu = (struct hv_flush_pcpu_ex **)
		     this_cpu_ptr(hyperv_pcpu_input_arg);

	flush = *flush_pcpu;

	if (unlikely(!flush)) {
		local_irq_restore(flags);
		goto do_native;
	}

	if (info->mm) {
		/*
		 * AddressSpace argument must match the CR3 with PCID bits
		 * stripped out.
		 */
		flush->address_space = virt_to_phys(info->mm->pgd);
		flush->address_space &= CR3_ADDR_MASK;
		flush->flags = 0;
	} else {
		flush->address_space = 0;
		flush->flags = HV_FLUSH_ALL_VIRTUAL_ADDRESS_SPACES;
	}

	flush->hv_vp_set.valid_bank_mask = 0;

	if (!cpumask_equal(cpus, cpu_present_mask)) {
		flush->hv_vp_set.format = HV_GENERIC_SET_SPARSE_4K;
		nr_bank = cpumask_to_vpset(&(flush->hv_vp_set), cpus);
	}

	if (!nr_bank) {
		flush->hv_vp_set.format = HV_GENERIC_SET_ALL;
		flush->flags |= HV_FLUSH_ALL_PROCESSORS;
	}

	/*
	 * We can flush not more than max_gvas with one hypercall. Flush the
	 * whole address space if we were asked to do more.
	 */
	max_gvas =
		(PAGE_SIZE - sizeof(*flush) - nr_bank *
		 sizeof(flush->hv_vp_set.bank_contents[0])) /
		sizeof(flush->gva_list[0]);

	if (info->end == TLB_FLUSH_ALL) {
		flush->flags |= HV_FLUSH_NON_GLOBAL_MAPPINGS_ONLY;
		status = hv_do_rep_hypercall(
			HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX,
			0, nr_bank, flush, NULL);
	} else if (info->end &&
		   ((info->end - info->start)/HV_TLB_FLUSH_UNIT) > max_gvas) {
		status = hv_do_rep_hypercall(
			HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX,
			0, nr_bank, flush, NULL);
	} else {
		gva_n = fill_gva_list(flush->gva_list, nr_bank,
				      info->start, info->end);
		status = hv_do_rep_hypercall(
			HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST_EX,
			gva_n, nr_bank, flush, NULL);
	}

	local_irq_restore(flags);

	if (!(status & HV_HYPERCALL_RESULT_MASK))
		return;
do_native:
	native_flush_tlb_others(cpus, info);
}

void hyperv_setup_mmu_ops(void)
{
	if (!(ms_hyperv.hints & HV_X64_REMOTE_TLB_FLUSH_RECOMMENDED))
		return;

	if (!(ms_hyperv.hints & HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED)) {
		pr_info("Using hypercall for remote TLB flush\n");
		pv_mmu_ops.flush_tlb_others = hyperv_flush_tlb_others;
	} else {
		pr_info("Using ext hypercall for remote TLB flush\n");
		pv_mmu_ops.flush_tlb_others = hyperv_flush_tlb_others_ex;
	}
}
Commit	Line	Data
2ffd9e33 VK	1	#define pr_fmt(fmt) "Hyper-V: " fmt
	2
	3	#include <linux/hyperv.h>
	4	#include <linux/log2.h>
	5	#include <linux/slab.h>
	6	#include <linux/types.h>
	7
	8	#include <asm/fpu/api.h>
	9	#include <asm/mshyperv.h>
	10	#include <asm/msr.h>
	11	#include <asm/tlbflush.h>
	12
773b79f7 VK	13	#define CREATE_TRACE_POINTS
	14	#include <asm/trace/hyperv.h>
	15
2ffd9e33 VK	16	/* HvFlushVirtualAddressSpace, HvFlushVirtualAddressList hypercalls */
	17	struct hv_flush_pcpu {
	18	u64 address_space;
	19	u64 flags;
	20	u64 processor_mask;
	21	u64 gva_list[];
	22	};
	23
628f54cc VK	24	/* HvFlushVirtualAddressSpaceEx, HvFlushVirtualAddressListEx hypercalls */
	25	struct hv_flush_pcpu_ex {
	26	u64 address_space;
	27	u64 flags;
800b8f03	28	struct hv_vpset hv_vp_set;
628f54cc VK	29	u64 gva_list[];
	30	};
	31
2ffd9e33 VK	32	/* Each gva in gva_list encodes up to 4096 pages to flush */
	33	#define HV_TLB_FLUSH_UNIT (4096 * PAGE_SIZE)
	34
628f54cc	35
2ffd9e33 VK	36	/*
	37	* Fills in gva_list starting from offset. Returns the number of items added.
	38	*/
	39	static inline int fill_gva_list(u64 gva_list[], int offset,
	40	unsigned long start, unsigned long end)
	41	{
	42	int gva_n = offset;
	43	unsigned long cur = start, diff;
	44
	45	do {
	46	diff = end > cur ? end - cur : 0;
	47
	48	gva_list[gva_n] = cur & PAGE_MASK;
	49	/*
	50	* Lower 12 bits encode the number of additional
	51	* pages to flush (in addition to the 'cur' page).
	52	*/
	53	if (diff >= HV_TLB_FLUSH_UNIT)
	54	gva_list[gva_n] \|= ~PAGE_MASK;
	55	else if (diff)
	56	gva_list[gva_n] \|= (diff - 1) >> PAGE_SHIFT;
	57
	58	cur += HV_TLB_FLUSH_UNIT;
	59	gva_n++;
	60
	61	} while (cur < end);
	62
	63	return gva_n - offset;
	64	}
	65
	66	static void hyperv_flush_tlb_others(const struct cpumask *cpus,
	67	const struct flush_tlb_info *info)
	68	{
	69	int cpu, vcpu, gva_n, max_gvas;
60d73a7c	70	struct hv_flush_pcpu **flush_pcpu;
2ffd9e33 VK	71	struct hv_flush_pcpu *flush;
	72	u64 status = U64_MAX;
	73	unsigned long flags;
	74
773b79f7 VK	75	trace_hyperv_mmu_flush_tlb_others(cpus, info);
773b79f7 VK	76
9a2d78e2	77	if (!hv_hypercall_pg)
2ffd9e33 VK	78	goto do_native;
	79
	80	if (cpumask_empty(cpus))
	81	return;
	82
	83	local_irq_save(flags);
	84
9a2d78e2 S	85	flush_pcpu = (struct hv_flush_pcpu **)
9a2d78e2 S	86	this_cpu_ptr(hyperv_pcpu_input_arg);
60d73a7c VK	87
	88	flush = *flush_pcpu;
	89
	90	if (unlikely(!flush)) {
	91	local_irq_restore(flags);
	92	goto do_native;
	93	}
2ffd9e33 VK	94
2ffd9e33 VK	95	if (info->mm) {
617ab45c VK	96	/*
	97	* AddressSpace argument must match the CR3 with PCID bits
	98	* stripped out.
	99	*/
2ffd9e33	100	flush->address_space = virt_to_phys(info->mm->pgd);
617ab45c	101	flush->address_space &= CR3_ADDR_MASK;
2ffd9e33 VK	102	flush->flags = 0;
	103	} else {
	104	flush->address_space = 0;
	105	flush->flags = HV_FLUSH_ALL_VIRTUAL_ADDRESS_SPACES;
	106	}
	107
	108	flush->processor_mask = 0;
	109	if (cpumask_equal(cpus, cpu_present_mask)) {
	110	flush->flags \|= HV_FLUSH_ALL_PROCESSORS;
	111	} else {
	112	for_each_cpu(cpu, cpus) {
	113	vcpu = hv_cpu_number_to_vp_number(cpu);
	114	if (vcpu >= 64)
	115	goto do_native;
	116
	117	__set_bit(vcpu, (unsigned long *)
	118	&flush->processor_mask);
	119	}
	120	}
	121
	122	/*
	123	* We can flush not more than max_gvas with one hypercall. Flush the
	124	* whole address space if we were asked to do more.
	125	*/
	126	max_gvas = (PAGE_SIZE - sizeof(*flush)) / sizeof(flush->gva_list[0]);
	127
	128	if (info->end == TLB_FLUSH_ALL) {
	129	flush->flags \|= HV_FLUSH_NON_GLOBAL_MAPPINGS_ONLY;
	130	status = hv_do_hypercall(HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE,
	131	flush, NULL);
	132	} else if (info->end &&
	133	((info->end - info->start)/HV_TLB_FLUSH_UNIT) > max_gvas) {
	134	status = hv_do_hypercall(HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE,
	135	flush, NULL);
	136	} else {
	137	gva_n = fill_gva_list(flush->gva_list, 0,
	138	info->start, info->end);
	139	status = hv_do_rep_hypercall(HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST,
	140	gva_n, 0, flush, NULL);
	141	}
	142
	143	local_irq_restore(flags);
	144
	145	if (!(status & HV_HYPERCALL_RESULT_MASK))
	146	return;
	147	do_native:
	148	native_flush_tlb_others(cpus, info);
	149	}
	150
628f54cc VK	151	static void hyperv_flush_tlb_others_ex(const struct cpumask *cpus,
	152	const struct flush_tlb_info *info)
	153	{
	154	int nr_bank = 0, max_gvas, gva_n;
60d73a7c	155	struct hv_flush_pcpu_ex **flush_pcpu;
628f54cc VK	156	struct hv_flush_pcpu_ex *flush;
	157	u64 status = U64_MAX;
	158	unsigned long flags;
	159
773b79f7 VK	160	trace_hyperv_mmu_flush_tlb_others(cpus, info);
773b79f7 VK	161
9a2d78e2	162	if (!hv_hypercall_pg)
628f54cc VK	163	goto do_native;
	164
	165	if (cpumask_empty(cpus))
	166	return;
	167
	168	local_irq_save(flags);
	169
9a2d78e2 S	170	flush_pcpu = (struct hv_flush_pcpu_ex **)
9a2d78e2 S	171	this_cpu_ptr(hyperv_pcpu_input_arg);
60d73a7c VK	172
	173	flush = *flush_pcpu;
	174
	175	if (unlikely(!flush)) {
	176	local_irq_restore(flags);
	177	goto do_native;
	178	}
628f54cc VK	179
628f54cc VK	180	if (info->mm) {
617ab45c VK	181	/*
	182	* AddressSpace argument must match the CR3 with PCID bits
	183	* stripped out.
	184	*/
628f54cc	185	flush->address_space = virt_to_phys(info->mm->pgd);
617ab45c	186	flush->address_space &= CR3_ADDR_MASK;
628f54cc VK	187	flush->flags = 0;
	188	} else {
	189	flush->address_space = 0;
	190	flush->flags = HV_FLUSH_ALL_VIRTUAL_ADDRESS_SPACES;
	191	}
	192
	193	flush->hv_vp_set.valid_bank_mask = 0;
	194
	195	if (!cpumask_equal(cpus, cpu_present_mask)) {
366f03b0	196	flush->hv_vp_set.format = HV_GENERIC_SET_SPARSE_4K;
800b8f03	197	nr_bank = cpumask_to_vpset(&(flush->hv_vp_set), cpus);
628f54cc VK	198	}
	199
	200	if (!nr_bank) {
	201	flush->hv_vp_set.format = HV_GENERIC_SET_ALL;
	202	flush->flags \|= HV_FLUSH_ALL_PROCESSORS;
	203	}
	204
	205	/*
	206	* We can flush not more than max_gvas with one hypercall. Flush the
	207	* whole address space if we were asked to do more.
	208	*/
	209	max_gvas =
	210	(PAGE_SIZE - sizeof(flush) - nr_bank
	211	sizeof(flush->hv_vp_set.bank_contents[0])) /
	212	sizeof(flush->gva_list[0]);
	213
	214	if (info->end == TLB_FLUSH_ALL) {
	215	flush->flags \|= HV_FLUSH_NON_GLOBAL_MAPPINGS_ONLY;
	216	status = hv_do_rep_hypercall(
	217	HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX,
ab7ff471	218	0, nr_bank, flush, NULL);
628f54cc VK	219	} else if (info->end &&
	220	((info->end - info->start)/HV_TLB_FLUSH_UNIT) > max_gvas) {
	221	status = hv_do_rep_hypercall(
	222	HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX,
ab7ff471	223	0, nr_bank, flush, NULL);
628f54cc VK	224	} else {
	225	gva_n = fill_gva_list(flush->gva_list, nr_bank,
	226	info->start, info->end);
	227	status = hv_do_rep_hypercall(
	228	HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST_EX,
ab7ff471	229	gva_n, nr_bank, flush, NULL);
628f54cc VK	230	}
	231
	232	local_irq_restore(flags);
	233
	234	if (!(status & HV_HYPERCALL_RESULT_MASK))
	235	return;
	236	do_native:
	237	native_flush_tlb_others(cpus, info);
	238	}
	239
2ffd9e33 VK	240	void hyperv_setup_mmu_ops(void)
2ffd9e33 VK	241	{
628f54cc VK	242	if (!(ms_hyperv.hints & HV_X64_REMOTE_TLB_FLUSH_RECOMMENDED))
	243	return;
	244
628f54cc	245	if (!(ms_hyperv.hints & HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED)) {
2ffd9e33 VK	246	pr_info("Using hypercall for remote TLB flush\n");
2ffd9e33 VK	247	pv_mmu_ops.flush_tlb_others = hyperv_flush_tlb_others;
628f54cc VK	248	} else {
	249	pr_info("Using ext hypercall for remote TLB flush\n");
	250	pv_mmu_ops.flush_tlb_others = hyperv_flush_tlb_others_ex;
2ffd9e33 VK	251	}
2ffd9e33 VK	252	}