[linux-2.6-block.git] / arch / x86 / hyperv / hv_apic.c

// SPDX-License-Identifier: GPL-2.0

/*
 * Hyper-V specific APIC code.
 *
 * Copyright (C) 2018, Microsoft, Inc.
 *
 * Author : K. Y. Srinivasan <kys@microsoft.com>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License version 2 as published
 * by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
 * NON INFRINGEMENT.  See the GNU General Public License for more
 * details.
 *
 */

#include <linux/types.h>
#include <linux/version.h>
#include <linux/vmalloc.h>
#include <linux/mm.h>
#include <linux/clockchips.h>
#include <linux/hyperv.h>
#include <linux/slab.h>
#include <linux/cpuhotplug.h>
#include <asm/hypervisor.h>
#include <asm/mshyperv.h>
#include <asm/apic.h>

#ifdef CONFIG_X86_64
#if IS_ENABLED(CONFIG_HYPERV)

static struct apic orig_apic;

static u64 hv_apic_icr_read(void)
{
	u64 reg_val;

	rdmsrl(HV_X64_MSR_ICR, reg_val);
	return reg_val;
}

static void hv_apic_icr_write(u32 low, u32 id)
{
	u64 reg_val;

	reg_val = SET_APIC_DEST_FIELD(id);
	reg_val = reg_val << 32;
	reg_val |= low;

	wrmsrl(HV_X64_MSR_ICR, reg_val);
}

static u32 hv_apic_read(u32 reg)
{
	u32 reg_val, hi;

	switch (reg) {
	case APIC_EOI:
		rdmsr(HV_X64_MSR_EOI, reg_val, hi);
		return reg_val;
	case APIC_TASKPRI:
		rdmsr(HV_X64_MSR_TPR, reg_val, hi);
		return reg_val;

	default:
		return native_apic_mem_read(reg);
	}
}

static void hv_apic_write(u32 reg, u32 val)
{
	switch (reg) {
	case APIC_EOI:
		wrmsr(HV_X64_MSR_EOI, val, 0);
		break;
	case APIC_TASKPRI:
		wrmsr(HV_X64_MSR_TPR, val, 0);
		break;
	default:
		native_apic_mem_write(reg, val);
	}
}

static void hv_apic_eoi_write(u32 reg, u32 val)
{
	wrmsr(HV_X64_MSR_EOI, val, 0);
}

/*
 * IPI implementation on Hyper-V.
 */
static bool __send_ipi_mask_ex(const struct cpumask *mask, int vector)
{
	struct ipi_arg_ex **arg;
	struct ipi_arg_ex *ipi_arg;
	unsigned long flags;
	int nr_bank = 0;
	int ret = 1;

	local_irq_save(flags);
	arg = (struct ipi_arg_ex **)this_cpu_ptr(hyperv_pcpu_input_arg);

	ipi_arg = *arg;
	if (unlikely(!ipi_arg))
		goto ipi_mask_ex_done;

	ipi_arg->vector = vector;
	ipi_arg->reserved = 0;
	ipi_arg->vp_set.valid_bank_mask = 0;

	if (!cpumask_equal(mask, cpu_present_mask)) {
		ipi_arg->vp_set.format = HV_GENERIC_SET_SPARSE_4K;
		nr_bank = cpumask_to_vpset(&(ipi_arg->vp_set), mask);
	}
	if (!nr_bank)
		ipi_arg->vp_set.format = HV_GENERIC_SET_ALL;

	ret = hv_do_rep_hypercall(HVCALL_SEND_IPI_EX, 0, nr_bank,
			      ipi_arg, NULL);

ipi_mask_ex_done:
	local_irq_restore(flags);
	return ((ret == 0) ? true : false);
}

static bool __send_ipi_mask(const struct cpumask *mask, int vector)
{
	int cur_cpu, vcpu;
	struct ipi_arg_non_ex **arg;
	struct ipi_arg_non_ex *ipi_arg;
	int ret = 1;
	unsigned long flags;

	if (cpumask_empty(mask))
		return true;

	if (!hv_hypercall_pg)
		return false;

	if ((vector < HV_IPI_LOW_VECTOR) || (vector > HV_IPI_HIGH_VECTOR))
		return false;

	if ((ms_hyperv.hints & HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED))
		return __send_ipi_mask_ex(mask, vector);

	local_irq_save(flags);
	arg = (struct ipi_arg_non_ex **)this_cpu_ptr(hyperv_pcpu_input_arg);

	ipi_arg = *arg;
	if (unlikely(!ipi_arg))
		goto ipi_mask_done;

	ipi_arg->vector = vector;
	ipi_arg->reserved = 0;
	ipi_arg->cpu_mask = 0;

	for_each_cpu(cur_cpu, mask) {
		vcpu = hv_cpu_number_to_vp_number(cur_cpu);
		/*
		 * This particular version of the IPI hypercall can
		 * only target upto 64 CPUs.
		 */
		if (vcpu >= 64)
			goto ipi_mask_done;

		__set_bit(vcpu, (unsigned long *)&ipi_arg->cpu_mask);
	}

	ret = hv_do_hypercall(HVCALL_SEND_IPI, ipi_arg, NULL);

ipi_mask_done:
	local_irq_restore(flags);
	return ((ret == 0) ? true : false);
}

static bool __send_ipi_one(int cpu, int vector)
{
	struct cpumask mask = CPU_MASK_NONE;

	cpumask_set_cpu(cpu, &mask);
	return __send_ipi_mask(&mask, vector);
}

static void hv_send_ipi(int cpu, int vector)
{
	if (!__send_ipi_one(cpu, vector))
		orig_apic.send_IPI(cpu, vector);
}

static void hv_send_ipi_mask(const struct cpumask *mask, int vector)
{
	if (!__send_ipi_mask(mask, vector))
		orig_apic.send_IPI_mask(mask, vector);
}

static void hv_send_ipi_mask_allbutself(const struct cpumask *mask, int vector)
{
	unsigned int this_cpu = smp_processor_id();
	struct cpumask new_mask;
	const struct cpumask *local_mask;

	cpumask_copy(&new_mask, mask);
	cpumask_clear_cpu(this_cpu, &new_mask);
	local_mask = &new_mask;
	if (!__send_ipi_mask(local_mask, vector))
		orig_apic.send_IPI_mask_allbutself(mask, vector);
}

static void hv_send_ipi_allbutself(int vector)
{
	hv_send_ipi_mask_allbutself(cpu_online_mask, vector);
}

static void hv_send_ipi_all(int vector)
{
	if (!__send_ipi_mask(cpu_online_mask, vector))
		orig_apic.send_IPI_all(vector);
}

static void hv_send_ipi_self(int vector)
{
	if (!__send_ipi_one(smp_processor_id(), vector))
		orig_apic.send_IPI_self(vector);
}

void __init hv_apic_init(void)
{
	if (ms_hyperv.hints & HV_X64_CLUSTER_IPI_RECOMMENDED) {
		if ((ms_hyperv.hints & HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED))
			pr_info("Hyper-V: Using ext hypercalls for IPI\n");
		else
			pr_info("Hyper-V: Using IPI hypercalls\n");
		/*
		 * Set the IPI entry points.
		 */
		orig_apic = *apic;

		apic->send_IPI = hv_send_ipi;
		apic->send_IPI_mask = hv_send_ipi_mask;
		apic->send_IPI_mask_allbutself = hv_send_ipi_mask_allbutself;
		apic->send_IPI_allbutself = hv_send_ipi_allbutself;
		apic->send_IPI_all = hv_send_ipi_all;
		apic->send_IPI_self = hv_send_ipi_self;
	}

	if (ms_hyperv.hints & HV_X64_APIC_ACCESS_RECOMMENDED) {
		pr_info("Hyper-V: Using MSR based APIC access\n");
		apic_set_eoi_write(hv_apic_eoi_write);
		apic->read      = hv_apic_read;
		apic->write     = hv_apic_write;
		apic->icr_write = hv_apic_icr_write;
		apic->icr_read  = hv_apic_icr_read;
	}
}

#endif /* CONFIG_HYPERV */
#endif /* CONFIG_X86_64 */
Commit	Line	Data
6b48cb5f S	1	// SPDX-License-Identifier: GPL-2.0
	2
	3	/*
	4	* Hyper-V specific APIC code.
	5	*
	6	* Copyright (C) 2018, Microsoft, Inc.
	7	*
	8	* Author : K. Y. Srinivasan <kys@microsoft.com>
	9	*
	10	* This program is free software; you can redistribute it and/or modify it
	11	* under the terms of the GNU General Public License version 2 as published
	12	* by the Free Software Foundation.
	13	*
	14	* This program is distributed in the hope that it will be useful, but
	15	* WITHOUT ANY WARRANTY; without even the implied warranty of
	16	* MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
	17	* NON INFRINGEMENT. See the GNU General Public License for more
	18	* details.
	19	*
	20	*/
	21
	22	#include <linux/types.h>
	23	#include <linux/version.h>
	24	#include <linux/vmalloc.h>
	25	#include <linux/mm.h>
	26	#include <linux/clockchips.h>
	27	#include <linux/hyperv.h>
	28	#include <linux/slab.h>
	29	#include <linux/cpuhotplug.h>
	30	#include <asm/hypervisor.h>
	31	#include <asm/mshyperv.h>
61eeb1f6	32	#include <asm/apic.h>
6b48cb5f S	33
	34	#ifdef CONFIG_X86_64
	35	#if IS_ENABLED(CONFIG_HYPERV)
	36
68bb7bfb S	37	static struct apic orig_apic;
68bb7bfb S	38
6b48cb5f S	39	static u64 hv_apic_icr_read(void)
	40	{
	41	u64 reg_val;
	42
	43	rdmsrl(HV_X64_MSR_ICR, reg_val);
	44	return reg_val;
	45	}
	46
	47	static void hv_apic_icr_write(u32 low, u32 id)
	48	{
	49	u64 reg_val;
	50
	51	reg_val = SET_APIC_DEST_FIELD(id);
	52	reg_val = reg_val << 32;
	53	reg_val \|= low;
	54
	55	wrmsrl(HV_X64_MSR_ICR, reg_val);
	56	}
	57
	58	static u32 hv_apic_read(u32 reg)
	59	{
	60	u32 reg_val, hi;
	61
	62	switch (reg) {
	63	case APIC_EOI:
	64	rdmsr(HV_X64_MSR_EOI, reg_val, hi);
	65	return reg_val;
	66	case APIC_TASKPRI:
	67	rdmsr(HV_X64_MSR_TPR, reg_val, hi);
	68	return reg_val;
	69
	70	default:
	71	return native_apic_mem_read(reg);
	72	}
	73	}
	74
	75	static void hv_apic_write(u32 reg, u32 val)
	76	{
	77	switch (reg) {
	78	case APIC_EOI:
	79	wrmsr(HV_X64_MSR_EOI, val, 0);
	80	break;
	81	case APIC_TASKPRI:
	82	wrmsr(HV_X64_MSR_TPR, val, 0);
	83	break;
	84	default:
	85	native_apic_mem_write(reg, val);
	86	}
	87	}
	88
	89	static void hv_apic_eoi_write(u32 reg, u32 val)
	90	{
	91	wrmsr(HV_X64_MSR_EOI, val, 0);
	92	}
	93
68bb7bfb S	94	/*
	95	* IPI implementation on Hyper-V.
	96	*/
366f03b0 S	97	static bool __send_ipi_mask_ex(const struct cpumask *mask, int vector)
	98	{
	99	struct ipi_arg_ex **arg;
	100	struct ipi_arg_ex *ipi_arg;
	101	unsigned long flags;
	102	int nr_bank = 0;
	103	int ret = 1;
	104
	105	local_irq_save(flags);
	106	arg = (struct ipi_arg_ex **)this_cpu_ptr(hyperv_pcpu_input_arg);
	107
	108	ipi_arg = *arg;
	109	if (unlikely(!ipi_arg))
	110	goto ipi_mask_ex_done;
	111
	112	ipi_arg->vector = vector;
	113	ipi_arg->reserved = 0;
	114	ipi_arg->vp_set.valid_bank_mask = 0;
	115
	116	if (!cpumask_equal(mask, cpu_present_mask)) {
	117	ipi_arg->vp_set.format = HV_GENERIC_SET_SPARSE_4K;
	118	nr_bank = cpumask_to_vpset(&(ipi_arg->vp_set), mask);
	119	}
	120	if (!nr_bank)
	121	ipi_arg->vp_set.format = HV_GENERIC_SET_ALL;
	122
	123	ret = hv_do_rep_hypercall(HVCALL_SEND_IPI_EX, 0, nr_bank,
	124	ipi_arg, NULL);
	125
	126	ipi_mask_ex_done:
	127	local_irq_restore(flags);
	128	return ((ret == 0) ? true : false);
	129	}
	130
68bb7bfb S	131	static bool __send_ipi_mask(const struct cpumask *mask, int vector)
	132	{
	133	int cur_cpu, vcpu;
	134	struct ipi_arg_non_ex **arg;
	135	struct ipi_arg_non_ex *ipi_arg;
	136	int ret = 1;
	137	unsigned long flags;
	138
	139	if (cpumask_empty(mask))
	140	return true;
	141
	142	if (!hv_hypercall_pg)
	143	return false;
	144
	145	if ((vector < HV_IPI_LOW_VECTOR) \|\| (vector > HV_IPI_HIGH_VECTOR))
	146	return false;
	147
366f03b0 S	148	if ((ms_hyperv.hints & HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED))
	149	return __send_ipi_mask_ex(mask, vector);
	150
68bb7bfb S	151	local_irq_save(flags);
	152	arg = (struct ipi_arg_non_ex **)this_cpu_ptr(hyperv_pcpu_input_arg);
	153
	154	ipi_arg = *arg;
	155	if (unlikely(!ipi_arg))
	156	goto ipi_mask_done;
	157
	158	ipi_arg->vector = vector;
	159	ipi_arg->reserved = 0;
	160	ipi_arg->cpu_mask = 0;
	161
	162	for_each_cpu(cur_cpu, mask) {
	163	vcpu = hv_cpu_number_to_vp_number(cur_cpu);
	164	/*
	165	* This particular version of the IPI hypercall can
	166	* only target upto 64 CPUs.
	167	*/
	168	if (vcpu >= 64)
	169	goto ipi_mask_done;
	170
	171	__set_bit(vcpu, (unsigned long *)&ipi_arg->cpu_mask);
	172	}
	173
	174	ret = hv_do_hypercall(HVCALL_SEND_IPI, ipi_arg, NULL);
	175
	176	ipi_mask_done:
	177	local_irq_restore(flags);
	178	return ((ret == 0) ? true : false);
	179	}
	180
	181	static bool __send_ipi_one(int cpu, int vector)
	182	{
	183	struct cpumask mask = CPU_MASK_NONE;
	184
	185	cpumask_set_cpu(cpu, &mask);
	186	return __send_ipi_mask(&mask, vector);
	187	}
	188
	189	static void hv_send_ipi(int cpu, int vector)
	190	{
	191	if (!__send_ipi_one(cpu, vector))
	192	orig_apic.send_IPI(cpu, vector);
	193	}
	194
	195	static void hv_send_ipi_mask(const struct cpumask *mask, int vector)
	196	{
	197	if (!__send_ipi_mask(mask, vector))
	198	orig_apic.send_IPI_mask(mask, vector);
	199	}
	200
	201	static void hv_send_ipi_mask_allbutself(const struct cpumask *mask, int vector)
	202	{
	203	unsigned int this_cpu = smp_processor_id();
	204	struct cpumask new_mask;
	205	const struct cpumask *local_mask;
	206
	207	cpumask_copy(&new_mask, mask);
	208	cpumask_clear_cpu(this_cpu, &new_mask);
	209	local_mask = &new_mask;
	210	if (!__send_ipi_mask(local_mask, vector))
	211	orig_apic.send_IPI_mask_allbutself(mask, vector);
	212	}
	213
	214	static void hv_send_ipi_allbutself(int vector)
215	{
216	hv_send_ipi_mask_allbutself(cpu_online_mask, vector);
217	}
218
219	static void hv_send_ipi_all(int vector)
220	{
221	if (!__send_ipi_mask(cpu_online_mask, vector))
222	orig_apic.send_IPI_all(vector);
223	}
224
225	static void hv_send_ipi_self(int vector)
226	{
227	if (!__send_ipi_one(smp_processor_id(), vector))
228	orig_apic.send_IPI_self(vector);
229	}
230
6b48cb5f S	231	void __init hv_apic_init(void)
6b48cb5f S	232	{
68bb7bfb	233	if (ms_hyperv.hints & HV_X64_CLUSTER_IPI_RECOMMENDED) {
366f03b0 S	234	if ((ms_hyperv.hints & HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED))
	235	pr_info("Hyper-V: Using ext hypercalls for IPI\n");
	236	else
	237	pr_info("Hyper-V: Using IPI hypercalls\n");
68bb7bfb S	238	/*
	239	* Set the IPI entry points.
	240	*/
	241	orig_apic = *apic;
	242
	243	apic->send_IPI = hv_send_ipi;
	244	apic->send_IPI_mask = hv_send_ipi_mask;
	245	apic->send_IPI_mask_allbutself = hv_send_ipi_mask_allbutself;
	246	apic->send_IPI_allbutself = hv_send_ipi_allbutself;
	247	apic->send_IPI_all = hv_send_ipi_all;
	248	apic->send_IPI_self = hv_send_ipi_self;
	249	}
	250
6b48cb5f S	251	if (ms_hyperv.hints & HV_X64_APIC_ACCESS_RECOMMENDED) {
	252	pr_info("Hyper-V: Using MSR based APIC access\n");
	253	apic_set_eoi_write(hv_apic_eoi_write);
	254	apic->read = hv_apic_read;
	255	apic->write = hv_apic_write;
	256	apic->icr_write = hv_apic_icr_write;
	257	apic->icr_read = hv_apic_icr_read;
	258	}
	259	}
	260
	261	#endif /* CONFIG_HYPERV */
	262	#endif /* CONFIG_X86_64 */