[linux-2.6-block.git] / arch / x86 / xen / smp.c

#include <linux/smp.h>
#include <linux/slab.h>
#include <linux/cpumask.h>
#include <linux/percpu.h>

#include <xen/events.h>

#include <xen/hvc-console.h>
#include "xen-ops.h"
#include "smp.h"

static DEFINE_PER_CPU(struct xen_common_irq, xen_resched_irq) = { .irq = -1 };
static DEFINE_PER_CPU(struct xen_common_irq, xen_callfunc_irq) = { .irq = -1 };
static DEFINE_PER_CPU(struct xen_common_irq, xen_callfuncsingle_irq) = { .irq = -1 };
static DEFINE_PER_CPU(struct xen_common_irq, xen_debug_irq) = { .irq = -1 };

static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id);
static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id);

/*
 * Reschedule call back.
 */
static irqreturn_t xen_reschedule_interrupt(int irq, void *dev_id)
{
	inc_irq_stat(irq_resched_count);
	scheduler_ipi();

	return IRQ_HANDLED;
}

void xen_smp_intr_free(unsigned int cpu)
{
	if (per_cpu(xen_resched_irq, cpu).irq >= 0) {
		unbind_from_irqhandler(per_cpu(xen_resched_irq, cpu).irq, NULL);
		per_cpu(xen_resched_irq, cpu).irq = -1;
		kfree(per_cpu(xen_resched_irq, cpu).name);
		per_cpu(xen_resched_irq, cpu).name = NULL;
	}
	if (per_cpu(xen_callfunc_irq, cpu).irq >= 0) {
		unbind_from_irqhandler(per_cpu(xen_callfunc_irq, cpu).irq, NULL);
		per_cpu(xen_callfunc_irq, cpu).irq = -1;
		kfree(per_cpu(xen_callfunc_irq, cpu).name);
		per_cpu(xen_callfunc_irq, cpu).name = NULL;
	}
	if (per_cpu(xen_debug_irq, cpu).irq >= 0) {
		unbind_from_irqhandler(per_cpu(xen_debug_irq, cpu).irq, NULL);
		per_cpu(xen_debug_irq, cpu).irq = -1;
		kfree(per_cpu(xen_debug_irq, cpu).name);
		per_cpu(xen_debug_irq, cpu).name = NULL;
	}
	if (per_cpu(xen_callfuncsingle_irq, cpu).irq >= 0) {
		unbind_from_irqhandler(per_cpu(xen_callfuncsingle_irq, cpu).irq,
				       NULL);
		per_cpu(xen_callfuncsingle_irq, cpu).irq = -1;
		kfree(per_cpu(xen_callfuncsingle_irq, cpu).name);
		per_cpu(xen_callfuncsingle_irq, cpu).name = NULL;
	}
}

int xen_smp_intr_init(unsigned int cpu)
{
	int rc;
	char *resched_name, *callfunc_name, *debug_name;

	resched_name = kasprintf(GFP_KERNEL, "resched%d", cpu);
	rc = bind_ipi_to_irqhandler(XEN_RESCHEDULE_VECTOR,
				    cpu,
				    xen_reschedule_interrupt,
				    IRQF_PERCPU|IRQF_NOBALANCING,
				    resched_name,
				    NULL);
	if (rc < 0)
		goto fail;
	per_cpu(xen_resched_irq, cpu).irq = rc;
	per_cpu(xen_resched_irq, cpu).name = resched_name;

	callfunc_name = kasprintf(GFP_KERNEL, "callfunc%d", cpu);
	rc = bind_ipi_to_irqhandler(XEN_CALL_FUNCTION_VECTOR,
				    cpu,
				    xen_call_function_interrupt,
				    IRQF_PERCPU|IRQF_NOBALANCING,
				    callfunc_name,
				    NULL);
	if (rc < 0)
		goto fail;
	per_cpu(xen_callfunc_irq, cpu).irq = rc;
	per_cpu(xen_callfunc_irq, cpu).name = callfunc_name;

	debug_name = kasprintf(GFP_KERNEL, "debug%d", cpu);
	rc = bind_virq_to_irqhandler(VIRQ_DEBUG, cpu, xen_debug_interrupt,
				     IRQF_PERCPU | IRQF_NOBALANCING,
				     debug_name, NULL);
	if (rc < 0)
		goto fail;
	per_cpu(xen_debug_irq, cpu).irq = rc;
	per_cpu(xen_debug_irq, cpu).name = debug_name;

	callfunc_name = kasprintf(GFP_KERNEL, "callfuncsingle%d", cpu);
	rc = bind_ipi_to_irqhandler(XEN_CALL_FUNCTION_SINGLE_VECTOR,
				    cpu,
				    xen_call_function_single_interrupt,
				    IRQF_PERCPU|IRQF_NOBALANCING,
				    callfunc_name,
				    NULL);
	if (rc < 0)
		goto fail;
	per_cpu(xen_callfuncsingle_irq, cpu).irq = rc;
	per_cpu(xen_callfuncsingle_irq, cpu).name = callfunc_name;

	return 0;

 fail:
	xen_smp_intr_free(cpu);
	return rc;
}

void xen_smp_send_reschedule(int cpu)
{
	xen_send_IPI_one(cpu, XEN_RESCHEDULE_VECTOR);
}

static void __xen_send_IPI_mask(const struct cpumask *mask,
			      int vector)
{
	unsigned cpu;

	for_each_cpu_and(cpu, mask, cpu_online_mask)
		xen_send_IPI_one(cpu, vector);
}

void xen_smp_send_call_function_ipi(const struct cpumask *mask)
{
	int cpu;

	__xen_send_IPI_mask(mask, XEN_CALL_FUNCTION_VECTOR);

	/* Make sure other vcpus get a chance to run if they need to. */
	for_each_cpu(cpu, mask) {
		if (xen_vcpu_stolen(cpu)) {
			HYPERVISOR_sched_op(SCHEDOP_yield, NULL);
			break;
		}
	}
}

void xen_smp_send_call_function_single_ipi(int cpu)
{
	__xen_send_IPI_mask(cpumask_of(cpu),
			  XEN_CALL_FUNCTION_SINGLE_VECTOR);
}

static inline int xen_map_vector(int vector)
{
	int xen_vector;

	switch (vector) {
	case RESCHEDULE_VECTOR:
		xen_vector = XEN_RESCHEDULE_VECTOR;
		break;
	case CALL_FUNCTION_VECTOR:
		xen_vector = XEN_CALL_FUNCTION_VECTOR;
		break;
	case CALL_FUNCTION_SINGLE_VECTOR:
		xen_vector = XEN_CALL_FUNCTION_SINGLE_VECTOR;
		break;
	case IRQ_WORK_VECTOR:
		xen_vector = XEN_IRQ_WORK_VECTOR;
		break;
#ifdef CONFIG_X86_64
	case NMI_VECTOR:
	case APIC_DM_NMI: /* Some use that instead of NMI_VECTOR */
		xen_vector = XEN_NMI_VECTOR;
		break;
#endif
	default:
		xen_vector = -1;
		printk(KERN_ERR "xen: vector 0x%x is not implemented\n",
			vector);
	}

	return xen_vector;
}

void xen_send_IPI_mask(const struct cpumask *mask,
			      int vector)
{
	int xen_vector = xen_map_vector(vector);

	if (xen_vector >= 0)
		__xen_send_IPI_mask(mask, xen_vector);
}

void xen_send_IPI_all(int vector)
{
	int xen_vector = xen_map_vector(vector);

	if (xen_vector >= 0)
		__xen_send_IPI_mask(cpu_online_mask, xen_vector);
}

void xen_send_IPI_self(int vector)
{
	int xen_vector = xen_map_vector(vector);

	if (xen_vector >= 0)
		xen_send_IPI_one(smp_processor_id(), xen_vector);
}

void xen_send_IPI_mask_allbutself(const struct cpumask *mask,
				int vector)
{
	unsigned cpu;
	unsigned int this_cpu = smp_processor_id();
	int xen_vector = xen_map_vector(vector);

	if (!(num_online_cpus() > 1) || (xen_vector < 0))
		return;

	for_each_cpu_and(cpu, mask, cpu_online_mask) {
		if (this_cpu == cpu)
			continue;

		xen_send_IPI_one(cpu, xen_vector);
	}
}

void xen_send_IPI_allbutself(int vector)
{
	xen_send_IPI_mask_allbutself(cpu_online_mask, vector);
}

static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id)
{
	irq_enter();
	generic_smp_call_function_interrupt();
	inc_irq_stat(irq_call_count);
	irq_exit();

	return IRQ_HANDLED;
}

static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id)
{
	irq_enter();
	generic_smp_call_function_single_interrupt();
	inc_irq_stat(irq_call_count);
	irq_exit();

	return IRQ_HANDLED;
}
Commit	Line	Data
f87e4cac	1	#include <linux/smp.h>
83b96794 VK	2	#include <linux/slab.h>
	3	#include <linux/cpumask.h>
	4	#include <linux/percpu.h>
f87e4cac	5
f87e4cac JF	6	#include <xen/events.h>
f87e4cac JF	7
ed467e69	8	#include <xen/hvc-console.h>
f87e4cac	9	#include "xen-ops.h"
a2ef5dc2	10	#include "smp.h"
f87e4cac	11
ee336e10 KRW	12	static DEFINE_PER_CPU(struct xen_common_irq, xen_resched_irq) = { .irq = -1 };
	13	static DEFINE_PER_CPU(struct xen_common_irq, xen_callfunc_irq) = { .irq = -1 };
	14	static DEFINE_PER_CPU(struct xen_common_irq, xen_callfuncsingle_irq) = { .irq = -1 };
9547689f	15	static DEFINE_PER_CPU(struct xen_common_irq, xen_debug_irq) = { .irq = -1 };
f87e4cac JF	16
f87e4cac JF	17	static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id);
3b16cf87	18	static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id);
f87e4cac JF	19
f87e4cac JF	20	/*
184748cc	21	* Reschedule call back.
f87e4cac JF	22	*/
	23	static irqreturn_t xen_reschedule_interrupt(int irq, void *dev_id)
	24	{
1b437c8c	25	inc_irq_stat(irq_resched_count);
184748cc	26	scheduler_ipi();
38bb5ab4	27
f87e4cac JF	28	return IRQ_HANDLED;
	29	}
	30
5fc509bc	31	void xen_smp_intr_free(unsigned int cpu)
53b94fdc	32	{
ee336e10	33	if (per_cpu(xen_resched_irq, cpu).irq >= 0) {
9547689f	34	unbind_from_irqhandler(per_cpu(xen_resched_irq, cpu).irq, NULL);
ee336e10	35	per_cpu(xen_resched_irq, cpu).irq = -1;
b85fffec KRW	36	kfree(per_cpu(xen_resched_irq, cpu).name);
b85fffec KRW	37	per_cpu(xen_resched_irq, cpu).name = NULL;
ee336e10 KRW	38	}
ee336e10 KRW	39	if (per_cpu(xen_callfunc_irq, cpu).irq >= 0) {
9547689f	40	unbind_from_irqhandler(per_cpu(xen_callfunc_irq, cpu).irq, NULL);
ee336e10	41	per_cpu(xen_callfunc_irq, cpu).irq = -1;
b85fffec KRW	42	kfree(per_cpu(xen_callfunc_irq, cpu).name);
b85fffec KRW	43	per_cpu(xen_callfunc_irq, cpu).name = NULL;
ee336e10 KRW	44	}
ee336e10 KRW	45	if (per_cpu(xen_debug_irq, cpu).irq >= 0) {
9547689f	46	unbind_from_irqhandler(per_cpu(xen_debug_irq, cpu).irq, NULL);
ee336e10	47	per_cpu(xen_debug_irq, cpu).irq = -1;
b85fffec KRW	48	kfree(per_cpu(xen_debug_irq, cpu).name);
b85fffec KRW	49	per_cpu(xen_debug_irq, cpu).name = NULL;
ee336e10 KRW	50	}
ee336e10 KRW	51	if (per_cpu(xen_callfuncsingle_irq, cpu).irq >= 0) {
9547689f	52	unbind_from_irqhandler(per_cpu(xen_callfuncsingle_irq, cpu).irq,
53b94fdc	53	NULL);
ee336e10	54	per_cpu(xen_callfuncsingle_irq, cpu).irq = -1;
b85fffec KRW	55	kfree(per_cpu(xen_callfuncsingle_irq, cpu).name);
b85fffec KRW	56	per_cpu(xen_callfuncsingle_irq, cpu).name = NULL;
ee336e10	57	}
04e95761	58	}
53b94fdc	59
5fc509bc	60	int xen_smp_intr_init(unsigned int cpu)
f87e4cac JF	61	{
f87e4cac JF	62	int rc;
04e95761	63	char resched_name, callfunc_name, *debug_name;
f87e4cac JF	64
	65	resched_name = kasprintf(GFP_KERNEL, "resched%d", cpu);
	66	rc = bind_ipi_to_irqhandler(XEN_RESCHEDULE_VECTOR,
	67	cpu,
	68	xen_reschedule_interrupt,
9d71cee6	69	IRQF_PERCPU\|IRQF_NOBALANCING,
f87e4cac JF	70	resched_name,
	71	NULL);
	72	if (rc < 0)
	73	goto fail;
9547689f	74	per_cpu(xen_resched_irq, cpu).irq = rc;
b85fffec	75	per_cpu(xen_resched_irq, cpu).name = resched_name;
f87e4cac JF	76
	77	callfunc_name = kasprintf(GFP_KERNEL, "callfunc%d", cpu);
	78	rc = bind_ipi_to_irqhandler(XEN_CALL_FUNCTION_VECTOR,
	79	cpu,
	80	xen_call_function_interrupt,
9d71cee6	81	IRQF_PERCPU\|IRQF_NOBALANCING,
f87e4cac JF	82	callfunc_name,
	83	NULL);
	84	if (rc < 0)
	85	goto fail;
9547689f	86	per_cpu(xen_callfunc_irq, cpu).irq = rc;
b85fffec	87	per_cpu(xen_callfunc_irq, cpu).name = callfunc_name;
f87e4cac	88
ee523ca1 JF	89	debug_name = kasprintf(GFP_KERNEL, "debug%d", cpu);
ee523ca1 JF	90	rc = bind_virq_to_irqhandler(VIRQ_DEBUG, cpu, xen_debug_interrupt,
9d71cee6	91	IRQF_PERCPU \| IRQF_NOBALANCING,
ee523ca1 JF	92	debug_name, NULL);
	93	if (rc < 0)
	94	goto fail;
9547689f	95	per_cpu(xen_debug_irq, cpu).irq = rc;
b85fffec	96	per_cpu(xen_debug_irq, cpu).name = debug_name;
ee523ca1	97
3b16cf87 JA	98	callfunc_name = kasprintf(GFP_KERNEL, "callfuncsingle%d", cpu);
	99	rc = bind_ipi_to_irqhandler(XEN_CALL_FUNCTION_SINGLE_VECTOR,
	100	cpu,
	101	xen_call_function_single_interrupt,
9d71cee6	102	IRQF_PERCPU\|IRQF_NOBALANCING,
3b16cf87 JA	103	callfunc_name,
	104	NULL);
	105	if (rc < 0)
	106	goto fail;
9547689f	107	per_cpu(xen_callfuncsingle_irq, cpu).irq = rc;
b85fffec	108	per_cpu(xen_callfuncsingle_irq, cpu).name = callfunc_name;
3b16cf87	109
04e95761 VK	110	return 0;
	111
	112	fail:
	113	xen_smp_intr_free(cpu);
	114	return rc;
	115	}
	116
a52482d9	117	void xen_smp_send_reschedule(int cpu)
f87e4cac JF	118	{
	119	xen_send_IPI_one(cpu, XEN_RESCHEDULE_VECTOR);
	120	}
	121
f447d56d BG	122	static void __xen_send_IPI_mask(const struct cpumask *mask,
f447d56d BG	123	int vector)
f87e4cac JF	124	{
	125	unsigned cpu;
	126
bcda016e	127	for_each_cpu_and(cpu, mask, cpu_online_mask)
f87e4cac JF	128	xen_send_IPI_one(cpu, vector);
	129	}
	130
a52482d9	131	void xen_smp_send_call_function_ipi(const struct cpumask *mask)
3b16cf87 JA	132	{
	133	int cpu;
	134
f447d56d	135	__xen_send_IPI_mask(mask, XEN_CALL_FUNCTION_VECTOR);
3b16cf87 JA	136
3b16cf87 JA	137	/* Make sure other vcpus get a chance to run if they need to. */
bcda016e	138	for_each_cpu(cpu, mask) {
3b16cf87	139	if (xen_vcpu_stolen(cpu)) {
1207cf8e	140	HYPERVISOR_sched_op(SCHEDOP_yield, NULL);
3b16cf87 JA	141	break;
	142	}
	143	}
	144	}
	145
a52482d9	146	void xen_smp_send_call_function_single_ipi(int cpu)
3b16cf87	147	{
f447d56d	148	__xen_send_IPI_mask(cpumask_of(cpu),
e7986739	149	XEN_CALL_FUNCTION_SINGLE_VECTOR);
3b16cf87 JA	150	}
3b16cf87 JA	151
f447d56d BG	152	static inline int xen_map_vector(int vector)
	153	{
	154	int xen_vector;
	155
	156	switch (vector) {
	157	case RESCHEDULE_VECTOR:
	158	xen_vector = XEN_RESCHEDULE_VECTOR;
	159	break;
	160	case CALL_FUNCTION_VECTOR:
	161	xen_vector = XEN_CALL_FUNCTION_VECTOR;
	162	break;
	163	case CALL_FUNCTION_SINGLE_VECTOR:
	164	xen_vector = XEN_CALL_FUNCTION_SINGLE_VECTOR;
	165	break;
1ff2b0c3 LM	166	case IRQ_WORK_VECTOR:
	167	xen_vector = XEN_IRQ_WORK_VECTOR;
	168	break;
6efa20e4 KRW	169	#ifdef CONFIG_X86_64
	170	case NMI_VECTOR:
	171	case APIC_DM_NMI: /* Some use that instead of NMI_VECTOR */
	172	xen_vector = XEN_NMI_VECTOR;
	173	break;
	174	#endif
f447d56d BG	175	default:
	176	xen_vector = -1;
	177	printk(KERN_ERR "xen: vector 0x%x is not implemented\n",
	178	vector);
	179	}
	180
	181	return xen_vector;
	182	}
	183
	184	void xen_send_IPI_mask(const struct cpumask *mask,
	185	int vector)
	186	{
	187	int xen_vector = xen_map_vector(vector);
	188
	189	if (xen_vector >= 0)
	190	__xen_send_IPI_mask(mask, xen_vector);
	191	}
	192
	193	void xen_send_IPI_all(int vector)
	194	{
	195	int xen_vector = xen_map_vector(vector);
	196
	197	if (xen_vector >= 0)
	198	__xen_send_IPI_mask(cpu_online_mask, xen_vector);
	199	}
	200
	201	void xen_send_IPI_self(int vector)
	202	{
	203	int xen_vector = xen_map_vector(vector);
	204
	205	if (xen_vector >= 0)
	206	xen_send_IPI_one(smp_processor_id(), xen_vector);
	207	}
	208
	209	void xen_send_IPI_mask_allbutself(const struct cpumask *mask,
	210	int vector)
	211	{
	212	unsigned cpu;
	213	unsigned int this_cpu = smp_processor_id();
1db01b49	214	int xen_vector = xen_map_vector(vector);
f447d56d	215
1db01b49	216	if (!(num_online_cpus() > 1) \|\| (xen_vector < 0))
f447d56d BG	217	return;
	218
	219	for_each_cpu_and(cpu, mask, cpu_online_mask) {
	220	if (this_cpu == cpu)
	221	continue;
	222
1db01b49	223	xen_send_IPI_one(cpu, xen_vector);
f447d56d BG	224	}
	225	}
	226
	227	void xen_send_IPI_allbutself(int vector)
	228	{
1db01b49	229	xen_send_IPI_mask_allbutself(cpu_online_mask, vector);
f447d56d BG	230	}
f447d56d BG	231
f87e4cac JF	232	static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id)
f87e4cac JF	233	{
f87e4cac	234	irq_enter();
3b16cf87	235	generic_smp_call_function_interrupt();
1b437c8c	236	inc_irq_stat(irq_call_count);
f87e4cac JF	237	irq_exit();
f87e4cac JF	238
f87e4cac JF	239	return IRQ_HANDLED;
	240	}
	241
3b16cf87	242	static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id)
f87e4cac	243	{
3b16cf87 JA	244	irq_enter();
3b16cf87 JA	245	generic_smp_call_function_single_interrupt();
1b437c8c	246	inc_irq_stat(irq_call_count);
3b16cf87	247	irq_exit();
f87e4cac	248
3b16cf87	249	return IRQ_HANDLED;
f87e4cac	250	}