[linux-2.6-block.git] / kernel / smp.c

/*
 * Generic helpers for smp ipi calls
 *
 * (C) Jens Axboe <jens.axboe@oracle.com> 2008
 *
 */
#include <linux/init.h>
#include <linux/module.h>
#include <linux/percpu.h>
#include <linux/rcupdate.h>
#include <linux/rculist.h>
#include <linux/smp.h>

static DEFINE_PER_CPU(struct call_single_queue, call_single_queue);
static LIST_HEAD(call_function_queue);
__cacheline_aligned_in_smp DEFINE_SPINLOCK(call_function_lock);

enum {
	CSD_FLAG_WAIT		= 0x01,
	CSD_FLAG_ALLOC		= 0x02,
	CSD_FLAG_LOCK		= 0x04,
};

struct call_function_data {
	struct call_single_data csd;
	spinlock_t lock;
	unsigned int refs;
	struct rcu_head rcu_head;
	unsigned long cpumask_bits[];
};

struct call_single_queue {
	struct list_head list;
	spinlock_t lock;
};

static int __cpuinit init_call_single_data(void)
{
	int i;

	for_each_possible_cpu(i) {
		struct call_single_queue *q = &per_cpu(call_single_queue, i);

		spin_lock_init(&q->lock);
		INIT_LIST_HEAD(&q->list);
	}
	return 0;
}
early_initcall(init_call_single_data);

static void csd_flag_wait(struct call_single_data *data)
{
	/* Wait for response */
	do {
		if (!(data->flags & CSD_FLAG_WAIT))
			break;
		cpu_relax();
	} while (1);
}

/*
 * Insert a previously allocated call_single_data element for execution
 * on the given CPU. data must already have ->func, ->info, and ->flags set.
 */
static void generic_exec_single(int cpu, struct call_single_data *data)
{
	struct call_single_queue *dst = &per_cpu(call_single_queue, cpu);
	int wait = data->flags & CSD_FLAG_WAIT, ipi;
	unsigned long flags;

	spin_lock_irqsave(&dst->lock, flags);
	ipi = list_empty(&dst->list);
	list_add_tail(&data->list, &dst->list);
	spin_unlock_irqrestore(&dst->lock, flags);

	/*
	 * Make the list addition visible before sending the ipi.
	 */
	smp_mb();

	if (ipi)
		arch_send_call_function_single_ipi(cpu);

	if (wait)
		csd_flag_wait(data);
}

static void rcu_free_call_data(struct rcu_head *head)
{
	struct call_function_data *data;

	data = container_of(head, struct call_function_data, rcu_head);

	kfree(data);
}

/*
 * Invoked by arch to handle an IPI for call function. Must be called with
 * interrupts disabled.
 */
void generic_smp_call_function_interrupt(void)
{
	struct call_function_data *data;
	int cpu = get_cpu();

	/*
	 * It's ok to use list_for_each_rcu() here even though we may delete
	 * 'pos', since list_del_rcu() doesn't clear ->next
	 */
	rcu_read_lock();
	list_for_each_entry_rcu(data, &call_function_queue, csd.list) {
		int refs;

		if (!cpumask_test_cpu(cpu, to_cpumask(data->cpumask_bits)))
			continue;

		data->csd.func(data->csd.info);

		spin_lock(&data->lock);
		cpumask_clear_cpu(cpu, to_cpumask(data->cpumask_bits));
		WARN_ON(data->refs == 0);
		data->refs--;
		refs = data->refs;
		spin_unlock(&data->lock);

		if (refs)
			continue;

		spin_lock(&call_function_lock);
		list_del_rcu(&data->csd.list);
		spin_unlock(&call_function_lock);

		if (data->csd.flags & CSD_FLAG_WAIT) {
			/*
			 * serialize stores to data with the flag clear
			 * and wakeup
			 */
			smp_wmb();
			data->csd.flags &= ~CSD_FLAG_WAIT;
		}
		if (data->csd.flags & CSD_FLAG_ALLOC)
			call_rcu(&data->rcu_head, rcu_free_call_data);
	}
	rcu_read_unlock();

	put_cpu();
}

/*
 * Invoked by arch to handle an IPI for call function single. Must be called
 * from the arch with interrupts disabled.
 */
void generic_smp_call_function_single_interrupt(void)
{
	struct call_single_queue *q = &__get_cpu_var(call_single_queue);
	LIST_HEAD(list);

	/*
	 * Need to see other stores to list head for checking whether
	 * list is empty without holding q->lock
	 */
	smp_read_barrier_depends();
	while (!list_empty(&q->list)) {
		unsigned int data_flags;

		spin_lock(&q->lock);
		list_replace_init(&q->list, &list);
		spin_unlock(&q->lock);

		while (!list_empty(&list)) {
			struct call_single_data *data;

			data = list_entry(list.next, struct call_single_data,
						list);
			list_del(&data->list);

			/*
			 * 'data' can be invalid after this call if
			 * flags == 0 (when called through
			 * generic_exec_single(), so save them away before
			 * making the call.
			 */
			data_flags = data->flags;

			data->func(data->info);

			if (data_flags & CSD_FLAG_WAIT) {
				smp_wmb();
				data->flags &= ~CSD_FLAG_WAIT;
			} else if (data_flags & CSD_FLAG_LOCK) {
				smp_wmb();
				data->flags &= ~CSD_FLAG_LOCK;
			} else if (data_flags & CSD_FLAG_ALLOC)
				kfree(data);
		}
		/*
		 * See comment on outer loop
		 */
		smp_read_barrier_depends();
	}
}

static DEFINE_PER_CPU(struct call_single_data, csd_data);

/*
 * smp_call_function_single - Run a function on a specific CPU
 * @func: The function to run. This must be fast and non-blocking.
 * @info: An arbitrary pointer to pass to the function.
 * @wait: If true, wait until function has completed on other CPUs.
 *
 * Returns 0 on success, else a negative status code. Note that @wait
 * will be implicitly turned on in case of allocation failures, since
 * we fall back to on-stack allocation.
 */
int smp_call_function_single(int cpu, void (*func) (void *info), void *info,
			     int wait)
{
	struct call_single_data d;
	unsigned long flags;
	/* prevent preemption and reschedule on another processor,
	   as well as CPU removal */
	int me = get_cpu();
	int err = 0;

	/* Can deadlock when called with interrupts disabled */
	WARN_ON(irqs_disabled());

	if (cpu == me) {
		local_irq_save(flags);
		func(info);
		local_irq_restore(flags);
	} else if ((unsigned)cpu < nr_cpu_ids && cpu_online(cpu)) {
		struct call_single_data *data;

		if (!wait) {
			/*
			 * We are calling a function on a single CPU
			 * and we are not going to wait for it to finish.
			 * We first try to allocate the data, but if we
			 * fail, we fall back to use a per cpu data to pass
			 * the information to that CPU. Since all callers
			 * of this code will use the same data, we must
			 * synchronize the callers to prevent a new caller
			 * from corrupting the data before the callee
			 * can access it.
			 *
			 * The CSD_FLAG_LOCK is used to let us know when
			 * the IPI handler is done with the data.
			 * The first caller will set it, and the callee
			 * will clear it. The next caller must wait for
			 * it to clear before we set it again. This
			 * will make sure the callee is done with the
			 * data before a new caller will use it.
			 */
			data = kmalloc(sizeof(*data), GFP_ATOMIC);
			if (data)
				data->flags = CSD_FLAG_ALLOC;
			else {
				data = &per_cpu(csd_data, me);
				while (data->flags & CSD_FLAG_LOCK)
					cpu_relax();
				data->flags = CSD_FLAG_LOCK;
			}
		} else {
			data = &d;
			data->flags = CSD_FLAG_WAIT;
		}

		data->func = func;
		data->info = info;
		generic_exec_single(cpu, data);
	} else {
		err = -ENXIO;	/* CPU not online */
	}

	put_cpu();
	return err;
}
EXPORT_SYMBOL(smp_call_function_single);

/**
 * __smp_call_function_single(): Run a function on another CPU
 * @cpu: The CPU to run on.
 * @data: Pre-allocated and setup data structure
 *
 * Like smp_call_function_single(), but allow caller to pass in a pre-allocated
 * data structure. Useful for embedding @data inside other structures, for
 * instance.
 *
 */
void __smp_call_function_single(int cpu, struct call_single_data *data)
{
	/* Can deadlock when called with interrupts disabled */
	WARN_ON((data->flags & CSD_FLAG_WAIT) && irqs_disabled());

	generic_exec_single(cpu, data);
}

/* FIXME: Shim for archs using old arch_send_call_function_ipi API. */
#ifndef arch_send_call_function_ipi_mask
#define arch_send_call_function_ipi_mask(maskp) \
	arch_send_call_function_ipi(*(maskp))
#endif

/**
 * smp_call_function_many(): Run a function on a set of other CPUs.
 * @mask: The set of cpus to run on (only runs on online subset).
 * @func: The function to run. This must be fast and non-blocking.
 * @info: An arbitrary pointer to pass to the function.
 * @wait: If true, wait (atomically) until function has completed on other CPUs.
 *
 * If @wait is true, then returns once @func has returned. Note that @wait
 * will be implicitly turned on in case of allocation failures, since
 * we fall back to on-stack allocation.
 *
 * You must not call this function with disabled interrupts or from a
 * hardware interrupt handler or from a bottom half handler. Preemption
 * must be disabled when calling this function.
 */
void smp_call_function_many(const struct cpumask *mask,
			    void (*func)(void *), void *info,
			    bool wait)
{
	struct call_function_data *data;
	unsigned long flags;
	int cpu, next_cpu;

	/* Can deadlock when called with interrupts disabled */
	WARN_ON(irqs_disabled());

	/* So, what's a CPU they want?  Ignoring this one. */
	cpu = cpumask_first_and(mask, cpu_online_mask);
	if (cpu == smp_processor_id())
		cpu = cpumask_next_and(cpu, mask, cpu_online_mask);
	/* No online cpus?  We're done. */
	if (cpu >= nr_cpu_ids)
		return;

	/* Do we have another CPU which isn't us? */
	next_cpu = cpumask_next_and(cpu, mask, cpu_online_mask);
	if (next_cpu == smp_processor_id())
		next_cpu = cpumask_next_and(next_cpu, mask, cpu_online_mask);

	/* Fastpath: do that cpu by itself. */
	if (next_cpu >= nr_cpu_ids) {
		smp_call_function_single(cpu, func, info, wait);
		return;
	}

	data = kmalloc(sizeof(*data) + cpumask_size(), GFP_ATOMIC);
	if (unlikely(!data)) {
		/* Slow path. */
		for_each_online_cpu(cpu) {
			if (cpu == smp_processor_id())
				continue;
			if (cpumask_test_cpu(cpu, mask))
				smp_call_function_single(cpu, func, info, wait);
		}
		return;
	}

	spin_lock_init(&data->lock);
	data->csd.flags = CSD_FLAG_ALLOC;
	if (wait)
		data->csd.flags |= CSD_FLAG_WAIT;
	data->csd.func = func;
	data->csd.info = info;
	cpumask_and(to_cpumask(data->cpumask_bits), mask, cpu_online_mask);
	cpumask_clear_cpu(smp_processor_id(), to_cpumask(data->cpumask_bits));
	data->refs = cpumask_weight(to_cpumask(data->cpumask_bits));

	spin_lock_irqsave(&call_function_lock, flags);
	list_add_tail_rcu(&data->csd.list, &call_function_queue);
	spin_unlock_irqrestore(&call_function_lock, flags);

	/*
	 * Make the list addition visible before sending the ipi.
	 */
	smp_mb();

	/* Send a message to all CPUs in the map */
	arch_send_call_function_ipi_mask(to_cpumask(data->cpumask_bits));

	/* optionally wait for the CPUs to complete */
	if (wait)
		csd_flag_wait(&data->csd);
}
EXPORT_SYMBOL(smp_call_function_many);

/**
 * smp_call_function(): Run a function on all other CPUs.
 * @func: The function to run. This must be fast and non-blocking.
 * @info: An arbitrary pointer to pass to the function.
 * @wait: If true, wait (atomically) until function has completed on other CPUs.
 *
 * Returns 0.
 *
 * If @wait is true, then returns once @func has returned; otherwise
 * it returns just before the target cpu calls @func. In case of allocation
 * failure, @wait will be implicitly turned on.
 *
 * You must not call this function with disabled interrupts or from a
 * hardware interrupt handler or from a bottom half handler.
 */
int smp_call_function(void (*func)(void *), void *info, int wait)
{
	preempt_disable();
	smp_call_function_many(cpu_online_mask, func, info, wait);
	preempt_enable();
	return 0;
}
EXPORT_SYMBOL(smp_call_function);

void ipi_call_lock(void)
{
	spin_lock(&call_function_lock);
}

void ipi_call_unlock(void)
{
	spin_unlock(&call_function_lock);
}

void ipi_call_lock_irq(void)
{
	spin_lock_irq(&call_function_lock);
}

void ipi_call_unlock_irq(void)
{
	spin_unlock_irq(&call_function_lock);
}
Commit	Line	Data
3d442233 JA	1	/*
	2	* Generic helpers for smp ipi calls
	3	*
	4	* (C) Jens Axboe <jens.axboe@oracle.com> 2008
	5	*
	6	*/
	7	#include <linux/init.h>
	8	#include <linux/module.h>
	9	#include <linux/percpu.h>
	10	#include <linux/rcupdate.h>
59190f42	11	#include <linux/rculist.h>
3d442233 JA	12	#include <linux/smp.h>
	13
	14	static DEFINE_PER_CPU(struct call_single_queue, call_single_queue);
	15	static LIST_HEAD(call_function_queue);
	16	__cacheline_aligned_in_smp DEFINE_SPINLOCK(call_function_lock);
	17
	18	enum {
	19	CSD_FLAG_WAIT = 0x01,
	20	CSD_FLAG_ALLOC = 0x02,
d7240b98	21	CSD_FLAG_LOCK = 0x04,
3d442233 JA	22	};
	23
	24	struct call_function_data {
	25	struct call_single_data csd;
	26	spinlock_t lock;
	27	unsigned int refs;
3d442233	28	struct rcu_head rcu_head;
54b11e6d	29	unsigned long cpumask_bits[];
3d442233 JA	30	};
	31
	32	struct call_single_queue {
	33	struct list_head list;
	34	spinlock_t lock;
	35	};
	36
7babe8db	37	static int __cpuinit init_call_single_data(void)
3d442233 JA	38	{
	39	int i;
	40
	41	for_each_possible_cpu(i) {
	42	struct call_single_queue *q = &per_cpu(call_single_queue, i);
	43
	44	spin_lock_init(&q->lock);
	45	INIT_LIST_HEAD(&q->list);
	46	}
7babe8db	47	return 0;
3d442233	48	}
7babe8db	49	early_initcall(init_call_single_data);
3d442233 JA	50
	51	static void csd_flag_wait(struct call_single_data *data)
	52	{
	53	/* Wait for response */
	54	do {
3d442233 JA	55	if (!(data->flags & CSD_FLAG_WAIT))
	56	break;
	57	cpu_relax();
	58	} while (1);
	59	}
	60
	61	/*
	62	* Insert a previously allocated call_single_data element for execution
	63	* on the given CPU. data must already have ->func, ->info, and ->flags set.
	64	*/
	65	static void generic_exec_single(int cpu, struct call_single_data *data)
	66	{
	67	struct call_single_queue *dst = &per_cpu(call_single_queue, cpu);
	68	int wait = data->flags & CSD_FLAG_WAIT, ipi;
	69	unsigned long flags;
	70
	71	spin_lock_irqsave(&dst->lock, flags);
	72	ipi = list_empty(&dst->list);
	73	list_add_tail(&data->list, &dst->list);
	74	spin_unlock_irqrestore(&dst->lock, flags);
	75
561920a0 SS	76	/*
	77	* Make the list addition visible before sending the ipi.
	78	*/
	79	smp_mb();
	80
3d442233 JA	81	if (ipi)
	82	arch_send_call_function_single_ipi(cpu);
	83
	84	if (wait)
	85	csd_flag_wait(data);
	86	}
	87
	88	static void rcu_free_call_data(struct rcu_head *head)
	89	{
	90	struct call_function_data *data;
	91
	92	data = container_of(head, struct call_function_data, rcu_head);
	93
	94	kfree(data);
	95	}
	96
	97	/*
	98	* Invoked by arch to handle an IPI for call function. Must be called with
	99	* interrupts disabled.
	100	*/
	101	void generic_smp_call_function_interrupt(void)
	102	{
	103	struct call_function_data *data;
	104	int cpu = get_cpu();
	105
	106	/*
	107	* It's ok to use list_for_each_rcu() here even though we may delete
	108	* 'pos', since list_del_rcu() doesn't clear ->next
	109	*/
	110	rcu_read_lock();
	111	list_for_each_entry_rcu(data, &call_function_queue, csd.list) {
	112	int refs;
	113
54b11e6d	114	if (!cpumask_test_cpu(cpu, to_cpumask(data->cpumask_bits)))
3d442233 JA	115	continue;
	116
	117	data->csd.func(data->csd.info);
	118
	119	spin_lock(&data->lock);
54b11e6d	120	cpumask_clear_cpu(cpu, to_cpumask(data->cpumask_bits));
3d442233 JA	121	WARN_ON(data->refs == 0);
	122	data->refs--;
	123	refs = data->refs;
	124	spin_unlock(&data->lock);
	125
	126	if (refs)
	127	continue;
	128
	129	spin_lock(&call_function_lock);
	130	list_del_rcu(&data->csd.list);
	131	spin_unlock(&call_function_lock);
	132
	133	if (data->csd.flags & CSD_FLAG_WAIT) {
	134	/*
	135	* serialize stores to data with the flag clear
	136	* and wakeup
	137	*/
	138	smp_wmb();
	139	data->csd.flags &= ~CSD_FLAG_WAIT;
c2fc1198 NP	140	}
c2fc1198 NP	141	if (data->csd.flags & CSD_FLAG_ALLOC)
3d442233 JA	142	call_rcu(&data->rcu_head, rcu_free_call_data);
	143	}
	144	rcu_read_unlock();
	145
	146	put_cpu();
	147	}
	148
	149	/*
	150	* Invoked by arch to handle an IPI for call function single. Must be called
	151	* from the arch with interrupts disabled.
	152	*/
	153	void generic_smp_call_function_single_interrupt(void)
	154	{
	155	struct call_single_queue *q = &__get_cpu_var(call_single_queue);
	156	LIST_HEAD(list);
	157
	158	/*
	159	* Need to see other stores to list head for checking whether
	160	* list is empty without holding q->lock
	161	*/
561920a0	162	smp_read_barrier_depends();
3d442233 JA	163	while (!list_empty(&q->list)) {
	164	unsigned int data_flags;
	165
	166	spin_lock(&q->lock);
	167	list_replace_init(&q->list, &list);
	168	spin_unlock(&q->lock);
	169
	170	while (!list_empty(&list)) {
	171	struct call_single_data *data;
	172
	173	data = list_entry(list.next, struct call_single_data,
	174	list);
	175	list_del(&data->list);
	176
	177	/*
	178	* 'data' can be invalid after this call if
	179	* flags == 0 (when called through
	180	* generic_exec_single(), so save them away before
	181	* making the call.
	182	*/
	183	data_flags = data->flags;
	184
	185	data->func(data->info);
	186
	187	if (data_flags & CSD_FLAG_WAIT) {
	188	smp_wmb();
	189	data->flags &= ~CSD_FLAG_WAIT;
d7240b98 SR	190	} else if (data_flags & CSD_FLAG_LOCK) {
	191	smp_wmb();
	192	data->flags &= ~CSD_FLAG_LOCK;
3d442233 JA	193	} else if (data_flags & CSD_FLAG_ALLOC)
	194	kfree(data);
	195	}
	196	/*
	197	* See comment on outer loop
	198	*/
561920a0	199	smp_read_barrier_depends();
3d442233 JA	200	}
	201	}
	202
d7240b98 SR	203	static DEFINE_PER_CPU(struct call_single_data, csd_data);
d7240b98 SR	204
3d442233 JA	205	/*
	206	* smp_call_function_single - Run a function on a specific CPU
	207	* @func: The function to run. This must be fast and non-blocking.
	208	* @info: An arbitrary pointer to pass to the function.
3d442233 JA	209	* @wait: If true, wait until function has completed on other CPUs.
	210	*
	211	* Returns 0 on success, else a negative status code. Note that @wait
	212	* will be implicitly turned on in case of allocation failures, since
	213	* we fall back to on-stack allocation.
	214	*/
	215	int smp_call_function_single(int cpu, void (func) (void info), void *info,
8691e5a8	216	int wait)
3d442233 JA	217	{
	218	struct call_single_data d;
	219	unsigned long flags;
f73be6de PA	220	/* prevent preemption and reschedule on another processor,
f73be6de PA	221	as well as CPU removal */
3d442233	222	int me = get_cpu();
f73be6de	223	int err = 0;
3d442233 JA	224
	225	/* Can deadlock when called with interrupts disabled */
	226	WARN_ON(irqs_disabled());
	227
	228	if (cpu == me) {
	229	local_irq_save(flags);
	230	func(info);
	231	local_irq_restore(flags);
4f4b6c1a	232	} else if ((unsigned)cpu < nr_cpu_ids && cpu_online(cpu)) {
d7240b98	233	struct call_single_data *data;
3d442233 JA	234
3d442233 JA	235	if (!wait) {
d7240b98 SR	236	/*
	237	* We are calling a function on a single CPU
	238	* and we are not going to wait for it to finish.
	239	* We first try to allocate the data, but if we
	240	* fail, we fall back to use a per cpu data to pass
	241	* the information to that CPU. Since all callers
	242	* of this code will use the same data, we must
	243	* synchronize the callers to prevent a new caller
	244	* from corrupting the data before the callee
	245	* can access it.
	246	*
	247	* The CSD_FLAG_LOCK is used to let us know when
	248	* the IPI handler is done with the data.
	249	* The first caller will set it, and the callee
	250	* will clear it. The next caller must wait for
	251	* it to clear before we set it again. This
	252	* will make sure the callee is done with the
	253	* data before a new caller will use it.
	254	*/
3d442233 JA	255	data = kmalloc(sizeof(*data), GFP_ATOMIC);
	256	if (data)
	257	data->flags = CSD_FLAG_ALLOC;
d7240b98 SR	258	else {
	259	data = &per_cpu(csd_data, me);
	260	while (data->flags & CSD_FLAG_LOCK)
	261	cpu_relax();
	262	data->flags = CSD_FLAG_LOCK;
	263	}
	264	} else {
3d442233 JA	265	data = &d;
	266	data->flags = CSD_FLAG_WAIT;
	267	}
	268
	269	data->func = func;
	270	data->info = info;
	271	generic_exec_single(cpu, data);
f73be6de PA	272	} else {
f73be6de PA	273	err = -ENXIO; /* CPU not online */
3d442233 JA	274	}
	275
	276	put_cpu();
f73be6de	277	return err;
3d442233 JA	278	}
	279	EXPORT_SYMBOL(smp_call_function_single);
	280
	281	/**
	282	* __smp_call_function_single(): Run a function on another CPU
	283	* @cpu: The CPU to run on.
	284	* @data: Pre-allocated and setup data structure
	285	*
	286	* Like smp_call_function_single(), but allow caller to pass in a pre-allocated
	287	* data structure. Useful for embedding @data inside other structures, for
	288	* instance.
	289	*
	290	*/
	291	void __smp_call_function_single(int cpu, struct call_single_data *data)
	292	{
	293	/* Can deadlock when called with interrupts disabled */
	294	WARN_ON((data->flags & CSD_FLAG_WAIT) && irqs_disabled());
	295
	296	generic_exec_single(cpu, data);
	297	}
	298
ce47d974 RR	299	/* FIXME: Shim for archs using old arch_send_call_function_ipi API. */
	300	#ifndef arch_send_call_function_ipi_mask
	301	#define arch_send_call_function_ipi_mask(maskp) \
	302	arch_send_call_function_ipi(*(maskp))
	303	#endif
	304
3d442233	305	/**
54b11e6d RR	306	* smp_call_function_many(): Run a function on a set of other CPUs.
54b11e6d RR	307	* @mask: The set of cpus to run on (only runs on online subset).
3d442233 JA	308	* @func: The function to run. This must be fast and non-blocking.
	309	* @info: An arbitrary pointer to pass to the function.
	310	* @wait: If true, wait (atomically) until function has completed on other CPUs.
	311	*
3d442233 JA	312	* If @wait is true, then returns once @func has returned. Note that @wait
	313	* will be implicitly turned on in case of allocation failures, since
	314	* we fall back to on-stack allocation.
	315	*
	316	* You must not call this function with disabled interrupts or from a
	317	* hardware interrupt handler or from a bottom half handler. Preemption
	318	* must be disabled when calling this function.
	319	*/
54b11e6d RR	320	void smp_call_function_many(const struct cpumask *mask,
	321	void (func)(void ), void *info,
	322	bool wait)
3d442233	323	{
54b11e6d	324	struct call_function_data *data;
3d442233	325	unsigned long flags;
54b11e6d	326	int cpu, next_cpu;
3d442233 JA	327
	328	/* Can deadlock when called with interrupts disabled */
	329	WARN_ON(irqs_disabled());
	330
54b11e6d RR	331	/* So, what's a CPU they want? Ignoring this one. */
	332	cpu = cpumask_first_and(mask, cpu_online_mask);
	333	if (cpu == smp_processor_id())
	334	cpu = cpumask_next_and(cpu, mask, cpu_online_mask);
	335	/* No online cpus? We're done. */
	336	if (cpu >= nr_cpu_ids)
	337	return;
	338
	339	/* Do we have another CPU which isn't us? */
	340	next_cpu = cpumask_next_and(cpu, mask, cpu_online_mask);
	341	if (next_cpu == smp_processor_id())
	342	next_cpu = cpumask_next_and(next_cpu, mask, cpu_online_mask);
	343
	344	/* Fastpath: do that cpu by itself. */
	345	if (next_cpu >= nr_cpu_ids) {
	346	smp_call_function_single(cpu, func, info, wait);
	347	return;
3d442233 JA	348	}
3d442233 JA	349
54b11e6d RR	350	data = kmalloc(sizeof(*data) + cpumask_size(), GFP_ATOMIC);
	351	if (unlikely(!data)) {
	352	/* Slow path. */
	353	for_each_online_cpu(cpu) {
	354	if (cpu == smp_processor_id())
	355	continue;
	356	if (cpumask_test_cpu(cpu, mask))
	357	smp_call_function_single(cpu, func, info, wait);
	358	}
	359	return;
3d442233 JA	360	}
	361
	362	spin_lock_init(&data->lock);
54b11e6d RR	363	data->csd.flags = CSD_FLAG_ALLOC;
	364	if (wait)
	365	data->csd.flags \|= CSD_FLAG_WAIT;
3d442233 JA	366	data->csd.func = func;
3d442233 JA	367	data->csd.info = info;
54b11e6d RR	368	cpumask_and(to_cpumask(data->cpumask_bits), mask, cpu_online_mask);
	369	cpumask_clear_cpu(smp_processor_id(), to_cpumask(data->cpumask_bits));
	370	data->refs = cpumask_weight(to_cpumask(data->cpumask_bits));
3d442233 JA	371
	372	spin_lock_irqsave(&call_function_lock, flags);
	373	list_add_tail_rcu(&data->csd.list, &call_function_queue);
	374	spin_unlock_irqrestore(&call_function_lock, flags);
	375
561920a0 SS	376	/*
	377	* Make the list addition visible before sending the ipi.
	378	*/
	379	smp_mb();
	380
3d442233	381	/* Send a message to all CPUs in the map */
ce47d974	382	arch_send_call_function_ipi_mask(to_cpumask(data->cpumask_bits));
3d442233 JA	383
3d442233 JA	384	/* optionally wait for the CPUs to complete */
54b11e6d	385	if (wait)
3d442233	386	csd_flag_wait(&data->csd);
3d442233	387	}
54b11e6d	388	EXPORT_SYMBOL(smp_call_function_many);
3d442233 JA	389
	390	/**
	391	* smp_call_function(): Run a function on all other CPUs.
	392	* @func: The function to run. This must be fast and non-blocking.
	393	* @info: An arbitrary pointer to pass to the function.
3d442233 JA	394	* @wait: If true, wait (atomically) until function has completed on other CPUs.
3d442233 JA	395	*
54b11e6d	396	* Returns 0.
3d442233 JA	397	*
	398	* If @wait is true, then returns once @func has returned; otherwise
	399	* it returns just before the target cpu calls @func. In case of allocation
	400	* failure, @wait will be implicitly turned on.
	401	*
	402	* You must not call this function with disabled interrupts or from a
	403	* hardware interrupt handler or from a bottom half handler.
	404	*/
8691e5a8	405	int smp_call_function(void (func)(void ), void *info, int wait)
3d442233	406	{
3d442233	407	preempt_disable();
54b11e6d	408	smp_call_function_many(cpu_online_mask, func, info, wait);
3d442233	409	preempt_enable();
54b11e6d	410	return 0;
3d442233 JA	411	}
	412	EXPORT_SYMBOL(smp_call_function);
	413
	414	void ipi_call_lock(void)
	415	{
	416	spin_lock(&call_function_lock);
	417	}
	418
	419	void ipi_call_unlock(void)
	420	{
	421	spin_unlock(&call_function_lock);
	422	}
	423
	424	void ipi_call_lock_irq(void)
	425	{
	426	spin_lock_irq(&call_function_lock);
	427	}
	428
	429	void ipi_call_unlock_irq(void)
	430	{
	431	spin_unlock_irq(&call_function_lock);
	432	}