[linux-2.6-block.git] / kernel / softirq.c

/*
 *	linux/kernel/softirq.c
 *
 *	Copyright (C) 1992 Linus Torvalds
 *
 * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
 */

#include <linux/module.h>
#include <linux/kernel_stat.h>
#include <linux/interrupt.h>
#include <linux/init.h>
#include <linux/mm.h>
#include <linux/notifier.h>
#include <linux/percpu.h>
#include <linux/cpu.h>
#include <linux/kthread.h>
#include <linux/rcupdate.h>
#include <linux/smp.h>
#include <linux/tick.h>

#include <asm/irq.h>
/*
   - No shared variables, all the data are CPU local.
   - If a softirq needs serialization, let it serialize itself
     by its own spinlocks.
   - Even if softirq is serialized, only local cpu is marked for
     execution. Hence, we get something sort of weak cpu binding.
     Though it is still not clear, will it result in better locality
     or will not.

   Examples:
   - NET RX softirq. It is multithreaded and does not require
     any global serialization.
   - NET TX softirq. It kicks software netdevice queues, hence
     it is logically serialized per device, but this serialization
     is invisible to common code.
   - Tasklets: serialized wrt itself.
 */

#ifndef __ARCH_IRQ_STAT
irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
EXPORT_SYMBOL(irq_stat);
#endif

static struct softirq_action softirq_vec[32] __cacheline_aligned_in_smp;

static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);

/*
 * we cannot loop indefinitely here to avoid userspace starvation,
 * but we also don't want to introduce a worst case 1/HZ latency
 * to the pending events, so lets the scheduler to balance
 * the softirq load for us.
 */
static inline void wakeup_softirqd(void)
{
	/* Interrupts are disabled: no need to stop preemption */
	struct task_struct *tsk = __get_cpu_var(ksoftirqd);

	if (tsk && tsk->state != TASK_RUNNING)
		wake_up_process(tsk);
}

/*
 * This one is for softirq.c-internal use,
 * where hardirqs are disabled legitimately:
 */
#ifdef CONFIG_TRACE_IRQFLAGS
static void __local_bh_disable(unsigned long ip)
{
	unsigned long flags;

	WARN_ON_ONCE(in_irq());

	raw_local_irq_save(flags);
	add_preempt_count(SOFTIRQ_OFFSET);
	/*
	 * Were softirqs turned off above:
	 */
	if (softirq_count() == SOFTIRQ_OFFSET)
		trace_softirqs_off(ip);
	raw_local_irq_restore(flags);
}
#else /* !CONFIG_TRACE_IRQFLAGS */
static inline void __local_bh_disable(unsigned long ip)
{
	add_preempt_count(SOFTIRQ_OFFSET);
	barrier();
}
#endif /* CONFIG_TRACE_IRQFLAGS */

void local_bh_disable(void)
{
	__local_bh_disable((unsigned long)__builtin_return_address(0));
}

EXPORT_SYMBOL(local_bh_disable);

void __local_bh_enable(void)
{
	WARN_ON_ONCE(in_irq());

	/*
	 * softirqs should never be enabled by __local_bh_enable(),
	 * it always nests inside local_bh_enable() sections:
	 */
	WARN_ON_ONCE(softirq_count() == SOFTIRQ_OFFSET);

	sub_preempt_count(SOFTIRQ_OFFSET);
}
EXPORT_SYMBOL_GPL(__local_bh_enable);

/*
 * Special-case - softirqs can safely be enabled in
 * cond_resched_softirq(), or by __do_softirq(),
 * without processing still-pending softirqs:
 */
void _local_bh_enable(void)
{
	WARN_ON_ONCE(in_irq());
	WARN_ON_ONCE(!irqs_disabled());

	if (softirq_count() == SOFTIRQ_OFFSET)
		trace_softirqs_on((unsigned long)__builtin_return_address(0));
	sub_preempt_count(SOFTIRQ_OFFSET);
}

EXPORT_SYMBOL(_local_bh_enable);

void local_bh_enable(void)
{
#ifdef CONFIG_TRACE_IRQFLAGS
	unsigned long flags;

	WARN_ON_ONCE(in_irq());
#endif
	WARN_ON_ONCE(irqs_disabled());

#ifdef CONFIG_TRACE_IRQFLAGS
	local_irq_save(flags);
#endif
	/*
	 * Are softirqs going to be turned on now:
	 */
	if (softirq_count() == SOFTIRQ_OFFSET)
		trace_softirqs_on((unsigned long)__builtin_return_address(0));
	/*
	 * Keep preemption disabled until we are done with
	 * softirq processing:
 	 */
 	sub_preempt_count(SOFTIRQ_OFFSET - 1);

	if (unlikely(!in_interrupt() && local_softirq_pending()))
		do_softirq();

	dec_preempt_count();
#ifdef CONFIG_TRACE_IRQFLAGS
	local_irq_restore(flags);
#endif
	preempt_check_resched();
}
EXPORT_SYMBOL(local_bh_enable);

void local_bh_enable_ip(unsigned long ip)
{
#ifdef CONFIG_TRACE_IRQFLAGS
	unsigned long flags;

	WARN_ON_ONCE(in_irq());

	local_irq_save(flags);
#endif
	/*
	 * Are softirqs going to be turned on now:
	 */
	if (softirq_count() == SOFTIRQ_OFFSET)
		trace_softirqs_on(ip);
	/*
	 * Keep preemption disabled until we are done with
	 * softirq processing:
 	 */
 	sub_preempt_count(SOFTIRQ_OFFSET - 1);

	if (unlikely(!in_interrupt() && local_softirq_pending()))
		do_softirq();

	dec_preempt_count();
#ifdef CONFIG_TRACE_IRQFLAGS
	local_irq_restore(flags);
#endif
	preempt_check_resched();
}
EXPORT_SYMBOL(local_bh_enable_ip);

/*
 * We restart softirq processing MAX_SOFTIRQ_RESTART times,
 * and we fall back to softirqd after that.
 *
 * This number has been established via experimentation.
 * The two things to balance is latency against fairness -
 * we want to handle softirqs as soon as possible, but they
 * should not be able to lock up the box.
 */
#define MAX_SOFTIRQ_RESTART 10

asmlinkage void __do_softirq(void)
{
	struct softirq_action *h;
	__u32 pending;
	int max_restart = MAX_SOFTIRQ_RESTART;
	int cpu;

	pending = local_softirq_pending();
	account_system_vtime(current);

	__local_bh_disable((unsigned long)__builtin_return_address(0));
	trace_softirq_enter();

	cpu = smp_processor_id();
restart:
	/* Reset the pending bitmask before enabling irqs */
	set_softirq_pending(0);

	local_irq_enable();

	h = softirq_vec;

	do {
		if (pending & 1) {
			h->action(h);
			rcu_bh_qsctr_inc(cpu);
		}
		h++;
		pending >>= 1;
	} while (pending);

	local_irq_disable();

	pending = local_softirq_pending();
	if (pending && --max_restart)
		goto restart;

	if (pending)
		wakeup_softirqd();

	trace_softirq_exit();

	account_system_vtime(current);
	_local_bh_enable();
}

#ifndef __ARCH_HAS_DO_SOFTIRQ

asmlinkage void do_softirq(void)
{
	__u32 pending;
	unsigned long flags;

	if (in_interrupt())
		return;

	local_irq_save(flags);

	pending = local_softirq_pending();

	if (pending)
		__do_softirq();

	local_irq_restore(flags);
}

EXPORT_SYMBOL(do_softirq);

#endif

/*
 * Enter an interrupt context.
 */
void irq_enter(void)
{
	__irq_enter();
#ifdef CONFIG_NO_HZ
	if (idle_cpu(smp_processor_id()))
		tick_nohz_update_jiffies();
#endif
}

#ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
# define invoke_softirq()	__do_softirq()
#else
# define invoke_softirq()	do_softirq()
#endif

/*
 * Exit an interrupt context. Process softirqs if needed and possible:
 */
void irq_exit(void)
{
	account_system_vtime(current);
	trace_hardirq_exit();
	sub_preempt_count(IRQ_EXIT_OFFSET);
	if (!in_interrupt() && local_softirq_pending())
		invoke_softirq();

#ifdef CONFIG_NO_HZ
	/* Make sure that timer wheel updates are propagated */
	if (!in_interrupt() && idle_cpu(smp_processor_id()) && !need_resched())
		tick_nohz_stop_sched_tick();
#endif
	preempt_enable_no_resched();
}

/*
 * This function must run with irqs disabled!
 */
inline fastcall void raise_softirq_irqoff(unsigned int nr)
{
	__raise_softirq_irqoff(nr);

	/*
	 * If we're in an interrupt or softirq, we're done
	 * (this also catches softirq-disabled code). We will
	 * actually run the softirq once we return from
	 * the irq or softirq.
	 *
	 * Otherwise we wake up ksoftirqd to make sure we
	 * schedule the softirq soon.
	 */
	if (!in_interrupt())
		wakeup_softirqd();
}

EXPORT_SYMBOL(raise_softirq_irqoff);

void fastcall raise_softirq(unsigned int nr)
{
	unsigned long flags;

	local_irq_save(flags);
	raise_softirq_irqoff(nr);
	local_irq_restore(flags);
}

void open_softirq(int nr, void (*action)(struct softirq_action*), void *data)
{
	softirq_vec[nr].data = data;
	softirq_vec[nr].action = action;
}

/* Tasklets */
struct tasklet_head
{
	struct tasklet_struct *list;
};

/* Some compilers disobey section attribute on statics when not
   initialized -- RR */
static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec) = { NULL };
static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec) = { NULL };

void fastcall __tasklet_schedule(struct tasklet_struct *t)
{
	unsigned long flags;

	local_irq_save(flags);
	t->next = __get_cpu_var(tasklet_vec).list;
	__get_cpu_var(tasklet_vec).list = t;
	raise_softirq_irqoff(TASKLET_SOFTIRQ);
	local_irq_restore(flags);
}

EXPORT_SYMBOL(__tasklet_schedule);

void fastcall __tasklet_hi_schedule(struct tasklet_struct *t)
{
	unsigned long flags;

	local_irq_save(flags);
	t->next = __get_cpu_var(tasklet_hi_vec).list;
	__get_cpu_var(tasklet_hi_vec).list = t;
	raise_softirq_irqoff(HI_SOFTIRQ);
	local_irq_restore(flags);
}

EXPORT_SYMBOL(__tasklet_hi_schedule);

static void tasklet_action(struct softirq_action *a)
{
	struct tasklet_struct *list;

	local_irq_disable();
	list = __get_cpu_var(tasklet_vec).list;
	__get_cpu_var(tasklet_vec).list = NULL;
	local_irq_enable();

	while (list) {
		struct tasklet_struct *t = list;

		list = list->next;

		if (tasklet_trylock(t)) {
			if (!atomic_read(&t->count)) {
				if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
					BUG();
				t->func(t->data);
				tasklet_unlock(t);
				continue;
			}
			tasklet_unlock(t);
		}

		local_irq_disable();
		t->next = __get_cpu_var(tasklet_vec).list;
		__get_cpu_var(tasklet_vec).list = t;
		__raise_softirq_irqoff(TASKLET_SOFTIRQ);
		local_irq_enable();
	}
}

static void tasklet_hi_action(struct softirq_action *a)
{
	struct tasklet_struct *list;

	local_irq_disable();
	list = __get_cpu_var(tasklet_hi_vec).list;
	__get_cpu_var(tasklet_hi_vec).list = NULL;
	local_irq_enable();

	while (list) {
		struct tasklet_struct *t = list;

		list = list->next;

		if (tasklet_trylock(t)) {
			if (!atomic_read(&t->count)) {
				if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
					BUG();
				t->func(t->data);
				tasklet_unlock(t);
				continue;
			}
			tasklet_unlock(t);
		}

		local_irq_disable();
		t->next = __get_cpu_var(tasklet_hi_vec).list;
		__get_cpu_var(tasklet_hi_vec).list = t;
		__raise_softirq_irqoff(HI_SOFTIRQ);
		local_irq_enable();
	}
}


void tasklet_init(struct tasklet_struct *t,
		  void (*func)(unsigned long), unsigned long data)
{
	t->next = NULL;
	t->state = 0;
	atomic_set(&t->count, 0);
	t->func = func;
	t->data = data;
}

EXPORT_SYMBOL(tasklet_init);

void tasklet_kill(struct tasklet_struct *t)
{
	if (in_interrupt())
		printk("Attempt to kill tasklet from interrupt\n");

	while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
		do
			yield();
		while (test_bit(TASKLET_STATE_SCHED, &t->state));
	}
	tasklet_unlock_wait(t);
	clear_bit(TASKLET_STATE_SCHED, &t->state);
}

EXPORT_SYMBOL(tasklet_kill);

void __init softirq_init(void)
{
	open_softirq(TASKLET_SOFTIRQ, tasklet_action, NULL);
	open_softirq(HI_SOFTIRQ, tasklet_hi_action, NULL);
}

static int ksoftirqd(void * __bind_cpu)
{
	current->flags |= PF_NOFREEZE;

	set_current_state(TASK_INTERRUPTIBLE);

	while (!kthread_should_stop()) {
		preempt_disable();
		if (!local_softirq_pending()) {
			preempt_enable_no_resched();
			schedule();
			preempt_disable();
		}

		__set_current_state(TASK_RUNNING);

		while (local_softirq_pending()) {
			/* Preempt disable stops cpu going offline.
			   If already offline, we'll be on wrong CPU:
			   don't process */
			if (cpu_is_offline((long)__bind_cpu))
				goto wait_to_die;
			do_softirq();
			preempt_enable_no_resched();
			cond_resched();
			preempt_disable();
		}
		preempt_enable();
		set_current_state(TASK_INTERRUPTIBLE);
	}
	__set_current_state(TASK_RUNNING);
	return 0;

wait_to_die:
	preempt_enable();
	/* Wait for kthread_stop */
	set_current_state(TASK_INTERRUPTIBLE);
	while (!kthread_should_stop()) {
		schedule();
		set_current_state(TASK_INTERRUPTIBLE);
	}
	__set_current_state(TASK_RUNNING);
	return 0;
}

#ifdef CONFIG_HOTPLUG_CPU
/*
 * tasklet_kill_immediate is called to remove a tasklet which can already be
 * scheduled for execution on @cpu.
 *
 * Unlike tasklet_kill, this function removes the tasklet
 * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
 *
 * When this function is called, @cpu must be in the CPU_DEAD state.
 */
void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
{
	struct tasklet_struct **i;

	BUG_ON(cpu_online(cpu));
	BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));

	if (!test_bit(TASKLET_STATE_SCHED, &t->state))
		return;

	/* CPU is dead, so no lock needed. */
	for (i = &per_cpu(tasklet_vec, cpu).list; *i; i = &(*i)->next) {
		if (*i == t) {
			*i = t->next;
			return;
		}
	}
	BUG();
}

static void takeover_tasklets(unsigned int cpu)
{
	struct tasklet_struct **i;

	/* CPU is dead, so no lock needed. */
	local_irq_disable();

	/* Find end, append list for that CPU. */
	for (i = &__get_cpu_var(tasklet_vec).list; *i; i = &(*i)->next);
	*i = per_cpu(tasklet_vec, cpu).list;
	per_cpu(tasklet_vec, cpu).list = NULL;
	raise_softirq_irqoff(TASKLET_SOFTIRQ);

	for (i = &__get_cpu_var(tasklet_hi_vec).list; *i; i = &(*i)->next);
	*i = per_cpu(tasklet_hi_vec, cpu).list;
	per_cpu(tasklet_hi_vec, cpu).list = NULL;
	raise_softirq_irqoff(HI_SOFTIRQ);

	local_irq_enable();
}
#endif /* CONFIG_HOTPLUG_CPU */

static int __cpuinit cpu_callback(struct notifier_block *nfb,
				  unsigned long action,
				  void *hcpu)
{
	int hotcpu = (unsigned long)hcpu;
	struct task_struct *p;

	switch (action) {
	case CPU_UP_PREPARE:
	case CPU_UP_PREPARE_FROZEN:
		p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
		if (IS_ERR(p)) {
			printk("ksoftirqd for %i failed\n", hotcpu);
			return NOTIFY_BAD;
		}
		kthread_bind(p, hotcpu);
  		per_cpu(ksoftirqd, hotcpu) = p;
 		break;
	case CPU_ONLINE:
	case CPU_ONLINE_FROZEN:
		wake_up_process(per_cpu(ksoftirqd, hotcpu));
		break;
#ifdef CONFIG_HOTPLUG_CPU
	case CPU_UP_CANCELED:
	case CPU_UP_CANCELED_FROZEN:
		if (!per_cpu(ksoftirqd, hotcpu))
			break;
		/* Unbind so it can run.  Fall thru. */
		kthread_bind(per_cpu(ksoftirqd, hotcpu),
			     any_online_cpu(cpu_online_map));
	case CPU_DEAD:
	case CPU_DEAD_FROZEN: {
		struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };

		p = per_cpu(ksoftirqd, hotcpu);
		per_cpu(ksoftirqd, hotcpu) = NULL;
		sched_setscheduler(p, SCHED_FIFO, &param);
		kthread_stop(p);
		takeover_tasklets(hotcpu);
		break;
	}
#endif /* CONFIG_HOTPLUG_CPU */
 	}
	return NOTIFY_OK;
}

static struct notifier_block __cpuinitdata cpu_nfb = {
	.notifier_call = cpu_callback
};

__init int spawn_ksoftirqd(void)
{
	void *cpu = (void *)(long)smp_processor_id();
	int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);

	BUG_ON(err == NOTIFY_BAD);
	cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
	register_cpu_notifier(&cpu_nfb);
	return 0;
}

#ifdef CONFIG_SMP
/*
 * Call a function on all processors
 */
int on_each_cpu(void (*func) (void *info), void *info, int retry, int wait)
{
	int ret = 0;

	preempt_disable();
	ret = smp_call_function(func, info, retry, wait);
	local_irq_disable();
	func(info);
	local_irq_enable();
	preempt_enable();
	return ret;
}
EXPORT_SYMBOL(on_each_cpu);
#endif
Commit	Line	Data
1da177e4 LT	1	/*
	2	* linux/kernel/softirq.c
	3	*
	4	* Copyright (C) 1992 Linus Torvalds
	5	*
	6	* Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
	7	*/
	8
	9	#include <linux/module.h>
	10	#include <linux/kernel_stat.h>
	11	#include <linux/interrupt.h>
	12	#include <linux/init.h>
	13	#include <linux/mm.h>
	14	#include <linux/notifier.h>
	15	#include <linux/percpu.h>
	16	#include <linux/cpu.h>
	17	#include <linux/kthread.h>
	18	#include <linux/rcupdate.h>
78eef01b	19	#include <linux/smp.h>
79bf2bb3	20	#include <linux/tick.h>
1da177e4 LT	21
	22	#include <asm/irq.h>
	23	/*
	24	- No shared variables, all the data are CPU local.
	25	- If a softirq needs serialization, let it serialize itself
	26	by its own spinlocks.
	27	- Even if softirq is serialized, only local cpu is marked for
	28	execution. Hence, we get something sort of weak cpu binding.
	29	Though it is still not clear, will it result in better locality
	30	or will not.
	31
	32	Examples:
	33	- NET RX softirq. It is multithreaded and does not require
	34	any global serialization.
	35	- NET TX softirq. It kicks software netdevice queues, hence
	36	it is logically serialized per device, but this serialization
	37	is invisible to common code.
	38	- Tasklets: serialized wrt itself.
	39	*/
	40
	41	#ifndef __ARCH_IRQ_STAT
	42	irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
	43	EXPORT_SYMBOL(irq_stat);
	44	#endif
	45
	46	static struct softirq_action softirq_vec[32] __cacheline_aligned_in_smp;
	47
	48	static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
	49
	50	/*
	51	* we cannot loop indefinitely here to avoid userspace starvation,
	52	* but we also don't want to introduce a worst case 1/HZ latency
	53	* to the pending events, so lets the scheduler to balance
	54	* the softirq load for us.
	55	*/
	56	static inline void wakeup_softirqd(void)
	57	{
	58	/* Interrupts are disabled: no need to stop preemption */
	59	struct task_struct *tsk = __get_cpu_var(ksoftirqd);
	60
	61	if (tsk && tsk->state != TASK_RUNNING)
	62	wake_up_process(tsk);
	63	}
	64
de30a2b3 IM	65	/*
	66	* This one is for softirq.c-internal use,
	67	* where hardirqs are disabled legitimately:
	68	*/
3c829c36	69	#ifdef CONFIG_TRACE_IRQFLAGS
de30a2b3 IM	70	static void __local_bh_disable(unsigned long ip)
	71	{
	72	unsigned long flags;
	73
	74	WARN_ON_ONCE(in_irq());
	75
	76	raw_local_irq_save(flags);
	77	add_preempt_count(SOFTIRQ_OFFSET);
	78	/*
	79	* Were softirqs turned off above:
	80	*/
	81	if (softirq_count() == SOFTIRQ_OFFSET)
	82	trace_softirqs_off(ip);
	83	raw_local_irq_restore(flags);
	84	}
3c829c36 TC	85	#else /* !CONFIG_TRACE_IRQFLAGS */
	86	static inline void __local_bh_disable(unsigned long ip)
	87	{
	88	add_preempt_count(SOFTIRQ_OFFSET);
	89	barrier();
	90	}
	91	#endif /* CONFIG_TRACE_IRQFLAGS */
de30a2b3 IM	92
	93	void local_bh_disable(void)
	94	{
	95	__local_bh_disable((unsigned long)__builtin_return_address(0));
	96	}
	97
	98	EXPORT_SYMBOL(local_bh_disable);
	99
	100	void __local_bh_enable(void)
	101	{
	102	WARN_ON_ONCE(in_irq());
	103
	104	/*
	105	* softirqs should never be enabled by __local_bh_enable(),
	106	* it always nests inside local_bh_enable() sections:
	107	*/
	108	WARN_ON_ONCE(softirq_count() == SOFTIRQ_OFFSET);
	109
	110	sub_preempt_count(SOFTIRQ_OFFSET);
	111	}
	112	EXPORT_SYMBOL_GPL(__local_bh_enable);
	113
	114	/*
	115	* Special-case - softirqs can safely be enabled in
	116	* cond_resched_softirq(), or by __do_softirq(),
	117	* without processing still-pending softirqs:
	118	*/
	119	void _local_bh_enable(void)
	120	{
	121	WARN_ON_ONCE(in_irq());
	122	WARN_ON_ONCE(!irqs_disabled());
	123
	124	if (softirq_count() == SOFTIRQ_OFFSET)
	125	trace_softirqs_on((unsigned long)__builtin_return_address(0));
	126	sub_preempt_count(SOFTIRQ_OFFSET);
	127	}
	128
	129	EXPORT_SYMBOL(_local_bh_enable);
	130
	131	void local_bh_enable(void)
	132	{
3c829c36	133	#ifdef CONFIG_TRACE_IRQFLAGS
de30a2b3 IM	134	unsigned long flags;
	135
	136	WARN_ON_ONCE(in_irq());
3c829c36	137	#endif
de30a2b3 IM	138	WARN_ON_ONCE(irqs_disabled());
de30a2b3 IM	139
3c829c36	140	#ifdef CONFIG_TRACE_IRQFLAGS
de30a2b3	141	local_irq_save(flags);
3c829c36	142	#endif
de30a2b3 IM	143	/*
	144	* Are softirqs going to be turned on now:
	145	*/
	146	if (softirq_count() == SOFTIRQ_OFFSET)
	147	trace_softirqs_on((unsigned long)__builtin_return_address(0));
	148	/*
	149	* Keep preemption disabled until we are done with
	150	* softirq processing:
	151	*/
	152	sub_preempt_count(SOFTIRQ_OFFSET - 1);
	153
	154	if (unlikely(!in_interrupt() && local_softirq_pending()))
	155	do_softirq();
	156
	157	dec_preempt_count();
3c829c36	158	#ifdef CONFIG_TRACE_IRQFLAGS
de30a2b3	159	local_irq_restore(flags);
3c829c36	160	#endif
de30a2b3 IM	161	preempt_check_resched();
	162	}
	163	EXPORT_SYMBOL(local_bh_enable);
	164
	165	void local_bh_enable_ip(unsigned long ip)
	166	{
3c829c36	167	#ifdef CONFIG_TRACE_IRQFLAGS
de30a2b3 IM	168	unsigned long flags;
	169
	170	WARN_ON_ONCE(in_irq());
	171
	172	local_irq_save(flags);
3c829c36	173	#endif
de30a2b3 IM	174	/*
	175	* Are softirqs going to be turned on now:
	176	*/
	177	if (softirq_count() == SOFTIRQ_OFFSET)
	178	trace_softirqs_on(ip);
	179	/*
	180	* Keep preemption disabled until we are done with
	181	* softirq processing:
	182	*/
	183	sub_preempt_count(SOFTIRQ_OFFSET - 1);
	184
	185	if (unlikely(!in_interrupt() && local_softirq_pending()))
	186	do_softirq();
	187
	188	dec_preempt_count();
3c829c36	189	#ifdef CONFIG_TRACE_IRQFLAGS
de30a2b3	190	local_irq_restore(flags);
3c829c36	191	#endif
de30a2b3 IM	192	preempt_check_resched();
	193	}
	194	EXPORT_SYMBOL(local_bh_enable_ip);
	195
1da177e4 LT	196	/*
	197	* We restart softirq processing MAX_SOFTIRQ_RESTART times,
	198	* and we fall back to softirqd after that.
	199	*
	200	* This number has been established via experimentation.
	201	* The two things to balance is latency against fairness -
	202	* we want to handle softirqs as soon as possible, but they
	203	* should not be able to lock up the box.
	204	*/
	205	#define MAX_SOFTIRQ_RESTART 10
	206
	207	asmlinkage void __do_softirq(void)
	208	{
	209	struct softirq_action *h;
	210	__u32 pending;
	211	int max_restart = MAX_SOFTIRQ_RESTART;
	212	int cpu;
	213
	214	pending = local_softirq_pending();
829035fd PM	215	account_system_vtime(current);
829035fd PM	216
de30a2b3 IM	217	__local_bh_disable((unsigned long)__builtin_return_address(0));
de30a2b3 IM	218	trace_softirq_enter();
1da177e4	219
1da177e4 LT	220	cpu = smp_processor_id();
	221	restart:
	222	/* Reset the pending bitmask before enabling irqs */
3f74478b	223	set_softirq_pending(0);
1da177e4	224
c70f5d66	225	local_irq_enable();
1da177e4 LT	226
	227	h = softirq_vec;
	228
	229	do {
	230	if (pending & 1) {
	231	h->action(h);
	232	rcu_bh_qsctr_inc(cpu);
	233	}
	234	h++;
	235	pending >>= 1;
	236	} while (pending);
	237
c70f5d66	238	local_irq_disable();
1da177e4 LT	239
	240	pending = local_softirq_pending();
	241	if (pending && --max_restart)
	242	goto restart;
	243
	244	if (pending)
	245	wakeup_softirqd();
	246
de30a2b3	247	trace_softirq_exit();
829035fd PM	248
829035fd PM	249	account_system_vtime(current);
de30a2b3	250	_local_bh_enable();
1da177e4 LT	251	}
	252
	253	#ifndef __ARCH_HAS_DO_SOFTIRQ
	254
	255	asmlinkage void do_softirq(void)
	256	{
	257	__u32 pending;
	258	unsigned long flags;
	259
	260	if (in_interrupt())
	261	return;
	262
	263	local_irq_save(flags);
	264
	265	pending = local_softirq_pending();
	266
	267	if (pending)
	268	__do_softirq();
	269
	270	local_irq_restore(flags);
	271	}
	272
	273	EXPORT_SYMBOL(do_softirq);
	274
	275	#endif
	276
dde4b2b5 IM	277	/*
	278	* Enter an interrupt context.
	279	*/
	280	void irq_enter(void)
	281	{
79bf2bb3 TG	282	__irq_enter();
	283	#ifdef CONFIG_NO_HZ
	284	if (idle_cpu(smp_processor_id()))
	285	tick_nohz_update_jiffies();
	286	#endif
dde4b2b5 IM	287	}
dde4b2b5 IM	288
1da177e4 LT	289	#ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
	290	# define invoke_softirq() __do_softirq()
	291	#else
	292	# define invoke_softirq() do_softirq()
	293	#endif
	294
	295	/*
	296	* Exit an interrupt context. Process softirqs if needed and possible:
	297	*/
	298	void irq_exit(void)
	299	{
	300	account_system_vtime(current);
de30a2b3	301	trace_hardirq_exit();
1da177e4 LT	302	sub_preempt_count(IRQ_EXIT_OFFSET);
	303	if (!in_interrupt() && local_softirq_pending())
	304	invoke_softirq();
79bf2bb3 TG	305
	306	#ifdef CONFIG_NO_HZ
	307	/* Make sure that timer wheel updates are propagated */
	308	if (!in_interrupt() && idle_cpu(smp_processor_id()) && !need_resched())
	309	tick_nohz_stop_sched_tick();
	310	#endif
1da177e4 LT	311	preempt_enable_no_resched();
	312	}
	313
	314	/*
	315	* This function must run with irqs disabled!
	316	*/
	317	inline fastcall void raise_softirq_irqoff(unsigned int nr)
	318	{
	319	__raise_softirq_irqoff(nr);
	320
	321	/*
	322	* If we're in an interrupt or softirq, we're done
	323	* (this also catches softirq-disabled code). We will
	324	* actually run the softirq once we return from
	325	* the irq or softirq.
	326	*
	327	* Otherwise we wake up ksoftirqd to make sure we
	328	* schedule the softirq soon.
	329	*/
	330	if (!in_interrupt())
	331	wakeup_softirqd();
	332	}
	333
	334	EXPORT_SYMBOL(raise_softirq_irqoff);
	335
	336	void fastcall raise_softirq(unsigned int nr)
	337	{
	338	unsigned long flags;
	339
	340	local_irq_save(flags);
	341	raise_softirq_irqoff(nr);
	342	local_irq_restore(flags);
	343	}
	344
	345	void open_softirq(int nr, void (action)(struct softirq_action), void *data)
	346	{
	347	softirq_vec[nr].data = data;
	348	softirq_vec[nr].action = action;
	349	}
	350
1da177e4 LT	351	/* Tasklets */
	352	struct tasklet_head
	353	{
	354	struct tasklet_struct *list;
	355	};
	356
	357	/* Some compilers disobey section attribute on statics when not
	358	initialized -- RR */
	359	static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec) = { NULL };
	360	static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec) = { NULL };
	361
	362	void fastcall __tasklet_schedule(struct tasklet_struct *t)
	363	{
	364	unsigned long flags;
	365
	366	local_irq_save(flags);
	367	t->next = __get_cpu_var(tasklet_vec).list;
	368	__get_cpu_var(tasklet_vec).list = t;
	369	raise_softirq_irqoff(TASKLET_SOFTIRQ);
	370	local_irq_restore(flags);
	371	}
	372
	373	EXPORT_SYMBOL(__tasklet_schedule);
	374
	375	void fastcall __tasklet_hi_schedule(struct tasklet_struct *t)
	376	{
	377	unsigned long flags;
	378
	379	local_irq_save(flags);
	380	t->next = __get_cpu_var(tasklet_hi_vec).list;
	381	__get_cpu_var(tasklet_hi_vec).list = t;
	382	raise_softirq_irqoff(HI_SOFTIRQ);
	383	local_irq_restore(flags);
	384	}
	385
	386	EXPORT_SYMBOL(__tasklet_hi_schedule);
	387
	388	static void tasklet_action(struct softirq_action *a)
	389	{
	390	struct tasklet_struct *list;
	391
	392	local_irq_disable();
	393	list = __get_cpu_var(tasklet_vec).list;
	394	__get_cpu_var(tasklet_vec).list = NULL;
	395	local_irq_enable();
	396
	397	while (list) {
	398	struct tasklet_struct *t = list;
	399
	400	list = list->next;
	401
	402	if (tasklet_trylock(t)) {
	403	if (!atomic_read(&t->count)) {
	404	if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
	405	BUG();
	406	t->func(t->data);
	407	tasklet_unlock(t);
	408	continue;
	409	}
	410	tasklet_unlock(t);
	411	}
	412
	413	local_irq_disable();
	414	t->next = __get_cpu_var(tasklet_vec).list;
415	__get_cpu_var(tasklet_vec).list = t;
416	__raise_softirq_irqoff(TASKLET_SOFTIRQ);
417	local_irq_enable();
418	}
419	}
420
421	static void tasklet_hi_action(struct softirq_action *a)
422	{
423	struct tasklet_struct *list;
424
425	local_irq_disable();
426	list = __get_cpu_var(tasklet_hi_vec).list;
427	__get_cpu_var(tasklet_hi_vec).list = NULL;
428	local_irq_enable();
429
430	while (list) {
431	struct tasklet_struct *t = list;
432
433	list = list->next;
434
435	if (tasklet_trylock(t)) {
436	if (!atomic_read(&t->count)) {
437	if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
438	BUG();
439	t->func(t->data);
440	tasklet_unlock(t);
441	continue;
442	}
443	tasklet_unlock(t);
444	}
445
446	local_irq_disable();
447	t->next = __get_cpu_var(tasklet_hi_vec).list;
448	__get_cpu_var(tasklet_hi_vec).list = t;
449	__raise_softirq_irqoff(HI_SOFTIRQ);
450	local_irq_enable();
451	}
452	}
453
454
455	void tasklet_init(struct tasklet_struct *t,
456	void (*func)(unsigned long), unsigned long data)
457	{
458	t->next = NULL;
459	t->state = 0;
460	atomic_set(&t->count, 0);
461	t->func = func;
462	t->data = data;
463	}
464
465	EXPORT_SYMBOL(tasklet_init);
466
467	void tasklet_kill(struct tasklet_struct *t)
468	{
469	if (in_interrupt())
470	printk("Attempt to kill tasklet from interrupt\n");
471
472	while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
473	do
474	yield();
475	while (test_bit(TASKLET_STATE_SCHED, &t->state));
476	}
477	tasklet_unlock_wait(t);
478	clear_bit(TASKLET_STATE_SCHED, &t->state);
479	}
480
481	EXPORT_SYMBOL(tasklet_kill);
482
483	void __init softirq_init(void)
484	{
485	open_softirq(TASKLET_SOFTIRQ, tasklet_action, NULL);
486	open_softirq(HI_SOFTIRQ, tasklet_hi_action, NULL);
487	}
488
489	static int ksoftirqd(void * __bind_cpu)
490	{
1da177e4 LT	491	current->flags \|= PF_NOFREEZE;
	492
	493	set_current_state(TASK_INTERRUPTIBLE);
	494
	495	while (!kthread_should_stop()) {
	496	preempt_disable();
	497	if (!local_softirq_pending()) {
	498	preempt_enable_no_resched();
	499	schedule();
	500	preempt_disable();
	501	}
	502
	503	__set_current_state(TASK_RUNNING);
	504
	505	while (local_softirq_pending()) {
	506	/* Preempt disable stops cpu going offline.
	507	If already offline, we'll be on wrong CPU:
	508	don't process */
	509	if (cpu_is_offline((long)__bind_cpu))
	510	goto wait_to_die;
	511	do_softirq();
	512	preempt_enable_no_resched();
	513	cond_resched();
	514	preempt_disable();
	515	}
	516	preempt_enable();
	517	set_current_state(TASK_INTERRUPTIBLE);
	518	}
	519	__set_current_state(TASK_RUNNING);
	520	return 0;
	521
	522	wait_to_die:
	523	preempt_enable();
	524	/* Wait for kthread_stop */
	525	set_current_state(TASK_INTERRUPTIBLE);
	526	while (!kthread_should_stop()) {
	527	schedule();
	528	set_current_state(TASK_INTERRUPTIBLE);
	529	}
	530	__set_current_state(TASK_RUNNING);
	531	return 0;
	532	}
	533
	534	#ifdef CONFIG_HOTPLUG_CPU
	535	/*
	536	* tasklet_kill_immediate is called to remove a tasklet which can already be
	537	* scheduled for execution on @cpu.
	538	*
	539	* Unlike tasklet_kill, this function removes the tasklet
	540	* _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
	541	*
	542	* When this function is called, @cpu must be in the CPU_DEAD state.
	543	*/
	544	void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
	545	{
	546	struct tasklet_struct **i;
	547
	548	BUG_ON(cpu_online(cpu));
	549	BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
	550
	551	if (!test_bit(TASKLET_STATE_SCHED, &t->state))
	552	return;
	553
	554	/* CPU is dead, so no lock needed. */
555	for (i = &per_cpu(tasklet_vec, cpu).list; i; i = &(i)->next) {
556	if (*i == t) {
557	*i = t->next;
558	return;
559	}
560	}
561	BUG();
562	}
563
564	static void takeover_tasklets(unsigned int cpu)
565	{
566	struct tasklet_struct **i;
567
568	/* CPU is dead, so no lock needed. */
569	local_irq_disable();
570
571	/* Find end, append list for that CPU. */
572	for (i = &__get_cpu_var(tasklet_vec).list; i; i = &(i)->next);
573	*i = per_cpu(tasklet_vec, cpu).list;
574	per_cpu(tasklet_vec, cpu).list = NULL;
575	raise_softirq_irqoff(TASKLET_SOFTIRQ);
576
577	for (i = &__get_cpu_var(tasklet_hi_vec).list; i; i = &(i)->next);
578	*i = per_cpu(tasklet_hi_vec, cpu).list;
579	per_cpu(tasklet_hi_vec, cpu).list = NULL;
580	raise_softirq_irqoff(HI_SOFTIRQ);
581
582	local_irq_enable();
583	}
584	#endif /* CONFIG_HOTPLUG_CPU */
585
8c78f307	586	static int __cpuinit cpu_callback(struct notifier_block *nfb,
1da177e4 LT	587	unsigned long action,
	588	void *hcpu)
	589	{
	590	int hotcpu = (unsigned long)hcpu;
	591	struct task_struct *p;
	592
	593	switch (action) {
	594	case CPU_UP_PREPARE:
8bb78442	595	case CPU_UP_PREPARE_FROZEN:
1da177e4 LT	596	p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
	597	if (IS_ERR(p)) {
	598	printk("ksoftirqd for %i failed\n", hotcpu);
	599	return NOTIFY_BAD;
	600	}
	601	kthread_bind(p, hotcpu);
	602	per_cpu(ksoftirqd, hotcpu) = p;
	603	break;
	604	case CPU_ONLINE:
8bb78442	605	case CPU_ONLINE_FROZEN:
1da177e4 LT	606	wake_up_process(per_cpu(ksoftirqd, hotcpu));
	607	break;
	608	#ifdef CONFIG_HOTPLUG_CPU
	609	case CPU_UP_CANCELED:
8bb78442	610	case CPU_UP_CANCELED_FROZEN:
fc75cdfa HC	611	if (!per_cpu(ksoftirqd, hotcpu))
fc75cdfa HC	612	break;
1da177e4	613	/* Unbind so it can run. Fall thru. */
a4c4af7c HC	614	kthread_bind(per_cpu(ksoftirqd, hotcpu),
a4c4af7c HC	615	any_online_cpu(cpu_online_map));
1da177e4	616	case CPU_DEAD:
1c6b4aa9 ST	617	case CPU_DEAD_FROZEN: {
	618	struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
	619
1da177e4 LT	620	p = per_cpu(ksoftirqd, hotcpu);
1da177e4 LT	621	per_cpu(ksoftirqd, hotcpu) = NULL;
1c6b4aa9	622	sched_setscheduler(p, SCHED_FIFO, &param);
1da177e4 LT	623	kthread_stop(p);
	624	takeover_tasklets(hotcpu);
	625	break;
1c6b4aa9	626	}
1da177e4 LT	627	#endif /* CONFIG_HOTPLUG_CPU */
	628	}
	629	return NOTIFY_OK;
	630	}
	631
8c78f307	632	static struct notifier_block __cpuinitdata cpu_nfb = {
1da177e4 LT	633	.notifier_call = cpu_callback
	634	};
	635
	636	__init int spawn_ksoftirqd(void)
	637	{
	638	void cpu = (void )(long)smp_processor_id();
07dccf33 AM	639	int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
	640
	641	BUG_ON(err == NOTIFY_BAD);
1da177e4 LT	642	cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
	643	register_cpu_notifier(&cpu_nfb);
	644	return 0;
	645	}
78eef01b AM	646
	647	#ifdef CONFIG_SMP
	648	/*
	649	* Call a function on all processors
	650	*/
	651	int on_each_cpu(void (func) (void info), void *info, int retry, int wait)
	652	{
	653	int ret = 0;
	654
	655	preempt_disable();
	656	ret = smp_call_function(func, info, retry, wait);
	657	local_irq_disable();
	658	func(info);
	659	local_irq_enable();
	660	preempt_enable();
	661	return ret;
	662	}
	663	EXPORT_SYMBOL(on_each_cpu);
	664	#endif