[linux-2.6-block.git] / arch / alpha / kernel / smp.c

/*
 *	linux/arch/alpha/kernel/smp.c
 *
 *      2001-07-09 Phil Ezolt (Phillip.Ezolt@compaq.com)
 *            Renamed modified smp_call_function to smp_call_function_on_cpu()
 *            Created an function that conforms to the old calling convention
 *            of smp_call_function().
 *
 *            This is helpful for DCPI.
 *
 */

#include <linux/errno.h>
#include <linux/kernel.h>
#include <linux/kernel_stat.h>
#include <linux/module.h>
#include <linux/sched.h>
#include <linux/mm.h>
#include <linux/err.h>
#include <linux/threads.h>
#include <linux/smp.h>
#include <linux/interrupt.h>
#include <linux/init.h>
#include <linux/delay.h>
#include <linux/spinlock.h>
#include <linux/irq.h>
#include <linux/cache.h>
#include <linux/profile.h>
#include <linux/bitops.h>
#include <linux/cpu.h>

#include <asm/hwrpb.h>
#include <asm/ptrace.h>
#include <linux/atomic.h>

#include <asm/io.h>
#include <asm/irq.h>
#include <asm/pgtable.h>
#include <asm/pgalloc.h>
#include <asm/mmu_context.h>
#include <asm/tlbflush.h>

#include "proto.h"
#include "irq_impl.h"


#define DEBUG_SMP 0
#if DEBUG_SMP
#define DBGS(args)	printk args
#else
#define DBGS(args)
#endif

/* A collection of per-processor data.  */
struct cpuinfo_alpha cpu_data[NR_CPUS];
EXPORT_SYMBOL(cpu_data);

/* A collection of single bit ipi messages.  */
static struct {
	unsigned long bits ____cacheline_aligned;
} ipi_data[NR_CPUS] __cacheline_aligned;

enum ipi_message_type {
	IPI_RESCHEDULE,
	IPI_CALL_FUNC,
	IPI_CALL_FUNC_SINGLE,
	IPI_CPU_STOP,
};

/* Set to a secondary's cpuid when it comes online.  */
static int smp_secondary_alive __devinitdata = 0;

int smp_num_probed;		/* Internal processor count */
int smp_num_cpus = 1;		/* Number that came online.  */
EXPORT_SYMBOL(smp_num_cpus);

/*
 * Called by both boot and secondaries to move global data into
 *  per-processor storage.
 */
static inline void __init
smp_store_cpu_info(int cpuid)
{
	cpu_data[cpuid].loops_per_jiffy = loops_per_jiffy;
	cpu_data[cpuid].last_asn = ASN_FIRST_VERSION;
	cpu_data[cpuid].need_new_asn = 0;
	cpu_data[cpuid].asn_lock = 0;
}

/*
 * Ideally sets up per-cpu profiling hooks.  Doesn't do much now...
 */
static inline void __init
smp_setup_percpu_timer(int cpuid)
{
	cpu_data[cpuid].prof_counter = 1;
	cpu_data[cpuid].prof_multiplier = 1;
}

static void __init
wait_boot_cpu_to_stop(int cpuid)
{
	unsigned long stop = jiffies + 10*HZ;

	while (time_before(jiffies, stop)) {
	        if (!smp_secondary_alive)
			return;
		barrier();
	}

	printk("wait_boot_cpu_to_stop: FAILED on CPU %d, hanging now\n", cpuid);
	for (;;)
		barrier();
}

/*
 * Where secondaries begin a life of C.
 */
void __cpuinit
smp_callin(void)
{
	int cpuid = hard_smp_processor_id();

	if (cpu_online(cpuid)) {
		printk("??, cpu 0x%x already present??\n", cpuid);
		BUG();
	}
	set_cpu_online(cpuid, true);

	/* Turn on machine checks.  */
	wrmces(7);

	/* Set trap vectors.  */
	trap_init();

	/* Set interrupt vector.  */
	wrent(entInt, 0);

	/* Get our local ticker going. */
	smp_setup_percpu_timer(cpuid);

	/* Call platform-specific callin, if specified */
	if (alpha_mv.smp_callin) alpha_mv.smp_callin();

	/* All kernel threads share the same mm context.  */
	atomic_inc(&init_mm.mm_count);
	current->active_mm = &init_mm;

	/* inform the notifiers about the new cpu */
	notify_cpu_starting(cpuid);

	/* Must have completely accurate bogos.  */
	local_irq_enable();

	/* Wait boot CPU to stop with irq enabled before running
	   calibrate_delay. */
	wait_boot_cpu_to_stop(cpuid);
	mb();
	calibrate_delay();

	smp_store_cpu_info(cpuid);
	/* Allow master to continue only after we written loops_per_jiffy.  */
	wmb();
	smp_secondary_alive = 1;

	DBGS(("smp_callin: commencing CPU %d current %p active_mm %p\n",
	      cpuid, current, current->active_mm));

	preempt_disable();
	/* Do nothing.  */
	cpu_idle();
}

/* Wait until hwrpb->txrdy is clear for cpu.  Return -1 on timeout.  */
static int __devinit
wait_for_txrdy (unsigned long cpumask)
{
	unsigned long timeout;

	if (!(hwrpb->txrdy & cpumask))
		return 0;

	timeout = jiffies + 10*HZ;
	while (time_before(jiffies, timeout)) {
		if (!(hwrpb->txrdy & cpumask))
			return 0;
		udelay(10);
		barrier();
	}

	return -1;
}

/*
 * Send a message to a secondary's console.  "START" is one such
 * interesting message.  ;-)
 */
static void __cpuinit
send_secondary_console_msg(char *str, int cpuid)
{
	struct percpu_struct *cpu;
	register char *cp1, *cp2;
	unsigned long cpumask;
	size_t len;

	cpu = (struct percpu_struct *)
		((char*)hwrpb
		 + hwrpb->processor_offset
		 + cpuid * hwrpb->processor_size);

	cpumask = (1UL << cpuid);
	if (wait_for_txrdy(cpumask))
		goto timeout;

	cp2 = str;
	len = strlen(cp2);
	*(unsigned int *)&cpu->ipc_buffer[0] = len;
	cp1 = (char *) &cpu->ipc_buffer[1];
	memcpy(cp1, cp2, len);

	/* atomic test and set */
	wmb();
	set_bit(cpuid, &hwrpb->rxrdy);

	if (wait_for_txrdy(cpumask))
		goto timeout;
	return;

 timeout:
	printk("Processor %x not ready\n", cpuid);
}

/*
 * A secondary console wants to send a message.  Receive it.
 */
static void
recv_secondary_console_msg(void)
{
	int mycpu, i, cnt;
	unsigned long txrdy = hwrpb->txrdy;
	char *cp1, *cp2, buf[80];
	struct percpu_struct *cpu;

	DBGS(("recv_secondary_console_msg: TXRDY 0x%lx.\n", txrdy));

	mycpu = hard_smp_processor_id();

	for (i = 0; i < NR_CPUS; i++) {
		if (!(txrdy & (1UL << i)))
			continue;

		DBGS(("recv_secondary_console_msg: "
		      "TXRDY contains CPU %d.\n", i));

		cpu = (struct percpu_struct *)
		  ((char*)hwrpb
		   + hwrpb->processor_offset
		   + i * hwrpb->processor_size);

 		DBGS(("recv_secondary_console_msg: on %d from %d"
		      " HALT_REASON 0x%lx FLAGS 0x%lx\n",
		      mycpu, i, cpu->halt_reason, cpu->flags));

		cnt = cpu->ipc_buffer[0] >> 32;
		if (cnt <= 0 || cnt >= 80)
			strcpy(buf, "<<< BOGUS MSG >>>");
		else {
			cp1 = (char *) &cpu->ipc_buffer[11];
			cp2 = buf;
			strcpy(cp2, cp1);
			
			while ((cp2 = strchr(cp2, '\r')) != 0) {
				*cp2 = ' ';
				if (cp2[1] == '\n')
					cp2[1] = ' ';
			}
		}

		DBGS((KERN_INFO "recv_secondary_console_msg: on %d "
		      "message is '%s'\n", mycpu, buf));
	}

	hwrpb->txrdy = 0;
}

/*
 * Convince the console to have a secondary cpu begin execution.
 */
static int __cpuinit
secondary_cpu_start(int cpuid, struct task_struct *idle)
{
	struct percpu_struct *cpu;
	struct pcb_struct *hwpcb, *ipcb;
	unsigned long timeout;
	  
	cpu = (struct percpu_struct *)
		((char*)hwrpb
		 + hwrpb->processor_offset
		 + cpuid * hwrpb->processor_size);
	hwpcb = (struct pcb_struct *) cpu->hwpcb;
	ipcb = &task_thread_info(idle)->pcb;

	/* Initialize the CPU's HWPCB to something just good enough for
	   us to get started.  Immediately after starting, we'll swpctx
	   to the target idle task's pcb.  Reuse the stack in the mean
	   time.  Precalculate the target PCBB.  */
	hwpcb->ksp = (unsigned long)ipcb + sizeof(union thread_union) - 16;
	hwpcb->usp = 0;
	hwpcb->ptbr = ipcb->ptbr;
	hwpcb->pcc = 0;
	hwpcb->asn = 0;
	hwpcb->unique = virt_to_phys(ipcb);
	hwpcb->flags = ipcb->flags;
	hwpcb->res1 = hwpcb->res2 = 0;

#if 0
	DBGS(("KSP 0x%lx PTBR 0x%lx VPTBR 0x%lx UNIQUE 0x%lx\n",
	      hwpcb->ksp, hwpcb->ptbr, hwrpb->vptb, hwpcb->unique));
#endif
	DBGS(("Starting secondary cpu %d: state 0x%lx pal_flags 0x%lx\n",
	      cpuid, idle->state, ipcb->flags));

	/* Setup HWRPB fields that SRM uses to activate secondary CPU */
	hwrpb->CPU_restart = __smp_callin;
	hwrpb->CPU_restart_data = (unsigned long) __smp_callin;

	/* Recalculate and update the HWRPB checksum */
	hwrpb_update_checksum(hwrpb);

	/*
	 * Send a "start" command to the specified processor.
	 */

	/* SRM III 3.4.1.3 */
	cpu->flags |= 0x22;	/* turn on Context Valid and Restart Capable */
	cpu->flags &= ~1;	/* turn off Bootstrap In Progress */
	wmb();

	send_secondary_console_msg("START\r\n", cpuid);

	/* Wait 10 seconds for an ACK from the console.  */
	timeout = jiffies + 10*HZ;
	while (time_before(jiffies, timeout)) {
		if (cpu->flags & 1)
			goto started;
		udelay(10);
		barrier();
	}
	printk(KERN_ERR "SMP: Processor %d failed to start.\n", cpuid);
	return -1;

 started:
	DBGS(("secondary_cpu_start: SUCCESS for CPU %d!!!\n", cpuid));
	return 0;
}

/*
 * Bring one cpu online.
 */
static int __cpuinit
smp_boot_one_cpu(int cpuid, struct task_struct *idle)
{
	unsigned long timeout;

	/* Signal the secondary to wait a moment.  */
	smp_secondary_alive = -1;

	/* Whirrr, whirrr, whirrrrrrrrr... */
	if (secondary_cpu_start(cpuid, idle))
		return -1;

	/* Notify the secondary CPU it can run calibrate_delay.  */
	mb();
	smp_secondary_alive = 0;

	/* We've been acked by the console; wait one second for
	   the task to start up for real.  */
	timeout = jiffies + 1*HZ;
	while (time_before(jiffies, timeout)) {
		if (smp_secondary_alive == 1)
			goto alive;
		udelay(10);
		barrier();
	}

	/* We failed to boot the CPU.  */

	printk(KERN_ERR "SMP: Processor %d is stuck.\n", cpuid);
	return -1;

 alive:
	/* Another "Red Snapper". */
	return 0;
}

/*
 * Called from setup_arch.  Detect an SMP system and which processors
 * are present.
 */
void __init
setup_smp(void)
{
	struct percpu_struct *cpubase, *cpu;
	unsigned long i;

	if (boot_cpuid != 0) {
		printk(KERN_WARNING "SMP: Booting off cpu %d instead of 0?\n",
		       boot_cpuid);
	}

	if (hwrpb->nr_processors > 1) {
		int boot_cpu_palrev;

		DBGS(("setup_smp: nr_processors %ld\n",
		      hwrpb->nr_processors));

		cpubase = (struct percpu_struct *)
			((char*)hwrpb + hwrpb->processor_offset);
		boot_cpu_palrev = cpubase->pal_revision;

		for (i = 0; i < hwrpb->nr_processors; i++) {
			cpu = (struct percpu_struct *)
				((char *)cpubase + i*hwrpb->processor_size);
			if ((cpu->flags & 0x1cc) == 0x1cc) {
				smp_num_probed++;
				set_cpu_possible(i, true);
				set_cpu_present(i, true);
				cpu->pal_revision = boot_cpu_palrev;
			}

			DBGS(("setup_smp: CPU %d: flags 0x%lx type 0x%lx\n",
			      i, cpu->flags, cpu->type));
			DBGS(("setup_smp: CPU %d: PAL rev 0x%lx\n",
			      i, cpu->pal_revision));
		}
	} else {
		smp_num_probed = 1;
	}

	printk(KERN_INFO "SMP: %d CPUs probed -- cpu_present_mask = %lx\n",
	       smp_num_probed, cpumask_bits(cpu_present_mask)[0]);
}

/*
 * Called by smp_init prepare the secondaries
 */
void __init
smp_prepare_cpus(unsigned int max_cpus)
{
	/* Take care of some initial bookkeeping.  */
	memset(ipi_data, 0, sizeof(ipi_data));

	current_thread_info()->cpu = boot_cpuid;

	smp_store_cpu_info(boot_cpuid);
	smp_setup_percpu_timer(boot_cpuid);

	/* Nothing to do on a UP box, or when told not to.  */
	if (smp_num_probed == 1 || max_cpus == 0) {
		init_cpu_possible(cpumask_of(boot_cpuid));
		init_cpu_present(cpumask_of(boot_cpuid));
		printk(KERN_INFO "SMP mode deactivated.\n");
		return;
	}

	printk(KERN_INFO "SMP starting up secondaries.\n");

	smp_num_cpus = smp_num_probed;
}

void __devinit
smp_prepare_boot_cpu(void)
{
}

int __cpuinit
__cpu_up(unsigned int cpu, struct task_struct *tidle)
{
	smp_boot_one_cpu(cpu, tidle);

	return cpu_online(cpu) ? 0 : -ENOSYS;
}

void __init
smp_cpus_done(unsigned int max_cpus)
{
	int cpu;
	unsigned long bogosum = 0;

	for(cpu = 0; cpu < NR_CPUS; cpu++) 
		if (cpu_online(cpu))
			bogosum += cpu_data[cpu].loops_per_jiffy;
	
	printk(KERN_INFO "SMP: Total of %d processors activated "
	       "(%lu.%02lu BogoMIPS).\n",
	       num_online_cpus(), 
	       (bogosum + 2500) / (500000/HZ),
	       ((bogosum + 2500) / (5000/HZ)) % 100);
}

\f
void
smp_percpu_timer_interrupt(struct pt_regs *regs)
{
	struct pt_regs *old_regs;
	int cpu = smp_processor_id();
	unsigned long user = user_mode(regs);
	struct cpuinfo_alpha *data = &cpu_data[cpu];

	old_regs = set_irq_regs(regs);

	/* Record kernel PC.  */
	profile_tick(CPU_PROFILING);

	if (!--data->prof_counter) {
		/* We need to make like a normal interrupt -- otherwise
		   timer interrupts ignore the global interrupt lock,
		   which would be a Bad Thing.  */
		irq_enter();

		update_process_times(user);

		data->prof_counter = data->prof_multiplier;

		irq_exit();
	}
	set_irq_regs(old_regs);
}

int
setup_profiling_timer(unsigned int multiplier)
{
	return -EINVAL;
}

\f
static void
send_ipi_message(const struct cpumask *to_whom, enum ipi_message_type operation)
{
	int i;

	mb();
	for_each_cpu(i, to_whom)
		set_bit(operation, &ipi_data[i].bits);

	mb();
	for_each_cpu(i, to_whom)
		wripir(i);
}

void
handle_ipi(struct pt_regs *regs)
{
	int this_cpu = smp_processor_id();
	unsigned long *pending_ipis = &ipi_data[this_cpu].bits;
	unsigned long ops;

#if 0
	DBGS(("handle_ipi: on CPU %d ops 0x%lx PC 0x%lx\n",
	      this_cpu, *pending_ipis, regs->pc));
#endif

	mb();	/* Order interrupt and bit testing. */
	while ((ops = xchg(pending_ipis, 0)) != 0) {
	  mb();	/* Order bit clearing and data access. */
	  do {
		unsigned long which;

		which = ops & -ops;
		ops &= ~which;
		which = __ffs(which);

		switch (which) {
		case IPI_RESCHEDULE:
			scheduler_ipi();
			break;

		case IPI_CALL_FUNC:
			generic_smp_call_function_interrupt();
			break;

		case IPI_CALL_FUNC_SINGLE:
			generic_smp_call_function_single_interrupt();
			break;

		case IPI_CPU_STOP:
			halt();

		default:
			printk(KERN_CRIT "Unknown IPI on CPU %d: %lu\n",
			       this_cpu, which);
			break;
		}
	  } while (ops);

	  mb();	/* Order data access and bit testing. */
	}

	cpu_data[this_cpu].ipi_count++;

	if (hwrpb->txrdy)
		recv_secondary_console_msg();
}

void
smp_send_reschedule(int cpu)
{
#ifdef DEBUG_IPI_MSG
	if (cpu == hard_smp_processor_id())
		printk(KERN_WARNING
		       "smp_send_reschedule: Sending IPI to self.\n");
#endif
	send_ipi_message(cpumask_of(cpu), IPI_RESCHEDULE);
}

void
smp_send_stop(void)
{
	cpumask_t to_whom;
	cpumask_copy(&to_whom, cpu_possible_mask);
	cpumask_clear_cpu(smp_processor_id(), &to_whom);
#ifdef DEBUG_IPI_MSG
	if (hard_smp_processor_id() != boot_cpu_id)
		printk(KERN_WARNING "smp_send_stop: Not on boot cpu.\n");
#endif
	send_ipi_message(&to_whom, IPI_CPU_STOP);
}

void arch_send_call_function_ipi_mask(const struct cpumask *mask)
{
	send_ipi_message(mask, IPI_CALL_FUNC);
}

void arch_send_call_function_single_ipi(int cpu)
{
	send_ipi_message(cpumask_of(cpu), IPI_CALL_FUNC_SINGLE);
}

static void
ipi_imb(void *ignored)
{
	imb();
}

void
smp_imb(void)
{
	/* Must wait other processors to flush their icache before continue. */
	if (on_each_cpu(ipi_imb, NULL, 1))
		printk(KERN_CRIT "smp_imb: timed out\n");
}
EXPORT_SYMBOL(smp_imb);

static void
ipi_flush_tlb_all(void *ignored)
{
	tbia();
}

void
flush_tlb_all(void)
{
	/* Although we don't have any data to pass, we do want to
	   synchronize with the other processors.  */
	if (on_each_cpu(ipi_flush_tlb_all, NULL, 1)) {
		printk(KERN_CRIT "flush_tlb_all: timed out\n");
	}
}

#define asn_locked() (cpu_data[smp_processor_id()].asn_lock)

static void
ipi_flush_tlb_mm(void *x)
{
	struct mm_struct *mm = (struct mm_struct *) x;
	if (mm == current->active_mm && !asn_locked())
		flush_tlb_current(mm);
	else
		flush_tlb_other(mm);
}

void
flush_tlb_mm(struct mm_struct *mm)
{
	preempt_disable();

	if (mm == current->active_mm) {
		flush_tlb_current(mm);
		if (atomic_read(&mm->mm_users) <= 1) {
			int cpu, this_cpu = smp_processor_id();
			for (cpu = 0; cpu < NR_CPUS; cpu++) {
				if (!cpu_online(cpu) || cpu == this_cpu)
					continue;
				if (mm->context[cpu])
					mm->context[cpu] = 0;
			}
			preempt_enable();
			return;
		}
	}

	if (smp_call_function(ipi_flush_tlb_mm, mm, 1)) {
		printk(KERN_CRIT "flush_tlb_mm: timed out\n");
	}

	preempt_enable();
}
EXPORT_SYMBOL(flush_tlb_mm);

struct flush_tlb_page_struct {
	struct vm_area_struct *vma;
	struct mm_struct *mm;
	unsigned long addr;
};

static void
ipi_flush_tlb_page(void *x)
{
	struct flush_tlb_page_struct *data = (struct flush_tlb_page_struct *)x;
	struct mm_struct * mm = data->mm;

	if (mm == current->active_mm && !asn_locked())
		flush_tlb_current_page(mm, data->vma, data->addr);
	else
		flush_tlb_other(mm);
}

void
flush_tlb_page(struct vm_area_struct *vma, unsigned long addr)
{
	struct flush_tlb_page_struct data;
	struct mm_struct *mm = vma->vm_mm;

	preempt_disable();

	if (mm == current->active_mm) {
		flush_tlb_current_page(mm, vma, addr);
		if (atomic_read(&mm->mm_users) <= 1) {
			int cpu, this_cpu = smp_processor_id();
			for (cpu = 0; cpu < NR_CPUS; cpu++) {
				if (!cpu_online(cpu) || cpu == this_cpu)
					continue;
				if (mm->context[cpu])
					mm->context[cpu] = 0;
			}
			preempt_enable();
			return;
		}
	}

	data.vma = vma;
	data.mm = mm;
	data.addr = addr;

	if (smp_call_function(ipi_flush_tlb_page, &data, 1)) {
		printk(KERN_CRIT "flush_tlb_page: timed out\n");
	}

	preempt_enable();
}
EXPORT_SYMBOL(flush_tlb_page);

void
flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
{
	/* On the Alpha we always flush the whole user tlb.  */
	flush_tlb_mm(vma->vm_mm);
}
EXPORT_SYMBOL(flush_tlb_range);

static void
ipi_flush_icache_page(void *x)
{
	struct mm_struct *mm = (struct mm_struct *) x;
	if (mm == current->active_mm && !asn_locked())
		__load_new_mm_context(mm);
	else
		flush_tlb_other(mm);
}

void
flush_icache_user_range(struct vm_area_struct *vma, struct page *page,
			unsigned long addr, int len)
{
	struct mm_struct *mm = vma->vm_mm;

	if ((vma->vm_flags & VM_EXEC) == 0)
		return;

	preempt_disable();

	if (mm == current->active_mm) {
		__load_new_mm_context(mm);
		if (atomic_read(&mm->mm_users) <= 1) {
			int cpu, this_cpu = smp_processor_id();
			for (cpu = 0; cpu < NR_CPUS; cpu++) {
				if (!cpu_online(cpu) || cpu == this_cpu)
					continue;
				if (mm->context[cpu])
					mm->context[cpu] = 0;
			}
			preempt_enable();
			return;
		}
	}

	if (smp_call_function(ipi_flush_icache_page, mm, 1)) {
		printk(KERN_CRIT "flush_icache_page: timed out\n");
	}

	preempt_enable();
}
Commit	Line	Data
1da177e4 LT	1	/*
	2	* linux/arch/alpha/kernel/smp.c
	3	*
	4	* 2001-07-09 Phil Ezolt (Phillip.Ezolt@compaq.com)
	5	* Renamed modified smp_call_function to smp_call_function_on_cpu()
	6	* Created an function that conforms to the old calling convention
	7	* of smp_call_function().
	8	*
	9	* This is helpful for DCPI.
	10	*
	11	*/
	12
	13	#include <linux/errno.h>
	14	#include <linux/kernel.h>
	15	#include <linux/kernel_stat.h>
	16	#include <linux/module.h>
	17	#include <linux/sched.h>
	18	#include <linux/mm.h>
4e950f6f	19	#include <linux/err.h>
1da177e4 LT	20	#include <linux/threads.h>
1da177e4 LT	21	#include <linux/smp.h>
1da177e4 LT	22	#include <linux/interrupt.h>
	23	#include <linux/init.h>
	24	#include <linux/delay.h>
	25	#include <linux/spinlock.h>
	26	#include <linux/irq.h>
	27	#include <linux/cache.h>
	28	#include <linux/profile.h>
	29	#include <linux/bitops.h>
574f34ce	30	#include <linux/cpu.h>
1da177e4 LT	31
	32	#include <asm/hwrpb.h>
	33	#include <asm/ptrace.h>
60063497	34	#include <linux/atomic.h>
1da177e4 LT	35
	36	#include <asm/io.h>
	37	#include <asm/irq.h>
	38	#include <asm/pgtable.h>
	39	#include <asm/pgalloc.h>
	40	#include <asm/mmu_context.h>
	41	#include <asm/tlbflush.h>
	42
	43	#include "proto.h"
	44	#include "irq_impl.h"
	45
	46
	47	#define DEBUG_SMP 0
	48	#if DEBUG_SMP
	49	#define DBGS(args) printk args
	50	#else
	51	#define DBGS(args)
	52	#endif
	53
	54	/* A collection of per-processor data. */
	55	struct cpuinfo_alpha cpu_data[NR_CPUS];
cff52daf	56	EXPORT_SYMBOL(cpu_data);
1da177e4 LT	57
	58	/* A collection of single bit ipi messages. */
	59	static struct {
	60	unsigned long bits ____cacheline_aligned;
	61	} ipi_data[NR_CPUS] __cacheline_aligned;
	62
	63	enum ipi_message_type {
	64	IPI_RESCHEDULE,
	65	IPI_CALL_FUNC,
c524a1d8	66	IPI_CALL_FUNC_SINGLE,
1da177e4 LT	67	IPI_CPU_STOP,
	68	};
	69
	70	/* Set to a secondary's cpuid when it comes online. */
cc040a8a	71	static int smp_secondary_alive __devinitdata = 0;
1da177e4	72
1da177e4 LT	73	int smp_num_probed; /* Internal processor count */
1da177e4 LT	74	int smp_num_cpus = 1; /* Number that came online. */
cff52daf	75	EXPORT_SYMBOL(smp_num_cpus);
1da177e4	76
1da177e4 LT	77	/*
	78	* Called by both boot and secondaries to move global data into
	79	* per-processor storage.
	80	*/
	81	static inline void __init
	82	smp_store_cpu_info(int cpuid)
	83	{
	84	cpu_data[cpuid].loops_per_jiffy = loops_per_jiffy;
	85	cpu_data[cpuid].last_asn = ASN_FIRST_VERSION;
	86	cpu_data[cpuid].need_new_asn = 0;
	87	cpu_data[cpuid].asn_lock = 0;
	88	}
	89
	90	/*
	91	* Ideally sets up per-cpu profiling hooks. Doesn't do much now...
	92	*/
	93	static inline void __init
	94	smp_setup_percpu_timer(int cpuid)
	95	{
	96	cpu_data[cpuid].prof_counter = 1;
	97	cpu_data[cpuid].prof_multiplier = 1;
	98	}
	99
	100	static void __init
	101	wait_boot_cpu_to_stop(int cpuid)
	102	{
	103	unsigned long stop = jiffies + 10*HZ;
	104
	105	while (time_before(jiffies, stop)) {
	106	if (!smp_secondary_alive)
	107	return;
	108	barrier();
	109	}
	110
	111	printk("wait_boot_cpu_to_stop: FAILED on CPU %d, hanging now\n", cpuid);
	112	for (;;)
	113	barrier();
	114	}
	115
	116	/*
	117	* Where secondaries begin a life of C.
	118	*/
7d6a8a1c	119	void __cpuinit
1da177e4 LT	120	smp_callin(void)
	121	{
	122	int cpuid = hard_smp_processor_id();
	123
1371be0f	124	if (cpu_online(cpuid)) {
1da177e4 LT	125	printk("??, cpu 0x%x already present??\n", cpuid);
	126	BUG();
	127	}
1371be0f	128	set_cpu_online(cpuid, true);
1da177e4 LT	129
	130	/* Turn on machine checks. */
	131	wrmces(7);
	132
	133	/* Set trap vectors. */
	134	trap_init();
	135
	136	/* Set interrupt vector. */
	137	wrent(entInt, 0);
	138
	139	/* Get our local ticker going. */
	140	smp_setup_percpu_timer(cpuid);
	141
	142	/* Call platform-specific callin, if specified */
	143	if (alpha_mv.smp_callin) alpha_mv.smp_callin();
	144
	145	/* All kernel threads share the same mm context. */
	146	atomic_inc(&init_mm.mm_count);
	147	current->active_mm = &init_mm;
	148
e545a614 MS	149	/* inform the notifiers about the new cpu */
	150	notify_cpu_starting(cpuid);
	151
1da177e4 LT	152	/* Must have completely accurate bogos. */
	153	local_irq_enable();
	154
	155	/* Wait boot CPU to stop with irq enabled before running
	156	calibrate_delay. */
	157	wait_boot_cpu_to_stop(cpuid);
	158	mb();
	159	calibrate_delay();
	160
	161	smp_store_cpu_info(cpuid);
	162	/* Allow master to continue only after we written loops_per_jiffy. */
	163	wmb();
	164	smp_secondary_alive = 1;
	165
	166	DBGS(("smp_callin: commencing CPU %d current %p active_mm %p\n",
	167	cpuid, current, current->active_mm));
	168
6a6c0272	169	preempt_disable();
1da177e4 LT	170	/* Do nothing. */
	171	cpu_idle();
	172	}
	173
	174	/* Wait until hwrpb->txrdy is clear for cpu. Return -1 on timeout. */
cc040a8a	175	static int __devinit
1da177e4 LT	176	wait_for_txrdy (unsigned long cpumask)
	177	{
	178	unsigned long timeout;
	179
	180	if (!(hwrpb->txrdy & cpumask))
	181	return 0;
	182
	183	timeout = jiffies + 10*HZ;
	184	while (time_before(jiffies, timeout)) {
	185	if (!(hwrpb->txrdy & cpumask))
	186	return 0;
	187	udelay(10);
	188	barrier();
	189	}
	190
	191	return -1;
	192	}
	193
	194	/*
	195	* Send a message to a secondary's console. "START" is one such
	196	* interesting message. ;-)
	197	*/
7d6a8a1c	198	static void __cpuinit
1da177e4 LT	199	send_secondary_console_msg(char *str, int cpuid)
	200	{
	201	struct percpu_struct *cpu;
	202	register char cp1, cp2;
	203	unsigned long cpumask;
	204	size_t len;
	205
	206	cpu = (struct percpu_struct *)
	207	((char*)hwrpb
	208	+ hwrpb->processor_offset
	209	+ cpuid * hwrpb->processor_size);
	210
	211	cpumask = (1UL << cpuid);
	212	if (wait_for_txrdy(cpumask))
	213	goto timeout;
	214
	215	cp2 = str;
	216	len = strlen(cp2);
	217	(unsigned int )&cpu->ipc_buffer[0] = len;
	218	cp1 = (char *) &cpu->ipc_buffer[1];
	219	memcpy(cp1, cp2, len);
	220
	221	/* atomic test and set */
	222	wmb();
	223	set_bit(cpuid, &hwrpb->rxrdy);
	224
	225	if (wait_for_txrdy(cpumask))
	226	goto timeout;
	227	return;
	228
	229	timeout:
	230	printk("Processor %x not ready\n", cpuid);
	231	}
	232
	233	/*
	234	* A secondary console wants to send a message. Receive it.
	235	*/
	236	static void
	237	recv_secondary_console_msg(void)
	238	{
	239	int mycpu, i, cnt;
	240	unsigned long txrdy = hwrpb->txrdy;
	241	char cp1, cp2, buf[80];
	242	struct percpu_struct *cpu;
	243
	244	DBGS(("recv_secondary_console_msg: TXRDY 0x%lx.\n", txrdy));
	245
	246	mycpu = hard_smp_processor_id();
	247
	248	for (i = 0; i < NR_CPUS; i++) {
	249	if (!(txrdy & (1UL << i)))
	250	continue;
	251
	252	DBGS(("recv_secondary_console_msg: "
	253	"TXRDY contains CPU %d.\n", i));
	254
	255	cpu = (struct percpu_struct *)
	256	((char*)hwrpb
	257	+ hwrpb->processor_offset
	258	+ i * hwrpb->processor_size);
	259
	260	DBGS(("recv_secondary_console_msg: on %d from %d"
	261	" HALT_REASON 0x%lx FLAGS 0x%lx\n",
	262	mycpu, i, cpu->halt_reason, cpu->flags));
263
264	cnt = cpu->ipc_buffer[0] >> 32;
265	if (cnt <= 0 \|\| cnt >= 80)
266	strcpy(buf, "<<< BOGUS MSG >>>");
267	else {
268	cp1 = (char *) &cpu->ipc_buffer[11];
269	cp2 = buf;
270	strcpy(cp2, cp1);
271
272	while ((cp2 = strchr(cp2, '\r')) != 0) {
273	*cp2 = ' ';
274	if (cp2[1] == '\n')
275	cp2[1] = ' ';
276	}
277	}
278
279	DBGS((KERN_INFO "recv_secondary_console_msg: on %d "
280	"message is '%s'\n", mycpu, buf));
281	}
282
283	hwrpb->txrdy = 0;
284	}
285
286	/*
287	* Convince the console to have a secondary cpu begin execution.
288	*/
7d6a8a1c	289	static int __cpuinit
1da177e4 LT	290	secondary_cpu_start(int cpuid, struct task_struct *idle)
	291	{
	292	struct percpu_struct *cpu;
	293	struct pcb_struct hwpcb, ipcb;
	294	unsigned long timeout;
	295
	296	cpu = (struct percpu_struct *)
	297	((char*)hwrpb
	298	+ hwrpb->processor_offset
	299	+ cpuid * hwrpb->processor_size);
	300	hwpcb = (struct pcb_struct *) cpu->hwpcb;
37bfbaf9	301	ipcb = &task_thread_info(idle)->pcb;
1da177e4 LT	302
	303	/* Initialize the CPU's HWPCB to something just good enough for
	304	us to get started. Immediately after starting, we'll swpctx
	305	to the target idle task's pcb. Reuse the stack in the mean
	306	time. Precalculate the target PCBB. */
	307	hwpcb->ksp = (unsigned long)ipcb + sizeof(union thread_union) - 16;
	308	hwpcb->usp = 0;
	309	hwpcb->ptbr = ipcb->ptbr;
	310	hwpcb->pcc = 0;
	311	hwpcb->asn = 0;
	312	hwpcb->unique = virt_to_phys(ipcb);
	313	hwpcb->flags = ipcb->flags;
	314	hwpcb->res1 = hwpcb->res2 = 0;
	315
	316	#if 0
	317	DBGS(("KSP 0x%lx PTBR 0x%lx VPTBR 0x%lx UNIQUE 0x%lx\n",
	318	hwpcb->ksp, hwpcb->ptbr, hwrpb->vptb, hwpcb->unique));
	319	#endif
	320	DBGS(("Starting secondary cpu %d: state 0x%lx pal_flags 0x%lx\n",
	321	cpuid, idle->state, ipcb->flags));
	322
	323	/* Setup HWRPB fields that SRM uses to activate secondary CPU */
	324	hwrpb->CPU_restart = __smp_callin;
	325	hwrpb->CPU_restart_data = (unsigned long) __smp_callin;
	326
	327	/* Recalculate and update the HWRPB checksum */
	328	hwrpb_update_checksum(hwrpb);
	329
	330	/*
	331	* Send a "start" command to the specified processor.
	332	*/
	333
	334	/* SRM III 3.4.1.3 */
	335	cpu->flags \|= 0x22; /* turn on Context Valid and Restart Capable */
	336	cpu->flags &= ~1; /* turn off Bootstrap In Progress */
	337	wmb();
	338
	339	send_secondary_console_msg("START\r\n", cpuid);
	340
	341	/* Wait 10 seconds for an ACK from the console. */
	342	timeout = jiffies + 10*HZ;
	343	while (time_before(jiffies, timeout)) {
	344	if (cpu->flags & 1)
	345	goto started;
	346	udelay(10);
	347	barrier();
	348	}
	349	printk(KERN_ERR "SMP: Processor %d failed to start.\n", cpuid);
	350	return -1;
	351
	352	started:
	353	DBGS(("secondary_cpu_start: SUCCESS for CPU %d!!!\n", cpuid));
	354	return 0;
	355	}
	356
	357	/*
	358	* Bring one cpu online.
	359	*/
ed5f6561	360	static int __cpuinit
2ec9415c	361	smp_boot_one_cpu(int cpuid, struct task_struct *idle)
1da177e4	362	{
1da177e4 LT	363	unsigned long timeout;
1da177e4 LT	364
1da177e4 LT	365	/* Signal the secondary to wait a moment. */
	366	smp_secondary_alive = -1;
	367
	368	/* Whirrr, whirrr, whirrrrrrrrr... */
	369	if (secondary_cpu_start(cpuid, idle))
	370	return -1;
	371
	372	/* Notify the secondary CPU it can run calibrate_delay. */
	373	mb();
	374	smp_secondary_alive = 0;
	375
	376	/* We've been acked by the console; wait one second for
	377	the task to start up for real. */
	378	timeout = jiffies + 1*HZ;
	379	while (time_before(jiffies, timeout)) {
	380	if (smp_secondary_alive == 1)
	381	goto alive;
	382	udelay(10);
	383	barrier();
	384	}
	385
	386	/* We failed to boot the CPU. */
	387
	388	printk(KERN_ERR "SMP: Processor %d is stuck.\n", cpuid);
	389	return -1;
	390
	391	alive:
	392	/* Another "Red Snapper". */
	393	return 0;
	394	}
	395
	396	/*
	397	* Called from setup_arch. Detect an SMP system and which processors
	398	* are present.
	399	*/
	400	void __init
	401	setup_smp(void)
	402	{
	403	struct percpu_struct cpubase, cpu;
	404	unsigned long i;
	405
	406	if (boot_cpuid != 0) {
	407	printk(KERN_WARNING "SMP: Booting off cpu %d instead of 0?\n",
	408	boot_cpuid);
	409	}
	410
	411	if (hwrpb->nr_processors > 1) {
	412	int boot_cpu_palrev;
	413
	414	DBGS(("setup_smp: nr_processors %ld\n",
	415	hwrpb->nr_processors));
	416
	417	cpubase = (struct percpu_struct *)
	418	((char*)hwrpb + hwrpb->processor_offset);
	419	boot_cpu_palrev = cpubase->pal_revision;
	420
	421	for (i = 0; i < hwrpb->nr_processors; i++) {
	422	cpu = (struct percpu_struct *)
	423	((char )cpubase + ihwrpb->processor_size);
	424	if ((cpu->flags & 0x1cc) == 0x1cc) {
	425	smp_num_probed++;
1371be0f RR	426	set_cpu_possible(i, true);
1371be0f RR	427	set_cpu_present(i, true);
1da177e4 LT	428	cpu->pal_revision = boot_cpu_palrev;
	429	}
	430
	431	DBGS(("setup_smp: CPU %d: flags 0x%lx type 0x%lx\n",
	432	i, cpu->flags, cpu->type));
	433	DBGS(("setup_smp: CPU %d: PAL rev 0x%lx\n",
	434	i, cpu->pal_revision));
	435	}
	436	} else {
	437	smp_num_probed = 1;
1da177e4	438	}
1da177e4	439
5f054e31	440	printk(KERN_INFO "SMP: %d CPUs probed -- cpu_present_mask = %lx\n",
81740fc6	441	smp_num_probed, cpumask_bits(cpu_present_mask)[0]);
1da177e4 LT	442	}
	443
	444	/*
	445	* Called by smp_init prepare the secondaries
	446	*/
	447	void __init
	448	smp_prepare_cpus(unsigned int max_cpus)
	449	{
1da177e4 LT	450	/* Take care of some initial bookkeeping. */
	451	memset(ipi_data, 0, sizeof(ipi_data));
	452
	453	current_thread_info()->cpu = boot_cpuid;
	454
	455	smp_store_cpu_info(boot_cpuid);
	456	smp_setup_percpu_timer(boot_cpuid);
	457
	458	/* Nothing to do on a UP box, or when told not to. */
	459	if (smp_num_probed == 1 \|\| max_cpus == 0) {
1371be0f RR	460	init_cpu_possible(cpumask_of(boot_cpuid));
1371be0f RR	461	init_cpu_present(cpumask_of(boot_cpuid));
1da177e4 LT	462	printk(KERN_INFO "SMP mode deactivated.\n");
	463	return;
	464	}
	465
	466	printk(KERN_INFO "SMP starting up secondaries.\n");
	467
328c2a8a	468	smp_num_cpus = smp_num_probed;
1da177e4 LT	469	}
	470
	471	void __devinit
	472	smp_prepare_boot_cpu(void)
	473	{
1da177e4 LT	474	}
1da177e4 LT	475
ed5f6561	476	int __cpuinit
8239c25f	477	__cpu_up(unsigned int cpu, struct task_struct *tidle)
1da177e4	478	{
2ec9415c	479	smp_boot_one_cpu(cpu, tidle);
1da177e4 LT	480
	481	return cpu_online(cpu) ? 0 : -ENOSYS;
	482	}
	483
	484	void __init
	485	smp_cpus_done(unsigned int max_cpus)
	486	{
	487	int cpu;
	488	unsigned long bogosum = 0;
	489
	490	for(cpu = 0; cpu < NR_CPUS; cpu++)
	491	if (cpu_online(cpu))
	492	bogosum += cpu_data[cpu].loops_per_jiffy;
	493
	494	printk(KERN_INFO "SMP: Total of %d processors activated "
	495	"(%lu.%02lu BogoMIPS).\n",
	496	num_online_cpus(),
	497	(bogosum + 2500) / (500000/HZ),
	498	((bogosum + 2500) / (5000/HZ)) % 100);
	499	}
	500
	501	\f
	502	void
	503	smp_percpu_timer_interrupt(struct pt_regs *regs)
	504	{
8774cb81	505	struct pt_regs *old_regs;
1da177e4 LT	506	int cpu = smp_processor_id();
	507	unsigned long user = user_mode(regs);
	508	struct cpuinfo_alpha *data = &cpu_data[cpu];
	509
8774cb81 AV	510	old_regs = set_irq_regs(regs);
8774cb81 AV	511
1da177e4	512	/* Record kernel PC. */
8774cb81	513	profile_tick(CPU_PROFILING);
1da177e4 LT	514
	515	if (!--data->prof_counter) {
	516	/* We need to make like a normal interrupt -- otherwise
	517	timer interrupts ignore the global interrupt lock,
	518	which would be a Bad Thing. */
	519	irq_enter();
	520
	521	update_process_times(user);
	522
	523	data->prof_counter = data->prof_multiplier;
	524
	525	irq_exit();
	526	}
8774cb81	527	set_irq_regs(old_regs);
1da177e4 LT	528	}
1da177e4 LT	529
ed5f6561	530	int
1da177e4 LT	531	setup_profiling_timer(unsigned int multiplier)
	532	{
	533	return -EINVAL;
	534	}
	535
	536	\f
	537	static void
81065e4f	538	send_ipi_message(const struct cpumask *to_whom, enum ipi_message_type operation)
1da177e4 LT	539	{
	540	int i;
	541
	542	mb();
81065e4f	543	for_each_cpu(i, to_whom)
1da177e4 LT	544	set_bit(operation, &ipi_data[i].bits);
	545
	546	mb();
81065e4f	547	for_each_cpu(i, to_whom)
1da177e4 LT	548	wripir(i);
	549	}
	550
1da177e4 LT	551	void
	552	handle_ipi(struct pt_regs *regs)
	553	{
	554	int this_cpu = smp_processor_id();
	555	unsigned long *pending_ipis = &ipi_data[this_cpu].bits;
	556	unsigned long ops;
	557
	558	#if 0
	559	DBGS(("handle_ipi: on CPU %d ops 0x%lx PC 0x%lx\n",
	560	this_cpu, *pending_ipis, regs->pc));
	561	#endif
	562
	563	mb(); /* Order interrupt and bit testing. */
	564	while ((ops = xchg(pending_ipis, 0)) != 0) {
	565	mb(); /* Order bit clearing and data access. */
	566	do {
	567	unsigned long which;
	568
	569	which = ops & -ops;
	570	ops &= ~which;
	571	which = __ffs(which);
	572
	573	switch (which) {
	574	case IPI_RESCHEDULE:
184748cc	575	scheduler_ipi();
1da177e4 LT	576	break;
	577
	578	case IPI_CALL_FUNC:
c524a1d8 JA	579	generic_smp_call_function_interrupt();
	580	break;
	581
	582	case IPI_CALL_FUNC_SINGLE:
	583	generic_smp_call_function_single_interrupt();
1da177e4	584	break;
1da177e4 LT	585
	586	case IPI_CPU_STOP:
	587	halt();
	588
	589	default:
	590	printk(KERN_CRIT "Unknown IPI on CPU %d: %lu\n",
	591	this_cpu, which);
	592	break;
	593	}
	594	} while (ops);
	595
	596	mb(); /* Order data access and bit testing. */
	597	}
	598
	599	cpu_data[this_cpu].ipi_count++;
	600
	601	if (hwrpb->txrdy)
	602	recv_secondary_console_msg();
	603	}
	604
	605	void
	606	smp_send_reschedule(int cpu)
	607	{
	608	#ifdef DEBUG_IPI_MSG
	609	if (cpu == hard_smp_processor_id())
	610	printk(KERN_WARNING
	611	"smp_send_reschedule: Sending IPI to self.\n");
	612	#endif
81065e4f	613	send_ipi_message(cpumask_of(cpu), IPI_RESCHEDULE);
1da177e4 LT	614	}
	615
	616	void
	617	smp_send_stop(void)
	618	{
81740fc6 KM	619	cpumask_t to_whom;
	620	cpumask_copy(&to_whom, cpu_possible_mask);
	621	cpumask_clear_cpu(smp_processor_id(), &to_whom);
1da177e4 LT	622	#ifdef DEBUG_IPI_MSG
	623	if (hard_smp_processor_id() != boot_cpu_id)
	624	printk(KERN_WARNING "smp_send_stop: Not on boot cpu.\n");
	625	#endif
81065e4f	626	send_ipi_message(&to_whom, IPI_CPU_STOP);
1da177e4 LT	627	}
1da177e4 LT	628
81065e4f	629	void arch_send_call_function_ipi_mask(const struct cpumask *mask)
1da177e4	630	{
c524a1d8	631	send_ipi_message(mask, IPI_CALL_FUNC);
1da177e4 LT	632	}
1da177e4 LT	633
c524a1d8	634	void arch_send_call_function_single_ipi(int cpu)
1da177e4	635	{
81065e4f	636	send_ipi_message(cpumask_of(cpu), IPI_CALL_FUNC_SINGLE);
1da177e4 LT	637	}
	638
	639	static void
	640	ipi_imb(void *ignored)
	641	{
	642	imb();
	643	}
	644
	645	void
	646	smp_imb(void)
	647	{
	648	/* Must wait other processors to flush their icache before continue. */
15c8b6c1	649	if (on_each_cpu(ipi_imb, NULL, 1))
1da177e4 LT	650	printk(KERN_CRIT "smp_imb: timed out\n");
1da177e4 LT	651	}
cff52daf	652	EXPORT_SYMBOL(smp_imb);
1da177e4 LT	653
	654	static void
	655	ipi_flush_tlb_all(void *ignored)
	656	{
	657	tbia();
	658	}
	659
	660	void
	661	flush_tlb_all(void)
	662	{
	663	/* Although we don't have any data to pass, we do want to
	664	synchronize with the other processors. */
15c8b6c1	665	if (on_each_cpu(ipi_flush_tlb_all, NULL, 1)) {
1da177e4 LT	666	printk(KERN_CRIT "flush_tlb_all: timed out\n");
	667	}
	668	}
	669
	670	#define asn_locked() (cpu_data[smp_processor_id()].asn_lock)
	671
	672	static void
	673	ipi_flush_tlb_mm(void *x)
	674	{
	675	struct mm_struct mm = (struct mm_struct ) x;
	676	if (mm == current->active_mm && !asn_locked())
	677	flush_tlb_current(mm);
	678	else
	679	flush_tlb_other(mm);
	680	}
	681
	682	void
	683	flush_tlb_mm(struct mm_struct *mm)
	684	{
	685	preempt_disable();
	686
	687	if (mm == current->active_mm) {
	688	flush_tlb_current(mm);
	689	if (atomic_read(&mm->mm_users) <= 1) {
	690	int cpu, this_cpu = smp_processor_id();
	691	for (cpu = 0; cpu < NR_CPUS; cpu++) {
	692	if (!cpu_online(cpu) \|\| cpu == this_cpu)
	693	continue;
	694	if (mm->context[cpu])
	695	mm->context[cpu] = 0;
	696	}
	697	preempt_enable();
	698	return;
	699	}
	700	}
	701
8691e5a8	702	if (smp_call_function(ipi_flush_tlb_mm, mm, 1)) {
1da177e4 LT	703	printk(KERN_CRIT "flush_tlb_mm: timed out\n");
	704	}
	705
	706	preempt_enable();
	707	}
cff52daf	708	EXPORT_SYMBOL(flush_tlb_mm);
1da177e4 LT	709
	710	struct flush_tlb_page_struct {
	711	struct vm_area_struct *vma;
	712	struct mm_struct *mm;
	713	unsigned long addr;
	714	};
	715
	716	static void
	717	ipi_flush_tlb_page(void *x)
	718	{
	719	struct flush_tlb_page_struct data = (struct flush_tlb_page_struct )x;
	720	struct mm_struct * mm = data->mm;
	721
	722	if (mm == current->active_mm && !asn_locked())
	723	flush_tlb_current_page(mm, data->vma, data->addr);
	724	else
	725	flush_tlb_other(mm);
	726	}
	727
	728	void
	729	flush_tlb_page(struct vm_area_struct *vma, unsigned long addr)
	730	{
	731	struct flush_tlb_page_struct data;
	732	struct mm_struct *mm = vma->vm_mm;
	733
	734	preempt_disable();
	735
	736	if (mm == current->active_mm) {
	737	flush_tlb_current_page(mm, vma, addr);
	738	if (atomic_read(&mm->mm_users) <= 1) {
	739	int cpu, this_cpu = smp_processor_id();
	740	for (cpu = 0; cpu < NR_CPUS; cpu++) {
	741	if (!cpu_online(cpu) \|\| cpu == this_cpu)
	742	continue;
	743	if (mm->context[cpu])
	744	mm->context[cpu] = 0;
	745	}
	746	preempt_enable();
	747	return;
	748	}
	749	}
	750
	751	data.vma = vma;
	752	data.mm = mm;
	753	data.addr = addr;
	754
8691e5a8	755	if (smp_call_function(ipi_flush_tlb_page, &data, 1)) {
1da177e4 LT	756	printk(KERN_CRIT "flush_tlb_page: timed out\n");
	757	}
	758
	759	preempt_enable();
	760	}
cff52daf	761	EXPORT_SYMBOL(flush_tlb_page);
1da177e4 LT	762
	763	void
	764	flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
	765	{
	766	/* On the Alpha we always flush the whole user tlb. */
	767	flush_tlb_mm(vma->vm_mm);
	768	}
cff52daf	769	EXPORT_SYMBOL(flush_tlb_range);
1da177e4 LT	770
	771	static void
	772	ipi_flush_icache_page(void *x)
	773	{
	774	struct mm_struct mm = (struct mm_struct ) x;
	775	if (mm == current->active_mm && !asn_locked())
	776	__load_new_mm_context(mm);
	777	else
	778	flush_tlb_other(mm);
	779	}
	780
	781	void
	782	flush_icache_user_range(struct vm_area_struct vma, struct page page,
	783	unsigned long addr, int len)
	784	{
	785	struct mm_struct *mm = vma->vm_mm;
	786
	787	if ((vma->vm_flags & VM_EXEC) == 0)
	788	return;
	789
	790	preempt_disable();
	791
	792	if (mm == current->active_mm) {
	793	__load_new_mm_context(mm);
	794	if (atomic_read(&mm->mm_users) <= 1) {
	795	int cpu, this_cpu = smp_processor_id();
	796	for (cpu = 0; cpu < NR_CPUS; cpu++) {
	797	if (!cpu_online(cpu) \|\| cpu == this_cpu)
	798	continue;
	799	if (mm->context[cpu])
	800	mm->context[cpu] = 0;
	801	}
	802	preempt_enable();
	803	return;
	804	}
	805	}
	806
8691e5a8	807	if (smp_call_function(ipi_flush_icache_page, mm, 1)) {
1da177e4 LT	808	printk(KERN_CRIT "flush_icache_page: timed out\n");
	809	}
	810
	811	preempt_enable();
	812	}