[linux-block.git] / arch / powerpc / mm / hugetlbpage-hash64.c

/*
 * PPC64 Huge TLB Page Support for hash based MMUs (POWER4 and later)
 *
 * Copyright (C) 2003 David Gibson, IBM Corporation.
 *
 * Based on the IA-32 version:
 * Copyright (C) 2002, Rohit Seth <rohit.seth@intel.com>
 */

#include <linux/mm.h>
#include <linux/hugetlb.h>
#include <asm/pgtable.h>
#include <asm/pgalloc.h>
#include <asm/cacheflush.h>
#include <asm/machdep.h>

int __hash_page_huge(unsigned long ea, unsigned long access, unsigned long vsid,
		     pte_t *ptep, unsigned long trap, int local, int ssize,
		     unsigned int shift, unsigned int mmu_psize)
{
	unsigned long vpn;
	unsigned long old_pte, new_pte;
	unsigned long rflags, pa, sz;
	long slot;

	BUG_ON(shift != mmu_psize_defs[mmu_psize].shift);

	/* Search the Linux page table for a match with va */
	vpn = hpt_vpn(ea, vsid, ssize);

	/* At this point, we have a pte (old_pte) which can be used to build
	 * or update an HPTE. There are 2 cases:
	 *
	 * 1. There is a valid (present) pte with no associated HPTE (this is
	 *	the most common case)
	 * 2. There is a valid (present) pte with an associated HPTE. The
	 *	current values of the pp bits in the HPTE prevent access
	 *	because we are doing software DIRTY bit management and the
	 *	page is currently not DIRTY.
	 */


	do {
		old_pte = pte_val(*ptep);
		/* If PTE busy, retry the access */
		if (unlikely(old_pte & _PAGE_BUSY))
			return 0;
		/* If PTE permissions don't match, take page fault */
		if (unlikely(access & ~old_pte))
			return 1;
		/* Try to lock the PTE, add ACCESSED and DIRTY if it was
		 * a write access */
		new_pte = old_pte | _PAGE_BUSY | _PAGE_ACCESSED;
		if (access & _PAGE_RW)
			new_pte |= _PAGE_DIRTY;
	} while(old_pte != __cmpxchg_u64((unsigned long *)ptep,
					 old_pte, new_pte));

	rflags = 0x2 | (!(new_pte & _PAGE_RW));
	/* _PAGE_EXEC -> HW_NO_EXEC since it's inverted */
	rflags |= ((new_pte & _PAGE_EXEC) ? 0 : HPTE_R_N);
	sz = ((1UL) << shift);
	if (!cpu_has_feature(CPU_FTR_COHERENT_ICACHE))
		/* No CPU has hugepages but lacks no execute, so we
		 * don't need to worry about that case */
		rflags = hash_page_do_lazy_icache(rflags, __pte(old_pte), trap);

	/* Check if pte already has an hpte (case 2) */
	if (unlikely(old_pte & _PAGE_HASHPTE)) {
		/* There MIGHT be an HPTE for this pte */
		unsigned long hash, slot;

		hash = hpt_hash(vpn, shift, ssize);
		if (old_pte & _PAGE_F_SECOND)
			hash = ~hash;
		slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
		slot += (old_pte & _PAGE_F_GIX) >> 12;

		if (ppc_md.hpte_updatepp(slot, rflags, vpn, mmu_psize,
					 ssize, local) == -1)
			old_pte &= ~_PAGE_HPTEFLAGS;
	}

	if (likely(!(old_pte & _PAGE_HASHPTE))) {
		unsigned long hash = hpt_hash(vpn, shift, ssize);
		unsigned long hpte_group;

		pa = pte_pfn(__pte(old_pte)) << PAGE_SHIFT;

repeat:
		hpte_group = ((hash & htab_hash_mask) *
			      HPTES_PER_GROUP) & ~0x7UL;

		/* clear HPTE slot informations in new PTE */
#ifdef CONFIG_PPC_64K_PAGES
		new_pte = (new_pte & ~_PAGE_HPTEFLAGS) | _PAGE_HPTE_SUB0;
#else
		new_pte = (new_pte & ~_PAGE_HPTEFLAGS) | _PAGE_HASHPTE;
#endif
		/* Add in WIMG bits */
		rflags |= (new_pte & (_PAGE_WRITETHRU | _PAGE_NO_CACHE |
				      _PAGE_COHERENT | _PAGE_GUARDED));

		/* Insert into the hash table, primary slot */
		slot = ppc_md.hpte_insert(hpte_group, vpn, pa, rflags, 0,
					  mmu_psize, ssize);

		/* Primary is full, try the secondary */
		if (unlikely(slot == -1)) {
			hpte_group = ((~hash & htab_hash_mask) *
				      HPTES_PER_GROUP) & ~0x7UL;
			slot = ppc_md.hpte_insert(hpte_group, vpn, pa, rflags,
						  HPTE_V_SECONDARY,
						  mmu_psize, ssize);
			if (slot == -1) {
				if (mftb() & 0x1)
					hpte_group = ((hash & htab_hash_mask) *
						      HPTES_PER_GROUP)&~0x7UL;

				ppc_md.hpte_remove(hpte_group);
				goto repeat;
                        }
		}

		/*
		 * Hypervisor failure. Restore old pte and return -1
		 * similar to __hash_page_*
		 */
		if (unlikely(slot == -2)) {
			*ptep = __pte(old_pte);
			hash_failure_debug(ea, access, vsid, trap, ssize,
					   mmu_psize, old_pte);
			return -1;
		}

		new_pte |= (slot << 12) & (_PAGE_F_SECOND | _PAGE_F_GIX);
	}

	/*
	 * No need to use ldarx/stdcx here
	 */
	*ptep = __pte(new_pte & ~_PAGE_BUSY);
	return 0;
}
Commit	Line	Data
883a3e52 DG	1	/*
	2	* PPC64 Huge TLB Page Support for hash based MMUs (POWER4 and later)
	3	*
	4	* Copyright (C) 2003 David Gibson, IBM Corporation.
	5	*
	6	* Based on the IA-32 version:
	7	* Copyright (C) 2002, Rohit Seth <rohit.seth@intel.com>
	8	*/
	9
	10	#include <linux/mm.h>
	11	#include <linux/hugetlb.h>
	12	#include <asm/pgtable.h>
	13	#include <asm/pgalloc.h>
	14	#include <asm/cacheflush.h>
	15	#include <asm/machdep.h>
	16
883a3e52 DG	17	int __hash_page_huge(unsigned long ea, unsigned long access, unsigned long vsid,
	18	pte_t *ptep, unsigned long trap, int local, int ssize,
	19	unsigned int shift, unsigned int mmu_psize)
	20	{
5524a27d	21	unsigned long vpn;
883a3e52	22	unsigned long old_pte, new_pte;
5524a27d	23	unsigned long rflags, pa, sz;
883a3e52	24	long slot;
883a3e52 DG	25
	26	BUG_ON(shift != mmu_psize_defs[mmu_psize].shift);
	27
	28	/* Search the Linux page table for a match with va */
5524a27d	29	vpn = hpt_vpn(ea, vsid, ssize);
883a3e52	30
171aa2ca	31	/* At this point, we have a pte (old_pte) which can be used to build
883a3e52 DG	32	* or update an HPTE. There are 2 cases:
	33	*
	34	* 1. There is a valid (present) pte with no associated HPTE (this is
	35	* the most common case)
	36	* 2. There is a valid (present) pte with an associated HPTE. The
	37	* current values of the pp bits in the HPTE prevent access
	38	* because we are doing software DIRTY bit management and the
	39	* page is currently not DIRTY.
	40	*/
	41
	42
	43	do {
	44	old_pte = pte_val(*ptep);
171aa2ca BH	45	/* If PTE busy, retry the access */
	46	if (unlikely(old_pte & _PAGE_BUSY))
	47	return 0;
	48	/* If PTE permissions don't match, take page fault */
	49	if (unlikely(access & ~old_pte))
	50	return 1;
	51	/* Try to lock the PTE, add ACCESSED and DIRTY if it was
	52	* a write access */
883a3e52	53	new_pte = old_pte \| _PAGE_BUSY \| _PAGE_ACCESSED;
171aa2ca BH	54	if (access & _PAGE_RW)
171aa2ca BH	55	new_pte \|= _PAGE_DIRTY;
883a3e52 DG	56	} while(old_pte != __cmpxchg_u64((unsigned long *)ptep,
	57	old_pte, new_pte));
	58
	59	rflags = 0x2 \| (!(new_pte & _PAGE_RW));
	60	/* _PAGE_EXEC -> HW_NO_EXEC since it's inverted */
	61	rflags \|= ((new_pte & _PAGE_EXEC) ? 0 : HPTE_R_N);
	62	sz = ((1UL) << shift);
	63	if (!cpu_has_feature(CPU_FTR_COHERENT_ICACHE))
	64	/* No CPU has hugepages but lacks no execute, so we
	65	* don't need to worry about that case */
0895ecda	66	rflags = hash_page_do_lazy_icache(rflags, __pte(old_pte), trap);
883a3e52 DG	67
	68	/* Check if pte already has an hpte (case 2) */
	69	if (unlikely(old_pte & _PAGE_HASHPTE)) {
	70	/* There MIGHT be an HPTE for this pte */
	71	unsigned long hash, slot;
	72
5524a27d	73	hash = hpt_hash(vpn, shift, ssize);
883a3e52 DG	74	if (old_pte & _PAGE_F_SECOND)
	75	hash = ~hash;
	76	slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
	77	slot += (old_pte & _PAGE_F_GIX) >> 12;
	78
5524a27d	79	if (ppc_md.hpte_updatepp(slot, rflags, vpn, mmu_psize,
883a3e52 DG	80	ssize, local) == -1)
	81	old_pte &= ~_PAGE_HPTEFLAGS;
	82	}
	83
	84	if (likely(!(old_pte & _PAGE_HASHPTE))) {
5524a27d	85	unsigned long hash = hpt_hash(vpn, shift, ssize);
883a3e52 DG	86	unsigned long hpte_group;
	87
	88	pa = pte_pfn(__pte(old_pte)) << PAGE_SHIFT;
	89
	90	repeat:
	91	hpte_group = ((hash & htab_hash_mask) *
	92	HPTES_PER_GROUP) & ~0x7UL;
	93
	94	/* clear HPTE slot informations in new PTE */
	95	#ifdef CONFIG_PPC_64K_PAGES
	96	new_pte = (new_pte & ~_PAGE_HPTEFLAGS) \| _PAGE_HPTE_SUB0;
	97	#else
	98	new_pte = (new_pte & ~_PAGE_HPTEFLAGS) \| _PAGE_HASHPTE;
	99	#endif
	100	/* Add in WIMG bits */
	101	rflags \|= (new_pte & (_PAGE_WRITETHRU \| _PAGE_NO_CACHE \|
	102	_PAGE_COHERENT \| _PAGE_GUARDED));
	103
	104	/* Insert into the hash table, primary slot */
5524a27d	105	slot = ppc_md.hpte_insert(hpte_group, vpn, pa, rflags, 0,
883a3e52 DG	106	mmu_psize, ssize);
	107
	108	/* Primary is full, try the secondary */
	109	if (unlikely(slot == -1)) {
	110	hpte_group = ((~hash & htab_hash_mask) *
	111	HPTES_PER_GROUP) & ~0x7UL;
5524a27d	112	slot = ppc_md.hpte_insert(hpte_group, vpn, pa, rflags,
883a3e52 DG	113	HPTE_V_SECONDARY,
	114	mmu_psize, ssize);
	115	if (slot == -1) {
	116	if (mftb() & 0x1)
	117	hpte_group = ((hash & htab_hash_mask) *
	118	HPTES_PER_GROUP)&~0x7UL;
	119
	120	ppc_md.hpte_remove(hpte_group);
	121	goto repeat;
	122	}
	123	}
	124
b1623e7e AB	125	/*
	126	* Hypervisor failure. Restore old pte and return -1
	127	* similar to __hash_page_*
	128	*/
	129	if (unlikely(slot == -2)) {
	130	*ptep = __pte(old_pte);
4b8692c0 BH	131	hash_failure_debug(ea, access, vsid, trap, ssize,
4b8692c0 BH	132	mmu_psize, old_pte);
171aa2ca	133	return -1;
b1623e7e	134	}
883a3e52 DG	135
	136	new_pte \|= (slot << 12) & (_PAGE_F_SECOND \| _PAGE_F_GIX);
	137	}
	138
	139	/*
	140	* No need to use ldarx/stdcx here
	141	*/
	142	*ptep = __pte(new_pte & ~_PAGE_BUSY);
171aa2ca	143	return 0;
883a3e52	144	}