[linux-block.git] / arch / powerpc / mm / hash64_64k.c

/*
 * Copyright IBM Corporation, 2015
 * Author Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of version 2 of the GNU Lesser General Public License
 * as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it would be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 *
 */

#include <linux/mm.h>
#include <asm/machdep.h>
#include <asm/mmu.h>
/*
 * index from 0 - 15
 */
bool __rpte_sub_valid(real_pte_t rpte, unsigned long index)
{
	unsigned long g_idx;
	unsigned long ptev = pte_val(rpte.pte);

	g_idx = (ptev & H_PAGE_COMBO_VALID) >> H_PAGE_F_GIX_SHIFT;
	index = index >> 2;
	if (g_idx & (0x1 << index))
		return true;
	else
		return false;
}
/*
 * index from 0 - 15
 */
static unsigned long mark_subptegroup_valid(unsigned long ptev, unsigned long index)
{
	unsigned long g_idx;

	if (!(ptev & H_PAGE_COMBO))
		return ptev;
	index = index >> 2;
	g_idx = 0x1 << index;

	return ptev | (g_idx << H_PAGE_F_GIX_SHIFT);
}

int __hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
		   pte_t *ptep, unsigned long trap, unsigned long flags,
		   int ssize, int subpg_prot)
{
	real_pte_t rpte;
	unsigned long *hidxp;
	unsigned long hpte_group;
	unsigned int subpg_index;
	unsigned long rflags, pa, hidx;
	unsigned long old_pte, new_pte, subpg_pte;
	unsigned long vpn, hash, slot;
	unsigned long shift = mmu_psize_defs[MMU_PAGE_4K].shift;

	/*
	 * atomically mark the linux large page PTE busy and dirty
	 */
	do {
		pte_t pte = READ_ONCE(*ptep);

		old_pte = pte_val(pte);
		/* If PTE busy, retry the access */
		if (unlikely(old_pte & H_PAGE_BUSY))
			return 0;
		/* If PTE permissions don't match, take page fault */
		if (unlikely(!check_pte_access(access, old_pte)))
			return 1;
		/*
		 * Try to lock the PTE, add ACCESSED and DIRTY if it was
		 * a write access. Since this is 4K insert of 64K page size
		 * also add H_PAGE_COMBO
		 */
		new_pte = old_pte | H_PAGE_BUSY | _PAGE_ACCESSED | H_PAGE_COMBO;
		if (access & _PAGE_WRITE)
			new_pte |= _PAGE_DIRTY;
	} while (!pte_xchg(ptep, __pte(old_pte), __pte(new_pte)));

	/*
	 * Handle the subpage protection bits
	 */
	subpg_pte = new_pte & ~subpg_prot;
	rflags = htab_convert_pte_flags(subpg_pte);

	if (!cpu_has_feature(CPU_FTR_NOEXECUTE) &&
	    !cpu_has_feature(CPU_FTR_COHERENT_ICACHE)) {

		/*
		 * No CPU has hugepages but lacks no execute, so we
		 * don't need to worry about that case
		 */
		rflags = hash_page_do_lazy_icache(rflags, __pte(old_pte), trap);
	}

	subpg_index = (ea & (PAGE_SIZE - 1)) >> shift;
	vpn  = hpt_vpn(ea, vsid, ssize);
	rpte = __real_pte(__pte(old_pte), ptep);
	/*
	 *None of the sub 4k page is hashed
	 */
	if (!(old_pte & H_PAGE_HASHPTE))
		goto htab_insert_hpte;
	/*
	 * Check if the pte was already inserted into the hash table
	 * as a 64k HW page, and invalidate the 64k HPTE if so.
	 */
	if (!(old_pte & H_PAGE_COMBO)) {
		flush_hash_page(vpn, rpte, MMU_PAGE_64K, ssize, flags);
		/*
		 * clear the old slot details from the old and new pte.
		 * On hash insert failure we use old pte value and we don't
		 * want slot information there if we have a insert failure.
		 */
		old_pte &= ~(H_PAGE_HASHPTE | H_PAGE_F_GIX | H_PAGE_F_SECOND);
		new_pte &= ~(H_PAGE_HASHPTE | H_PAGE_F_GIX | H_PAGE_F_SECOND);
		goto htab_insert_hpte;
	}
	/*
	 * Check for sub page valid and update
	 */
	if (__rpte_sub_valid(rpte, subpg_index)) {
		int ret;

		hash = hpt_hash(vpn, shift, ssize);
		hidx = __rpte_to_hidx(rpte, subpg_index);
		if (hidx & _PTEIDX_SECONDARY)
			hash = ~hash;
		slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
		slot += hidx & _PTEIDX_GROUP_IX;

		ret = ppc_md.hpte_updatepp(slot, rflags, vpn,
					   MMU_PAGE_4K, MMU_PAGE_4K,
					   ssize, flags);
		/*
		 *if we failed because typically the HPTE wasn't really here
		 * we try an insertion.
		 */
		if (ret == -1)
			goto htab_insert_hpte;

		*ptep = __pte(new_pte & ~H_PAGE_BUSY);
		return 0;
	}

htab_insert_hpte:
	/*
	 * handle H_PAGE_4K_PFN case
	 */
	if (old_pte & H_PAGE_4K_PFN) {
		/*
		 * All the sub 4k page have the same
		 * physical address.
		 */
		pa = pte_pfn(__pte(old_pte)) << HW_PAGE_SHIFT;
	} else {
		pa = pte_pfn(__pte(old_pte)) << PAGE_SHIFT;
		pa += (subpg_index << shift);
	}
	hash = hpt_hash(vpn, shift, ssize);
repeat:
	hpte_group = ((hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL;

	/* Insert into the hash table, primary slot */
	slot = ppc_md.hpte_insert(hpte_group, vpn, pa, rflags, 0,
				  MMU_PAGE_4K, MMU_PAGE_4K, ssize);
	/*
	 * Primary is full, try the secondary
	 */
	if (unlikely(slot == -1)) {
		hpte_group = ((~hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL;
		slot = ppc_md.hpte_insert(hpte_group, vpn, pa,
					  rflags, HPTE_V_SECONDARY,
					  MMU_PAGE_4K, MMU_PAGE_4K, ssize);
		if (slot == -1) {
			if (mftb() & 0x1)
				hpte_group = ((hash & htab_hash_mask) *
					      HPTES_PER_GROUP) & ~0x7UL;
			ppc_md.hpte_remove(hpte_group);
			/*
			 * FIXME!! Should be try the group from which we removed ?
			 */
			goto repeat;
		}
	}
	/*
	 * Hypervisor failure. Restore old pte and return -1
	 * similar to __hash_page_*
	 */
	if (unlikely(slot == -2)) {
		*ptep = __pte(old_pte);
		hash_failure_debug(ea, access, vsid, trap, ssize,
				   MMU_PAGE_4K, MMU_PAGE_4K, old_pte);
		return -1;
	}
	/*
	 * Insert slot number & secondary bit in PTE second half,
	 * clear H_PAGE_BUSY and set appropriate HPTE slot bit
	 * Since we have H_PAGE_BUSY set on ptep, we can be sure
	 * nobody is undating hidx.
	 */
	hidxp = (unsigned long *)(ptep + PTRS_PER_PTE);
	rpte.hidx &= ~(0xfUL << (subpg_index << 2));
	*hidxp = rpte.hidx  | (slot << (subpg_index << 2));
	new_pte = mark_subptegroup_valid(new_pte, subpg_index);
	new_pte |=  H_PAGE_HASHPTE;
	/*
	 * check __real_pte for details on matching smp_rmb()
	 */
	smp_wmb();
	*ptep = __pte(new_pte & ~H_PAGE_BUSY);
	return 0;
}

int __hash_page_64K(unsigned long ea, unsigned long access,
		    unsigned long vsid, pte_t *ptep, unsigned long trap,
		    unsigned long flags, int ssize)
{
	unsigned long hpte_group;
	unsigned long rflags, pa;
	unsigned long old_pte, new_pte;
	unsigned long vpn, hash, slot;
	unsigned long shift = mmu_psize_defs[MMU_PAGE_64K].shift;

	/*
	 * atomically mark the linux large page PTE busy and dirty
	 */
	do {
		pte_t pte = READ_ONCE(*ptep);

		old_pte = pte_val(pte);
		/* If PTE busy, retry the access */
		if (unlikely(old_pte & H_PAGE_BUSY))
			return 0;
		/* If PTE permissions don't match, take page fault */
		if (unlikely(!check_pte_access(access, old_pte)))
			return 1;
		/*
		 * Check if PTE has the cache-inhibit bit set
		 * If so, bail out and refault as a 4k page
		 */
		if (!mmu_has_feature(MMU_FTR_CI_LARGE_PAGE) &&
		    unlikely(pte_ci(pte)))
			return 0;
		/*
		 * Try to lock the PTE, add ACCESSED and DIRTY if it was
		 * a write access.
		 */
		new_pte = old_pte | H_PAGE_BUSY | _PAGE_ACCESSED;
		if (access & _PAGE_WRITE)
			new_pte |= _PAGE_DIRTY;
	} while (!pte_xchg(ptep, __pte(old_pte), __pte(new_pte)));

	rflags = htab_convert_pte_flags(new_pte);

	if (!cpu_has_feature(CPU_FTR_NOEXECUTE) &&
	    !cpu_has_feature(CPU_FTR_COHERENT_ICACHE))
		rflags = hash_page_do_lazy_icache(rflags, __pte(old_pte), trap);

	vpn  = hpt_vpn(ea, vsid, ssize);
	if (unlikely(old_pte & H_PAGE_HASHPTE)) {
		/*
		 * There MIGHT be an HPTE for this pte
		 */
		hash = hpt_hash(vpn, shift, ssize);
		if (old_pte & H_PAGE_F_SECOND)
			hash = ~hash;
		slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
		slot += (old_pte & H_PAGE_F_GIX) >> H_PAGE_F_GIX_SHIFT;

		if (ppc_md.hpte_updatepp(slot, rflags, vpn, MMU_PAGE_64K,
					 MMU_PAGE_64K, ssize, flags) == -1)
			old_pte &= ~_PAGE_HPTEFLAGS;
	}

	if (likely(!(old_pte & H_PAGE_HASHPTE))) {

		pa = pte_pfn(__pte(old_pte)) << PAGE_SHIFT;
		hash = hpt_hash(vpn, shift, ssize);

repeat:
		hpte_group = ((hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL;

		/* Insert into the hash table, primary slot */
		slot = ppc_md.hpte_insert(hpte_group, vpn, pa, rflags, 0,
				  MMU_PAGE_64K, MMU_PAGE_64K, ssize);
		/*
		 * Primary is full, try the secondary
		 */
		if (unlikely(slot == -1)) {
			hpte_group = ((~hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL;
			slot = ppc_md.hpte_insert(hpte_group, vpn, pa,
						  rflags, HPTE_V_SECONDARY,
						  MMU_PAGE_64K, MMU_PAGE_64K, ssize);
			if (slot == -1) {
				if (mftb() & 0x1)
					hpte_group = ((hash & htab_hash_mask) *
						      HPTES_PER_GROUP) & ~0x7UL;
				ppc_md.hpte_remove(hpte_group);
				/*
				 * FIXME!! Should be try the group from which we removed ?
				 */
				goto repeat;
			}
		}
		/*
		 * Hypervisor failure. Restore old pte and return -1
		 * similar to __hash_page_*
		 */
		if (unlikely(slot == -2)) {
			*ptep = __pte(old_pte);
			hash_failure_debug(ea, access, vsid, trap, ssize,
					   MMU_PAGE_64K, MMU_PAGE_64K, old_pte);
			return -1;
		}
		new_pte = (new_pte & ~_PAGE_HPTEFLAGS) | H_PAGE_HASHPTE;
		new_pte |= (slot << H_PAGE_F_GIX_SHIFT) &
			(H_PAGE_F_SECOND | H_PAGE_F_GIX);
	}
	*ptep = __pte(new_pte & ~H_PAGE_BUSY);
	return 0;
}
Commit	Line	Data
91f1da99 AK	1	/*
	2	* Copyright IBM Corporation, 2015
	3	* Author Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
	4	*
	5	* This program is free software; you can redistribute it and/or modify it
	6	* under the terms of version 2 of the GNU Lesser General Public License
	7	* as published by the Free Software Foundation.
	8	*
	9	* This program is distributed in the hope that it would be useful, but
	10	* WITHOUT ANY WARRANTY; without even the implied warranty of
	11	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
	12	*
	13	*/
	14
	15	#include <linux/mm.h>
	16	#include <asm/machdep.h>
	17	#include <asm/mmu.h>
bf680d51 AK	18	/*
	19	* index from 0 - 15
	20	*/
	21	bool __rpte_sub_valid(real_pte_t rpte, unsigned long index)
	22	{
	23	unsigned long g_idx;
	24	unsigned long ptev = pte_val(rpte.pte);
	25
945537df	26	g_idx = (ptev & H_PAGE_COMBO_VALID) >> H_PAGE_F_GIX_SHIFT;
bf680d51 AK	27	index = index >> 2;
	28	if (g_idx & (0x1 << index))
	29	return true;
	30	else
	31	return false;
	32	}
	33	/*
	34	* index from 0 - 15
	35	*/
	36	static unsigned long mark_subptegroup_valid(unsigned long ptev, unsigned long index)
	37	{
	38	unsigned long g_idx;
	39
945537df	40	if (!(ptev & H_PAGE_COMBO))
bf680d51 AK	41	return ptev;
	42	index = index >> 2;
	43	g_idx = 0x1 << index;
	44
945537df	45	return ptev \| (g_idx << H_PAGE_F_GIX_SHIFT);
bf680d51	46	}
91f1da99 AK	47
	48	int __hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
	49	pte_t *ptep, unsigned long trap, unsigned long flags,
	50	int ssize, int subpg_prot)
	51	{
	52	real_pte_t rpte;
	53	unsigned long *hidxp;
	54	unsigned long hpte_group;
	55	unsigned int subpg_index;
	56	unsigned long rflags, pa, hidx;
	57	unsigned long old_pte, new_pte, subpg_pte;
	58	unsigned long vpn, hash, slot;
	59	unsigned long shift = mmu_psize_defs[MMU_PAGE_4K].shift;
	60
	61	/*
	62	* atomically mark the linux large page PTE busy and dirty
	63	*/
	64	do {
	65	pte_t pte = READ_ONCE(*ptep);
	66
	67	old_pte = pte_val(pte);
	68	/* If PTE busy, retry the access */
945537df	69	if (unlikely(old_pte & H_PAGE_BUSY))
91f1da99 AK	70	return 0;
91f1da99 AK	71	/* If PTE permissions don't match, take page fault */
ac29c640	72	if (unlikely(!check_pte_access(access, old_pte)))
91f1da99 AK	73	return 1;
	74	/*
	75	* Try to lock the PTE, add ACCESSED and DIRTY if it was
	76	* a write access. Since this is 4K insert of 64K page size
945537df	77	* also add H_PAGE_COMBO
91f1da99	78	*/
945537df	79	new_pte = old_pte \| H_PAGE_BUSY \| _PAGE_ACCESSED \| H_PAGE_COMBO;
c7d54842	80	if (access & _PAGE_WRITE)
91f1da99	81	new_pte \|= _PAGE_DIRTY;
3910a7f4 ME	82	} while (!pte_xchg(ptep, __pte(old_pte), __pte(new_pte)));
3910a7f4 ME	83
91f1da99 AK	84	/*
	85	* Handle the subpage protection bits
	86	*/
	87	subpg_pte = new_pte & ~subpg_prot;
c6a3c495	88	rflags = htab_convert_pte_flags(subpg_pte);
91f1da99 AK	89
	90	if (!cpu_has_feature(CPU_FTR_NOEXECUTE) &&
	91	!cpu_has_feature(CPU_FTR_COHERENT_ICACHE)) {
	92
	93	/*
	94	* No CPU has hugepages but lacks no execute, so we
	95	* don't need to worry about that case
	96	*/
	97	rflags = hash_page_do_lazy_icache(rflags, __pte(old_pte), trap);
	98	}
	99
	100	subpg_index = (ea & (PAGE_SIZE - 1)) >> shift;
	101	vpn = hpt_vpn(ea, vsid, ssize);
	102	rpte = __real_pte(__pte(old_pte), ptep);
	103	/*
	104	*None of the sub 4k page is hashed
	105	*/
945537df	106	if (!(old_pte & H_PAGE_HASHPTE))
91f1da99 AK	107	goto htab_insert_hpte;
	108	/*
	109	* Check if the pte was already inserted into the hash table
	110	* as a 64k HW page, and invalidate the 64k HPTE if so.
	111	*/
945537df	112	if (!(old_pte & H_PAGE_COMBO)) {
91f1da99	113	flush_hash_page(vpn, rpte, MMU_PAGE_64K, ssize, flags);
9ab3ac23 AK	114	/*
	115	* clear the old slot details from the old and new pte.
	116	* On hash insert failure we use old pte value and we don't
	117	* want slot information there if we have a insert failure.
	118	*/
945537df AK	119	old_pte &= ~(H_PAGE_HASHPTE \| H_PAGE_F_GIX \| H_PAGE_F_SECOND);
945537df AK	120	new_pte &= ~(H_PAGE_HASHPTE \| H_PAGE_F_GIX \| H_PAGE_F_SECOND);
91f1da99 AK	121	goto htab_insert_hpte;
	122	}
	123	/*
	124	* Check for sub page valid and update
	125	*/
	126	if (__rpte_sub_valid(rpte, subpg_index)) {
	127	int ret;
	128
	129	hash = hpt_hash(vpn, shift, ssize);
	130	hidx = __rpte_to_hidx(rpte, subpg_index);
	131	if (hidx & _PTEIDX_SECONDARY)
	132	hash = ~hash;
	133	slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
	134	slot += hidx & _PTEIDX_GROUP_IX;
	135
	136	ret = ppc_md.hpte_updatepp(slot, rflags, vpn,
	137	MMU_PAGE_4K, MMU_PAGE_4K,
	138	ssize, flags);
	139	/*
	140	*if we failed because typically the HPTE wasn't really here
	141	* we try an insertion.
	142	*/
	143	if (ret == -1)
	144	goto htab_insert_hpte;
	145
945537df	146	*ptep = __pte(new_pte & ~H_PAGE_BUSY);
91f1da99 AK	147	return 0;
	148	}
	149
	150	htab_insert_hpte:
	151	/*
945537df	152	* handle H_PAGE_4K_PFN case
91f1da99	153	*/
945537df	154	if (old_pte & H_PAGE_4K_PFN) {
91f1da99 AK	155	/*
	156	* All the sub 4k page have the same
	157	* physical address.
	158	*/
	159	pa = pte_pfn(__pte(old_pte)) << HW_PAGE_SHIFT;
	160	} else {
	161	pa = pte_pfn(__pte(old_pte)) << PAGE_SHIFT;
	162	pa += (subpg_index << shift);
	163	}
	164	hash = hpt_hash(vpn, shift, ssize);
	165	repeat:
	166	hpte_group = ((hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL;
	167
	168	/* Insert into the hash table, primary slot */
	169	slot = ppc_md.hpte_insert(hpte_group, vpn, pa, rflags, 0,
	170	MMU_PAGE_4K, MMU_PAGE_4K, ssize);
	171	/*
	172	* Primary is full, try the secondary
	173	*/
	174	if (unlikely(slot == -1)) {
	175	hpte_group = ((~hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL;
	176	slot = ppc_md.hpte_insert(hpte_group, vpn, pa,
	177	rflags, HPTE_V_SECONDARY,
	178	MMU_PAGE_4K, MMU_PAGE_4K, ssize);
	179	if (slot == -1) {
	180	if (mftb() & 0x1)
	181	hpte_group = ((hash & htab_hash_mask) *
	182	HPTES_PER_GROUP) & ~0x7UL;
	183	ppc_md.hpte_remove(hpte_group);
	184	/*
	185	* FIXME!! Should be try the group from which we removed ?
	186	*/
	187	goto repeat;
	188	}
	189	}
	190	/*
e9a68147	191	* Hypervisor failure. Restore old pte and return -1
91f1da99 AK	192	* similar to __hash_page_*
	193	*/
	194	if (unlikely(slot == -2)) {
	195	*ptep = __pte(old_pte);
	196	hash_failure_debug(ea, access, vsid, trap, ssize,
	197	MMU_PAGE_4K, MMU_PAGE_4K, old_pte);
	198	return -1;
	199	}
	200	/*
	201	* Insert slot number & secondary bit in PTE second half,
945537df AK	202	* clear H_PAGE_BUSY and set appropriate HPTE slot bit
945537df AK	203	* Since we have H_PAGE_BUSY set on ptep, we can be sure
91f1da99 AK	204	* nobody is undating hidx.
	205	*/
	206	hidxp = (unsigned long *)(ptep + PTRS_PER_PTE);
91f1da99 AK	207	rpte.hidx &= ~(0xfUL << (subpg_index << 2));
91f1da99 AK	208	*hidxp = rpte.hidx \| (slot << (subpg_index << 2));
bf680d51	209	new_pte = mark_subptegroup_valid(new_pte, subpg_index);
945537df	210	new_pte \|= H_PAGE_HASHPTE;
91f1da99 AK	211	/*
	212	* check __real_pte for details on matching smp_rmb()
	213	*/
	214	smp_wmb();
945537df	215	*ptep = __pte(new_pte & ~H_PAGE_BUSY);
91f1da99 AK	216	return 0;
91f1da99 AK	217	}
89ff7250 AK	218
	219	int __hash_page_64K(unsigned long ea, unsigned long access,
	220	unsigned long vsid, pte_t *ptep, unsigned long trap,
	221	unsigned long flags, int ssize)
	222	{
89ff7250 AK	223	unsigned long hpte_group;
	224	unsigned long rflags, pa;
	225	unsigned long old_pte, new_pte;
	226	unsigned long vpn, hash, slot;
	227	unsigned long shift = mmu_psize_defs[MMU_PAGE_64K].shift;
	228
	229	/*
	230	* atomically mark the linux large page PTE busy and dirty
	231	*/
	232	do {
	233	pte_t pte = READ_ONCE(*ptep);
	234
	235	old_pte = pte_val(pte);
	236	/* If PTE busy, retry the access */
945537df	237	if (unlikely(old_pte & H_PAGE_BUSY))
89ff7250 AK	238	return 0;
89ff7250 AK	239	/* If PTE permissions don't match, take page fault */
ac29c640	240	if (unlikely(!check_pte_access(access, old_pte)))
89ff7250 AK	241	return 1;
	242	/*
	243	* Check if PTE has the cache-inhibit bit set
	244	* If so, bail out and refault as a 4k page
	245	*/
	246	if (!mmu_has_feature(MMU_FTR_CI_LARGE_PAGE) &&
30bda41a	247	unlikely(pte_ci(pte)))
89ff7250 AK	248	return 0;
	249	/*
	250	* Try to lock the PTE, add ACCESSED and DIRTY if it was
1ec3f937	251	* a write access.
89ff7250	252	*/
945537df	253	new_pte = old_pte \| H_PAGE_BUSY \| _PAGE_ACCESSED;
c7d54842	254	if (access & _PAGE_WRITE)
89ff7250	255	new_pte \|= _PAGE_DIRTY;
3910a7f4	256	} while (!pte_xchg(ptep, __pte(old_pte), __pte(new_pte)));
c6a3c495 AK	257
c6a3c495 AK	258	rflags = htab_convert_pte_flags(new_pte);
89ff7250 AK	259
	260	if (!cpu_has_feature(CPU_FTR_NOEXECUTE) &&
	261	!cpu_has_feature(CPU_FTR_COHERENT_ICACHE))
	262	rflags = hash_page_do_lazy_icache(rflags, __pte(old_pte), trap);
	263
	264	vpn = hpt_vpn(ea, vsid, ssize);
945537df	265	if (unlikely(old_pte & H_PAGE_HASHPTE)) {
89ff7250 AK	266	/*
	267	* There MIGHT be an HPTE for this pte
	268	*/
	269	hash = hpt_hash(vpn, shift, ssize);
945537df	270	if (old_pte & H_PAGE_F_SECOND)
89ff7250 AK	271	hash = ~hash;
89ff7250 AK	272	slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
945537df	273	slot += (old_pte & H_PAGE_F_GIX) >> H_PAGE_F_GIX_SHIFT;
89ff7250 AK	274
	275	if (ppc_md.hpte_updatepp(slot, rflags, vpn, MMU_PAGE_64K,
	276	MMU_PAGE_64K, ssize, flags) == -1)
	277	old_pte &= ~_PAGE_HPTEFLAGS;
	278	}
	279
945537df	280	if (likely(!(old_pte & H_PAGE_HASHPTE))) {
89ff7250 AK	281
	282	pa = pte_pfn(__pte(old_pte)) << PAGE_SHIFT;
	283	hash = hpt_hash(vpn, shift, ssize);
	284
	285	repeat:
	286	hpte_group = ((hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL;
	287
	288	/* Insert into the hash table, primary slot */
	289	slot = ppc_md.hpte_insert(hpte_group, vpn, pa, rflags, 0,
	290	MMU_PAGE_64K, MMU_PAGE_64K, ssize);
	291	/*
	292	* Primary is full, try the secondary
	293	*/
	294	if (unlikely(slot == -1)) {
	295	hpte_group = ((~hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL;
	296	slot = ppc_md.hpte_insert(hpte_group, vpn, pa,
	297	rflags, HPTE_V_SECONDARY,
	298	MMU_PAGE_64K, MMU_PAGE_64K, ssize);
	299	if (slot == -1) {
	300	if (mftb() & 0x1)
	301	hpte_group = ((hash & htab_hash_mask) *
	302	HPTES_PER_GROUP) & ~0x7UL;
	303	ppc_md.hpte_remove(hpte_group);
	304	/*
	305	* FIXME!! Should be try the group from which we removed ?
	306	*/
	307	goto repeat;
	308	}
	309	}
	310	/*
e9a68147	311	* Hypervisor failure. Restore old pte and return -1
89ff7250 AK	312	* similar to __hash_page_*
	313	*/
	314	if (unlikely(slot == -2)) {
	315	*ptep = __pte(old_pte);
	316	hash_failure_debug(ea, access, vsid, trap, ssize,
	317	MMU_PAGE_64K, MMU_PAGE_64K, old_pte);
	318	return -1;
	319	}
945537df AK	320	new_pte = (new_pte & ~_PAGE_HPTEFLAGS) \| H_PAGE_HASHPTE;
	321	new_pte \|= (slot << H_PAGE_F_GIX_SHIFT) &
	322	(H_PAGE_F_SECOND \| H_PAGE_F_GIX);
89ff7250	323	}
945537df	324	*ptep = __pte(new_pte & ~H_PAGE_BUSY);
89ff7250 AK	325	return 0;
89ff7250 AK	326	}