[linux-block.git] / arch / powerpc / mm / book3s64 / hash_64k.c

/*
 * Copyright IBM Corporation, 2015
 * Author Aneesh Kumar K.V <aneesh.kumar@linux.ibm.com>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of version 2 of the GNU Lesser General Public License
 * as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it would be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 *
 */

#include <linux/mm.h>
#include <asm/machdep.h>
#include <asm/mmu.h>

/*
 * Return true, if the entry has a slot value which
 * the software considers as invalid.
 */
static inline bool hpte_soft_invalid(unsigned long hidx)
{
	return ((hidx & 0xfUL) == 0xfUL);
}

/*
 * index from 0 - 15
 */
bool __rpte_sub_valid(real_pte_t rpte, unsigned long index)
{
	return !(hpte_soft_invalid(__rpte_to_hidx(rpte, index)));
}

int __hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
		   pte_t *ptep, unsigned long trap, unsigned long flags,
		   int ssize, int subpg_prot)
{
	real_pte_t rpte;
	unsigned long hpte_group;
	unsigned int subpg_index;
	unsigned long rflags, pa;
	unsigned long old_pte, new_pte, subpg_pte;
	unsigned long vpn, hash, slot, gslot;
	unsigned long shift = mmu_psize_defs[MMU_PAGE_4K].shift;

	/*
	 * atomically mark the linux large page PTE busy and dirty
	 */
	do {
		pte_t pte = READ_ONCE(*ptep);

		old_pte = pte_val(pte);
		/* If PTE busy, retry the access */
		if (unlikely(old_pte & H_PAGE_BUSY))
			return 0;
		/* If PTE permissions don't match, take page fault */
		if (unlikely(!check_pte_access(access, old_pte)))
			return 1;
		/*
		 * Try to lock the PTE, add ACCESSED and DIRTY if it was
		 * a write access. Since this is 4K insert of 64K page size
		 * also add H_PAGE_COMBO
		 */
		new_pte = old_pte | H_PAGE_BUSY | _PAGE_ACCESSED | H_PAGE_COMBO;
		if (access & _PAGE_WRITE)
			new_pte |= _PAGE_DIRTY;
	} while (!pte_xchg(ptep, __pte(old_pte), __pte(new_pte)));

	/*
	 * Handle the subpage protection bits
	 */
	subpg_pte = new_pte & ~subpg_prot;
	rflags = htab_convert_pte_flags(subpg_pte, flags);

	if (cpu_has_feature(CPU_FTR_NOEXECUTE) &&
	    !cpu_has_feature(CPU_FTR_COHERENT_ICACHE)) {

		/*
		 * No CPU has hugepages but lacks no execute, so we
		 * don't need to worry about that case
		 */
		rflags = hash_page_do_lazy_icache(rflags, __pte(old_pte), trap);
	}

	subpg_index = (ea & (PAGE_SIZE - 1)) >> shift;
	vpn  = hpt_vpn(ea, vsid, ssize);
	rpte = __real_pte(__pte(old_pte), ptep, PTRS_PER_PTE);
	/*
	 *None of the sub 4k page is hashed
	 */
	if (!(old_pte & H_PAGE_HASHPTE))
		goto htab_insert_hpte;
	/*
	 * Check if the pte was already inserted into the hash table
	 * as a 64k HW page, and invalidate the 64k HPTE if so.
	 */
	if (!(old_pte & H_PAGE_COMBO)) {
		flush_hash_page(vpn, rpte, MMU_PAGE_64K, ssize, flags);
		/*
		 * clear the old slot details from the old and new pte.
		 * On hash insert failure we use old pte value and we don't
		 * want slot information there if we have a insert failure.
		 */
		old_pte &= ~H_PAGE_HASHPTE;
		new_pte &= ~H_PAGE_HASHPTE;
		goto htab_insert_hpte;
	}
	/*
	 * Check for sub page valid and update
	 */
	if (__rpte_sub_valid(rpte, subpg_index)) {
		int ret;

		gslot = pte_get_hash_gslot(vpn, shift, ssize, rpte,
					   subpg_index);
		ret = mmu_hash_ops.hpte_updatepp(gslot, rflags, vpn,
						 MMU_PAGE_4K, MMU_PAGE_4K,
						 ssize, flags);

		/*
		 * If we failed because typically the HPTE wasn't really here
		 * we try an insertion.
		 */
		if (ret == -1)
			goto htab_insert_hpte;

		*ptep = __pte(new_pte & ~H_PAGE_BUSY);
		return 0;
	}

htab_insert_hpte:

	/*
	 * Initialize all hidx entries to invalid value, the first time
	 * the PTE is about to allocate a 4K HPTE.
	 */
	if (!(old_pte & H_PAGE_COMBO))
		rpte.hidx = INVALID_RPTE_HIDX;

	/*
	 * handle H_PAGE_4K_PFN case
	 */
	if (old_pte & H_PAGE_4K_PFN) {
		/*
		 * All the sub 4k page have the same
		 * physical address.
		 */
		pa = pte_pfn(__pte(old_pte)) << HW_PAGE_SHIFT;
	} else {
		pa = pte_pfn(__pte(old_pte)) << PAGE_SHIFT;
		pa += (subpg_index << shift);
	}
	hash = hpt_hash(vpn, shift, ssize);
repeat:
	hpte_group = (hash & htab_hash_mask) * HPTES_PER_GROUP;

	/* Insert into the hash table, primary slot */
	slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa, rflags, 0,
					MMU_PAGE_4K, MMU_PAGE_4K, ssize);
	/*
	 * Primary is full, try the secondary
	 */
	if (unlikely(slot == -1)) {
		bool soft_invalid;

		hpte_group = (~hash & htab_hash_mask) * HPTES_PER_GROUP;
		slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa,
						rflags, HPTE_V_SECONDARY,
						MMU_PAGE_4K, MMU_PAGE_4K,
						ssize);

		soft_invalid = hpte_soft_invalid(slot);
		if (unlikely(soft_invalid)) {
			/*
			 * We got a valid slot from a hardware point of view.
			 * but we cannot use it, because we use this special
			 * value; as defined by hpte_soft_invalid(), to track
			 * invalid slots. We cannot use it. So invalidate it.
			 */
			gslot = slot & _PTEIDX_GROUP_IX;
			mmu_hash_ops.hpte_invalidate(hpte_group + gslot, vpn,
						     MMU_PAGE_4K, MMU_PAGE_4K,
						     ssize, 0);
		}

		if (unlikely(slot == -1 || soft_invalid)) {
			/*
			 * For soft invalid slot, let's ensure that we release a
			 * slot from the primary, with the hope that we will
			 * acquire that slot next time we try. This will ensure
			 * that we do not get the same soft-invalid slot.
			 */
			if (soft_invalid || (mftb() & 0x1))
				hpte_group = (hash & htab_hash_mask) * HPTES_PER_GROUP;

			mmu_hash_ops.hpte_remove(hpte_group);
			/*
			 * FIXME!! Should be try the group from which we removed ?
			 */
			goto repeat;
		}
	}
	/*
	 * Hypervisor failure. Restore old pte and return -1
	 * similar to __hash_page_*
	 */
	if (unlikely(slot == -2)) {
		*ptep = __pte(old_pte);
		hash_failure_debug(ea, access, vsid, trap, ssize,
				   MMU_PAGE_4K, MMU_PAGE_4K, old_pte);
		return -1;
	}

	new_pte |= pte_set_hidx(ptep, rpte, subpg_index, slot, PTRS_PER_PTE);
	new_pte |= H_PAGE_HASHPTE;

	*ptep = __pte(new_pte & ~H_PAGE_BUSY);
	return 0;
}

int __hash_page_64K(unsigned long ea, unsigned long access,
		    unsigned long vsid, pte_t *ptep, unsigned long trap,
		    unsigned long flags, int ssize)
{
	real_pte_t rpte;
	unsigned long hpte_group;
	unsigned long rflags, pa;
	unsigned long old_pte, new_pte;
	unsigned long vpn, hash, slot;
	unsigned long shift = mmu_psize_defs[MMU_PAGE_64K].shift;

	/*
	 * atomically mark the linux large page PTE busy and dirty
	 */
	do {
		pte_t pte = READ_ONCE(*ptep);

		old_pte = pte_val(pte);
		/* If PTE busy, retry the access */
		if (unlikely(old_pte & H_PAGE_BUSY))
			return 0;
		/* If PTE permissions don't match, take page fault */
		if (unlikely(!check_pte_access(access, old_pte)))
			return 1;
		/*
		 * Check if PTE has the cache-inhibit bit set
		 * If so, bail out and refault as a 4k page
		 */
		if (!mmu_has_feature(MMU_FTR_CI_LARGE_PAGE) &&
		    unlikely(pte_ci(pte)))
			return 0;
		/*
		 * Try to lock the PTE, add ACCESSED and DIRTY if it was
		 * a write access.
		 */
		new_pte = old_pte | H_PAGE_BUSY | _PAGE_ACCESSED;
		if (access & _PAGE_WRITE)
			new_pte |= _PAGE_DIRTY;
	} while (!pte_xchg(ptep, __pte(old_pte), __pte(new_pte)));

	rflags = htab_convert_pte_flags(new_pte, flags);
	rpte = __real_pte(__pte(old_pte), ptep, PTRS_PER_PTE);

	if (cpu_has_feature(CPU_FTR_NOEXECUTE) &&
	    !cpu_has_feature(CPU_FTR_COHERENT_ICACHE))
		rflags = hash_page_do_lazy_icache(rflags, __pte(old_pte), trap);

	vpn  = hpt_vpn(ea, vsid, ssize);
	if (unlikely(old_pte & H_PAGE_HASHPTE)) {
		unsigned long gslot;

		/*
		 * There MIGHT be an HPTE for this pte
		 */
		gslot = pte_get_hash_gslot(vpn, shift, ssize, rpte, 0);
		if (mmu_hash_ops.hpte_updatepp(gslot, rflags, vpn, MMU_PAGE_64K,
					       MMU_PAGE_64K, ssize,
					       flags) == -1)
			old_pte &= ~_PAGE_HPTEFLAGS;
	}

	if (likely(!(old_pte & H_PAGE_HASHPTE))) {

		pa = pte_pfn(__pte(old_pte)) << PAGE_SHIFT;
		hash = hpt_hash(vpn, shift, ssize);

repeat:
		hpte_group = (hash & htab_hash_mask) * HPTES_PER_GROUP;

		/* Insert into the hash table, primary slot */
		slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa, rflags, 0,
						MMU_PAGE_64K, MMU_PAGE_64K,
						ssize);
		/*
		 * Primary is full, try the secondary
		 */
		if (unlikely(slot == -1)) {
			hpte_group = (~hash & htab_hash_mask) * HPTES_PER_GROUP;
			slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa,
							rflags,
							HPTE_V_SECONDARY,
							MMU_PAGE_64K,
							MMU_PAGE_64K, ssize);
			if (slot == -1) {
				if (mftb() & 0x1)
					hpte_group = (hash & htab_hash_mask) *
							HPTES_PER_GROUP;
				mmu_hash_ops.hpte_remove(hpte_group);
				/*
				 * FIXME!! Should be try the group from which we removed ?
				 */
				goto repeat;
			}
		}
		/*
		 * Hypervisor failure. Restore old pte and return -1
		 * similar to __hash_page_*
		 */
		if (unlikely(slot == -2)) {
			*ptep = __pte(old_pte);
			hash_failure_debug(ea, access, vsid, trap, ssize,
					   MMU_PAGE_64K, MMU_PAGE_64K, old_pte);
			return -1;
		}

		new_pte = (new_pte & ~_PAGE_HPTEFLAGS) | H_PAGE_HASHPTE;
		new_pte |= pte_set_hidx(ptep, rpte, 0, slot, PTRS_PER_PTE);
	}
	*ptep = __pte(new_pte & ~H_PAGE_BUSY);
	return 0;
}
Commit	Line	Data
91f1da99 AK	1	/*
91f1da99 AK	2	* Copyright IBM Corporation, 2015
47d99948	3	* Author Aneesh Kumar K.V <aneesh.kumar@linux.ibm.com>
91f1da99 AK	4	*
	5	* This program is free software; you can redistribute it and/or modify it
	6	* under the terms of version 2 of the GNU Lesser General Public License
	7	* as published by the Free Software Foundation.
	8	*
	9	* This program is distributed in the hope that it would be useful, but
	10	* WITHOUT ANY WARRANTY; without even the implied warranty of
	11	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
	12	*
	13	*/
	14
	15	#include <linux/mm.h>
	16	#include <asm/machdep.h>
	17	#include <asm/mmu.h>
9d2edb18	18
bf680d51	19	/*
9d2edb18 RP	20	* Return true, if the entry has a slot value which
9d2edb18 RP	21	* the software considers as invalid.
bf680d51	22	*/
9d2edb18	23	static inline bool hpte_soft_invalid(unsigned long hidx)
bf680d51	24	{
9d2edb18	25	return ((hidx & 0xfUL) == 0xfUL);
bf680d51	26	}
9d2edb18	27
bf680d51 AK	28	/*
	29	* index from 0 - 15
	30	*/
9d2edb18	31	bool __rpte_sub_valid(real_pte_t rpte, unsigned long index)
bf680d51	32	{
9d2edb18	33	return !(hpte_soft_invalid(__rpte_to_hidx(rpte, index)));
bf680d51	34	}
91f1da99 AK	35
	36	int __hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
	37	pte_t *ptep, unsigned long trap, unsigned long flags,
	38	int ssize, int subpg_prot)
	39	{
	40	real_pte_t rpte;
91f1da99 AK	41	unsigned long hpte_group;
91f1da99 AK	42	unsigned int subpg_index;
9d2edb18	43	unsigned long rflags, pa;
91f1da99	44	unsigned long old_pte, new_pte, subpg_pte;
9d2edb18	45	unsigned long vpn, hash, slot, gslot;
91f1da99 AK	46	unsigned long shift = mmu_psize_defs[MMU_PAGE_4K].shift;
	47
	48	/*
	49	* atomically mark the linux large page PTE busy and dirty
	50	*/
	51	do {
	52	pte_t pte = READ_ONCE(*ptep);
	53
	54	old_pte = pte_val(pte);
	55	/* If PTE busy, retry the access */
945537df	56	if (unlikely(old_pte & H_PAGE_BUSY))
91f1da99 AK	57	return 0;
91f1da99 AK	58	/* If PTE permissions don't match, take page fault */
ac29c640	59	if (unlikely(!check_pte_access(access, old_pte)))
91f1da99 AK	60	return 1;
	61	/*
	62	* Try to lock the PTE, add ACCESSED and DIRTY if it was
	63	* a write access. Since this is 4K insert of 64K page size
945537df	64	* also add H_PAGE_COMBO
91f1da99	65	*/
945537df	66	new_pte = old_pte \| H_PAGE_BUSY \| _PAGE_ACCESSED \| H_PAGE_COMBO;
c7d54842	67	if (access & _PAGE_WRITE)
91f1da99	68	new_pte \|= _PAGE_DIRTY;
3910a7f4 ME	69	} while (!pte_xchg(ptep, __pte(old_pte), __pte(new_pte)));
3910a7f4 ME	70
91f1da99 AK	71	/*
	72	* Handle the subpage protection bits
	73	*/
	74	subpg_pte = new_pte & ~subpg_prot;
d94b827e	75	rflags = htab_convert_pte_flags(subpg_pte, flags);
91f1da99	76
dd7b2f03	77	if (cpu_has_feature(CPU_FTR_NOEXECUTE) &&
91f1da99 AK	78	!cpu_has_feature(CPU_FTR_COHERENT_ICACHE)) {
	79
	80	/*
	81	* No CPU has hugepages but lacks no execute, so we
	82	* don't need to worry about that case
	83	*/
	84	rflags = hash_page_do_lazy_icache(rflags, __pte(old_pte), trap);
	85	}
	86
	87	subpg_index = (ea & (PAGE_SIZE - 1)) >> shift;
	88	vpn = hpt_vpn(ea, vsid, ssize);
ff31e105	89	rpte = __real_pte(__pte(old_pte), ptep, PTRS_PER_PTE);
91f1da99 AK	90	/*
	91	*None of the sub 4k page is hashed
	92	*/
945537df	93	if (!(old_pte & H_PAGE_HASHPTE))
91f1da99 AK	94	goto htab_insert_hpte;
	95	/*
	96	* Check if the pte was already inserted into the hash table
	97	* as a 64k HW page, and invalidate the 64k HPTE if so.
	98	*/
945537df	99	if (!(old_pte & H_PAGE_COMBO)) {
91f1da99	100	flush_hash_page(vpn, rpte, MMU_PAGE_64K, ssize, flags);
9ab3ac23 AK	101	/*
	102	* clear the old slot details from the old and new pte.
	103	* On hash insert failure we use old pte value and we don't
	104	* want slot information there if we have a insert failure.
	105	*/
bf9a95f9 RP	106	old_pte &= ~H_PAGE_HASHPTE;
bf9a95f9 RP	107	new_pte &= ~H_PAGE_HASHPTE;
91f1da99 AK	108	goto htab_insert_hpte;
	109	}
	110	/*
	111	* Check for sub page valid and update
	112	*/
	113	if (__rpte_sub_valid(rpte, subpg_index)) {
	114	int ret;
	115
9d2edb18 RP	116	gslot = pte_get_hash_gslot(vpn, shift, ssize, rpte,
	117	subpg_index);
	118	ret = mmu_hash_ops.hpte_updatepp(gslot, rflags, vpn,
7025776e BH	119	MMU_PAGE_4K, MMU_PAGE_4K,
7025776e BH	120	ssize, flags);
9d2edb18	121
91f1da99	122	/*
9d2edb18	123	* If we failed because typically the HPTE wasn't really here
91f1da99 AK	124	* we try an insertion.
	125	*/
	126	if (ret == -1)
	127	goto htab_insert_hpte;
	128
945537df	129	*ptep = __pte(new_pte & ~H_PAGE_BUSY);
91f1da99 AK	130	return 0;
	131	}
	132
	133	htab_insert_hpte:
9d2edb18 RP	134
	135	/*
	136	* Initialize all hidx entries to invalid value, the first time
	137	* the PTE is about to allocate a 4K HPTE.
	138	*/
	139	if (!(old_pte & H_PAGE_COMBO))
	140	rpte.hidx = INVALID_RPTE_HIDX;
	141
91f1da99	142	/*
945537df	143	* handle H_PAGE_4K_PFN case
91f1da99	144	*/
945537df	145	if (old_pte & H_PAGE_4K_PFN) {
91f1da99 AK	146	/*
	147	* All the sub 4k page have the same
	148	* physical address.
	149	*/
	150	pa = pte_pfn(__pte(old_pte)) << HW_PAGE_SHIFT;
	151	} else {
	152	pa = pte_pfn(__pte(old_pte)) << PAGE_SHIFT;
	153	pa += (subpg_index << shift);
	154	}
	155	hash = hpt_hash(vpn, shift, ssize);
	156	repeat:
1531cff4	157	hpte_group = (hash & htab_hash_mask) * HPTES_PER_GROUP;
91f1da99 AK	158
91f1da99 AK	159	/* Insert into the hash table, primary slot */
7025776e BH	160	slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa, rflags, 0,
7025776e BH	161	MMU_PAGE_4K, MMU_PAGE_4K, ssize);
91f1da99 AK	162	/*
	163	* Primary is full, try the secondary
	164	*/
	165	if (unlikely(slot == -1)) {
9d2edb18 RP	166	bool soft_invalid;
9d2edb18 RP	167
1531cff4	168	hpte_group = (~hash & htab_hash_mask) * HPTES_PER_GROUP;
7025776e BH	169	slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa,
	170	rflags, HPTE_V_SECONDARY,
	171	MMU_PAGE_4K, MMU_PAGE_4K,
	172	ssize);
9d2edb18 RP	173
	174	soft_invalid = hpte_soft_invalid(slot);
	175	if (unlikely(soft_invalid)) {
	176	/*
	177	* We got a valid slot from a hardware point of view.
	178	* but we cannot use it, because we use this special
	179	* value; as defined by hpte_soft_invalid(), to track
	180	* invalid slots. We cannot use it. So invalidate it.
	181	*/
	182	gslot = slot & _PTEIDX_GROUP_IX;
	183	mmu_hash_ops.hpte_invalidate(hpte_group + gslot, vpn,
	184	MMU_PAGE_4K, MMU_PAGE_4K,
	185	ssize, 0);
	186	}
	187
	188	if (unlikely(slot == -1 \|\| soft_invalid)) {
	189	/*
	190	* For soft invalid slot, let's ensure that we release a
	191	* slot from the primary, with the hope that we will
	192	* acquire that slot next time we try. This will ensure
	193	* that we do not get the same soft-invalid slot.
	194	*/
	195	if (soft_invalid \|\| (mftb() & 0x1))
1531cff4	196	hpte_group = (hash & htab_hash_mask) * HPTES_PER_GROUP;
9d2edb18	197
7025776e	198	mmu_hash_ops.hpte_remove(hpte_group);
91f1da99 AK	199	/*
	200	* FIXME!! Should be try the group from which we removed ?
	201	*/
	202	goto repeat;
	203	}
	204	}
	205	/*
e9a68147	206	* Hypervisor failure. Restore old pte and return -1
91f1da99 AK	207	* similar to __hash_page_*
	208	*/
	209	if (unlikely(slot == -2)) {
	210	*ptep = __pte(old_pte);
	211	hash_failure_debug(ea, access, vsid, trap, ssize,
	212	MMU_PAGE_4K, MMU_PAGE_4K, old_pte);
	213	return -1;
	214	}
9d2edb18	215
ff31e105	216	new_pte \|= pte_set_hidx(ptep, rpte, subpg_index, slot, PTRS_PER_PTE);
9d2edb18 RP	217	new_pte \|= H_PAGE_HASHPTE;
9d2edb18 RP	218
945537df	219	*ptep = __pte(new_pte & ~H_PAGE_BUSY);
91f1da99 AK	220	return 0;
91f1da99 AK	221	}
89ff7250 AK	222
	223	int __hash_page_64K(unsigned long ea, unsigned long access,
	224	unsigned long vsid, pte_t *ptep, unsigned long trap,
	225	unsigned long flags, int ssize)
	226	{
bf9a95f9	227	real_pte_t rpte;
89ff7250 AK	228	unsigned long hpte_group;
	229	unsigned long rflags, pa;
	230	unsigned long old_pte, new_pte;
	231	unsigned long vpn, hash, slot;
	232	unsigned long shift = mmu_psize_defs[MMU_PAGE_64K].shift;
	233
	234	/*
	235	* atomically mark the linux large page PTE busy and dirty
	236	*/
	237	do {
	238	pte_t pte = READ_ONCE(*ptep);
	239
	240	old_pte = pte_val(pte);
	241	/* If PTE busy, retry the access */
945537df	242	if (unlikely(old_pte & H_PAGE_BUSY))
89ff7250 AK	243	return 0;
89ff7250 AK	244	/* If PTE permissions don't match, take page fault */
ac29c640	245	if (unlikely(!check_pte_access(access, old_pte)))
89ff7250 AK	246	return 1;
	247	/*
	248	* Check if PTE has the cache-inhibit bit set
	249	* If so, bail out and refault as a 4k page
	250	*/
	251	if (!mmu_has_feature(MMU_FTR_CI_LARGE_PAGE) &&
30bda41a	252	unlikely(pte_ci(pte)))
89ff7250 AK	253	return 0;
	254	/*
	255	* Try to lock the PTE, add ACCESSED and DIRTY if it was
1ec3f937	256	* a write access.
89ff7250	257	*/
945537df	258	new_pte = old_pte \| H_PAGE_BUSY \| _PAGE_ACCESSED;
c7d54842	259	if (access & _PAGE_WRITE)
89ff7250	260	new_pte \|= _PAGE_DIRTY;
3910a7f4	261	} while (!pte_xchg(ptep, __pte(old_pte), __pte(new_pte)));
c6a3c495	262
d94b827e	263	rflags = htab_convert_pte_flags(new_pte, flags);
ff31e105	264	rpte = __real_pte(__pte(old_pte), ptep, PTRS_PER_PTE);
89ff7250	265
dd7b2f03	266	if (cpu_has_feature(CPU_FTR_NOEXECUTE) &&
89ff7250 AK	267	!cpu_has_feature(CPU_FTR_COHERENT_ICACHE))
	268	rflags = hash_page_do_lazy_icache(rflags, __pte(old_pte), trap);
	269
	270	vpn = hpt_vpn(ea, vsid, ssize);
945537df	271	if (unlikely(old_pte & H_PAGE_HASHPTE)) {
bf9a95f9 RP	272	unsigned long gslot;
bf9a95f9 RP	273
89ff7250 AK	274	/*
	275	* There MIGHT be an HPTE for this pte
	276	*/
bf9a95f9 RP	277	gslot = pte_get_hash_gslot(vpn, shift, ssize, rpte, 0);
bf9a95f9 RP	278	if (mmu_hash_ops.hpte_updatepp(gslot, rflags, vpn, MMU_PAGE_64K,
7025776e BH	279	MMU_PAGE_64K, ssize,
7025776e BH	280	flags) == -1)
89ff7250 AK	281	old_pte &= ~_PAGE_HPTEFLAGS;
	282	}
	283
945537df	284	if (likely(!(old_pte & H_PAGE_HASHPTE))) {
89ff7250 AK	285
	286	pa = pte_pfn(__pte(old_pte)) << PAGE_SHIFT;
	287	hash = hpt_hash(vpn, shift, ssize);
	288
	289	repeat:
1531cff4	290	hpte_group = (hash & htab_hash_mask) * HPTES_PER_GROUP;
89ff7250 AK	291
89ff7250 AK	292	/* Insert into the hash table, primary slot */
7025776e BH	293	slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa, rflags, 0,
	294	MMU_PAGE_64K, MMU_PAGE_64K,
	295	ssize);
89ff7250 AK	296	/*
	297	* Primary is full, try the secondary
	298	*/
	299	if (unlikely(slot == -1)) {
1531cff4	300	hpte_group = (~hash & htab_hash_mask) * HPTES_PER_GROUP;
7025776e BH	301	slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa,
	302	rflags,
	303	HPTE_V_SECONDARY,
	304	MMU_PAGE_64K,
	305	MMU_PAGE_64K, ssize);
89ff7250 AK	306	if (slot == -1) {
89ff7250 AK	307	if (mftb() & 0x1)
1531cff4 AK	308	hpte_group = (hash & htab_hash_mask) *
1531cff4 AK	309	HPTES_PER_GROUP;
7025776e	310	mmu_hash_ops.hpte_remove(hpte_group);
89ff7250 AK	311	/*
	312	* FIXME!! Should be try the group from which we removed ?
	313	*/
	314	goto repeat;
	315	}
	316	}
	317	/*
e9a68147	318	* Hypervisor failure. Restore old pte and return -1
89ff7250 AK	319	* similar to __hash_page_*
	320	*/
	321	if (unlikely(slot == -2)) {
	322	*ptep = __pte(old_pte);
	323	hash_failure_debug(ea, access, vsid, trap, ssize,
	324	MMU_PAGE_64K, MMU_PAGE_64K, old_pte);
	325	return -1;
	326	}
bf9a95f9	327
945537df	328	new_pte = (new_pte & ~_PAGE_HPTEFLAGS) \| H_PAGE_HASHPTE;
ff31e105	329	new_pte \|= pte_set_hidx(ptep, rpte, 0, slot, PTRS_PER_PTE);
89ff7250	330	}
945537df	331	*ptep = __pte(new_pte & ~H_PAGE_BUSY);
89ff7250 AK	332	return 0;
89ff7250 AK	333	}