[linux-2.6-block.git] / arch / powerpc / include / asm / mmu_context.h

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __ASM_POWERPC_MMU_CONTEXT_H
#define __ASM_POWERPC_MMU_CONTEXT_H
#ifdef __KERNEL__

#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/sched.h>
#include <linux/spinlock.h>
#include <asm/mmu.h>	
#include <asm/cputable.h>
#include <asm/cputhreads.h>

/*
 * Most if the context management is out of line
 */
extern int init_new_context(struct task_struct *tsk, struct mm_struct *mm);
extern void destroy_context(struct mm_struct *mm);
#ifdef CONFIG_SPAPR_TCE_IOMMU
struct mm_iommu_table_group_mem_t;

extern int isolate_lru_page(struct page *page);	/* from internal.h */
extern bool mm_iommu_preregistered(struct mm_struct *mm);
extern long mm_iommu_new(struct mm_struct *mm,
		unsigned long ua, unsigned long entries,
		struct mm_iommu_table_group_mem_t **pmem);
extern long mm_iommu_newdev(struct mm_struct *mm, unsigned long ua,
		unsigned long entries, unsigned long dev_hpa,
		struct mm_iommu_table_group_mem_t **pmem);
extern long mm_iommu_put(struct mm_struct *mm,
		struct mm_iommu_table_group_mem_t *mem);
extern void mm_iommu_init(struct mm_struct *mm);
extern void mm_iommu_cleanup(struct mm_struct *mm);
extern struct mm_iommu_table_group_mem_t *mm_iommu_lookup(struct mm_struct *mm,
		unsigned long ua, unsigned long size);
extern struct mm_iommu_table_group_mem_t *mm_iommu_lookup_rm(
		struct mm_struct *mm, unsigned long ua, unsigned long size);
extern struct mm_iommu_table_group_mem_t *mm_iommu_get(struct mm_struct *mm,
		unsigned long ua, unsigned long entries);
extern long mm_iommu_ua_to_hpa(struct mm_iommu_table_group_mem_t *mem,
		unsigned long ua, unsigned int pageshift, unsigned long *hpa);
extern long mm_iommu_ua_to_hpa_rm(struct mm_iommu_table_group_mem_t *mem,
		unsigned long ua, unsigned int pageshift, unsigned long *hpa);
extern void mm_iommu_ua_mark_dirty_rm(struct mm_struct *mm, unsigned long ua);
extern bool mm_iommu_is_devmem(struct mm_struct *mm, unsigned long hpa,
		unsigned int pageshift, unsigned long *size);
extern long mm_iommu_mapped_inc(struct mm_iommu_table_group_mem_t *mem);
extern void mm_iommu_mapped_dec(struct mm_iommu_table_group_mem_t *mem);
#else
static inline bool mm_iommu_is_devmem(struct mm_struct *mm, unsigned long hpa,
		unsigned int pageshift, unsigned long *size)
{
	return false;
}
#endif
extern void switch_slb(struct task_struct *tsk, struct mm_struct *mm);
extern void set_context(unsigned long id, pgd_t *pgd);

#ifdef CONFIG_PPC_BOOK3S_64
extern void radix__switch_mmu_context(struct mm_struct *prev,
				      struct mm_struct *next);
static inline void switch_mmu_context(struct mm_struct *prev,
				      struct mm_struct *next,
				      struct task_struct *tsk)
{
	if (radix_enabled())
		return radix__switch_mmu_context(prev, next);
	return switch_slb(tsk, next);
}

extern int hash__alloc_context_id(void);
extern void hash__reserve_context_id(int id);
extern void __destroy_context(int context_id);
static inline void mmu_context_init(void) { }

static inline int alloc_extended_context(struct mm_struct *mm,
					 unsigned long ea)
{
	int context_id;

	int index = ea >> MAX_EA_BITS_PER_CONTEXT;

	context_id = hash__alloc_context_id();
	if (context_id < 0)
		return context_id;

	VM_WARN_ON(mm->context.extended_id[index]);
	mm->context.extended_id[index] = context_id;
	return context_id;
}

static inline bool need_extra_context(struct mm_struct *mm, unsigned long ea)
{
	int context_id;

	context_id = get_user_context(&mm->context, ea);
	if (!context_id)
		return true;
	return false;
}

#else
extern void switch_mmu_context(struct mm_struct *prev, struct mm_struct *next,
			       struct task_struct *tsk);
extern unsigned long __init_new_context(void);
extern void __destroy_context(unsigned long context_id);
extern void mmu_context_init(void);
static inline int alloc_extended_context(struct mm_struct *mm,
					 unsigned long ea)
{
	/* non book3s_64 should never find this called */
	WARN_ON(1);
	return -ENOMEM;
}

static inline bool need_extra_context(struct mm_struct *mm, unsigned long ea)
{
	return false;
}
#endif

#if defined(CONFIG_KVM_BOOK3S_HV_POSSIBLE) && defined(CONFIG_PPC_RADIX_MMU)
extern void radix_kvm_prefetch_workaround(struct mm_struct *mm);
#else
static inline void radix_kvm_prefetch_workaround(struct mm_struct *mm) { }
#endif

extern void switch_cop(struct mm_struct *next);
extern int use_cop(unsigned long acop, struct mm_struct *mm);
extern void drop_cop(unsigned long acop, struct mm_struct *mm);

#ifdef CONFIG_PPC_BOOK3S_64
static inline void inc_mm_active_cpus(struct mm_struct *mm)
{
	atomic_inc(&mm->context.active_cpus);
}

static inline void dec_mm_active_cpus(struct mm_struct *mm)
{
	atomic_dec(&mm->context.active_cpus);
}

static inline void mm_context_add_copro(struct mm_struct *mm)
{
	/*
	 * If any copro is in use, increment the active CPU count
	 * in order to force TLB invalidations to be global as to
	 * propagate to the Nest MMU.
	 */
	if (atomic_inc_return(&mm->context.copros) == 1)
		inc_mm_active_cpus(mm);
}

static inline void mm_context_remove_copro(struct mm_struct *mm)
{
	int c;

	/*
	 * When removing the last copro, we need to broadcast a global
	 * flush of the full mm, as the next TLBI may be local and the
	 * nMMU and/or PSL need to be cleaned up.
	 *
	 * Both the 'copros' and 'active_cpus' counts are looked at in
	 * flush_all_mm() to determine the scope (local/global) of the
	 * TLBIs, so we need to flush first before decrementing
	 * 'copros'. If this API is used by several callers for the
	 * same context, it can lead to over-flushing. It's hopefully
	 * not common enough to be a problem.
	 *
	 * Skip on hash, as we don't know how to do the proper flush
	 * for the time being. Invalidations will remain global if
	 * used on hash. Note that we can't drop 'copros' either, as
	 * it could make some invalidations local with no flush
	 * in-between.
	 */
	if (radix_enabled()) {
		flush_all_mm(mm);

		c = atomic_dec_if_positive(&mm->context.copros);
		/* Detect imbalance between add and remove */
		WARN_ON(c < 0);

		if (c == 0)
			dec_mm_active_cpus(mm);
	}
}
#else
static inline void inc_mm_active_cpus(struct mm_struct *mm) { }
static inline void dec_mm_active_cpus(struct mm_struct *mm) { }
static inline void mm_context_add_copro(struct mm_struct *mm) { }
static inline void mm_context_remove_copro(struct mm_struct *mm) { }
#endif


extern void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next,
			       struct task_struct *tsk);

static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
			     struct task_struct *tsk)
{
	unsigned long flags;

	local_irq_save(flags);
	switch_mm_irqs_off(prev, next, tsk);
	local_irq_restore(flags);
}
#define switch_mm_irqs_off switch_mm_irqs_off


#define deactivate_mm(tsk,mm)	do { } while (0)

/*
 * After we have set current->mm to a new value, this activates
 * the context for the new mm so we see the new mappings.
 */
static inline void activate_mm(struct mm_struct *prev, struct mm_struct *next)
{
	switch_mm(prev, next, current);
}

/* We don't currently use enter_lazy_tlb() for anything */
static inline void enter_lazy_tlb(struct mm_struct *mm,
				  struct task_struct *tsk)
{
	/* 64-bit Book3E keeps track of current PGD in the PACA */
#ifdef CONFIG_PPC_BOOK3E_64
	get_paca()->pgd = NULL;
#endif
}

#ifdef CONFIG_PPC_BOOK3E_64
static inline void arch_exit_mmap(struct mm_struct *mm)
{
}
#else
extern void arch_exit_mmap(struct mm_struct *mm);
#endif

static inline void arch_unmap(struct mm_struct *mm,
			      struct vm_area_struct *vma,
			      unsigned long start, unsigned long end)
{
	if (start <= mm->context.vdso_base && mm->context.vdso_base < end)
		mm->context.vdso_base = 0;
}

static inline void arch_bprm_mm_init(struct mm_struct *mm,
				     struct vm_area_struct *vma)
{
}

#ifdef CONFIG_PPC_MEM_KEYS
bool arch_vma_access_permitted(struct vm_area_struct *vma, bool write,
			       bool execute, bool foreign);
void arch_dup_pkeys(struct mm_struct *oldmm, struct mm_struct *mm);
#else /* CONFIG_PPC_MEM_KEYS */
static inline bool arch_vma_access_permitted(struct vm_area_struct *vma,
		bool write, bool execute, bool foreign)
{
	/* by default, allow everything */
	return true;
}

#define pkey_mm_init(mm)
#define thread_pkey_regs_save(thread)
#define thread_pkey_regs_restore(new_thread, old_thread)
#define thread_pkey_regs_init(thread)
#define arch_dup_pkeys(oldmm, mm)

static inline u64 pte_to_hpte_pkey_bits(u64 pteflags)
{
	return 0x0UL;
}

#endif /* CONFIG_PPC_MEM_KEYS */

static inline int arch_dup_mmap(struct mm_struct *oldmm,
				struct mm_struct *mm)
{
	arch_dup_pkeys(oldmm, mm);
	return 0;
}

#endif /* __KERNEL__ */
#endif /* __ASM_POWERPC_MMU_CONTEXT_H */
Commit	Line	Data
b2441318	1	/* SPDX-License-Identifier: GPL-2.0 */
047ea784 PM	2	#ifndef __ASM_POWERPC_MMU_CONTEXT_H
047ea784 PM	3	#define __ASM_POWERPC_MMU_CONTEXT_H
88ced031	4	#ifdef __KERNEL__
047ea784	5
5e696617 BH	6	#include <linux/kernel.h>
	7	#include <linux/mm.h>
	8	#include <linux/sched.h>
	9	#include <linux/spinlock.h>
80a7cc6c KG	10	#include <asm/mmu.h>
80a7cc6c KG	11	#include <asm/cputable.h>
5e696617	12	#include <asm/cputhreads.h>
80a7cc6c KG	13
80a7cc6c KG	14	/*
5e696617	15	* Most if the context management is out of line
80a7cc6c	16	*/
1da177e4 LT	17	extern int init_new_context(struct task_struct tsk, struct mm_struct mm);
1da177e4 LT	18	extern void destroy_context(struct mm_struct *mm);
15b244a8 AK	19	#ifdef CONFIG_SPAPR_TCE_IOMMU
	20	struct mm_iommu_table_group_mem_t;
	21
2e5bbb54	22	extern int isolate_lru_page(struct page page); / from internal.h */
d7baee69	23	extern bool mm_iommu_preregistered(struct mm_struct *mm);
e0bf78b0	24	extern long mm_iommu_new(struct mm_struct *mm,
d7baee69	25	unsigned long ua, unsigned long entries,
15b244a8	26	struct mm_iommu_table_group_mem_t **pmem);
c10c21ef AK	27	extern long mm_iommu_newdev(struct mm_struct *mm, unsigned long ua,
	28	unsigned long entries, unsigned long dev_hpa,
	29	struct mm_iommu_table_group_mem_t **pmem);
d7baee69 AK	30	extern long mm_iommu_put(struct mm_struct *mm,
d7baee69 AK	31	struct mm_iommu_table_group_mem_t *mem);
88f54a35 AK	32	extern void mm_iommu_init(struct mm_struct *mm);
88f54a35 AK	33	extern void mm_iommu_cleanup(struct mm_struct *mm);
d7baee69 AK	34	extern struct mm_iommu_table_group_mem_t mm_iommu_lookup(struct mm_struct mm,
d7baee69 AK	35	unsigned long ua, unsigned long size);
6b5c19c5 AK	36	extern struct mm_iommu_table_group_mem_t *mm_iommu_lookup_rm(
6b5c19c5 AK	37	struct mm_struct *mm, unsigned long ua, unsigned long size);
e0bf78b0	38	extern struct mm_iommu_table_group_mem_t mm_iommu_get(struct mm_struct mm,
d7baee69	39	unsigned long ua, unsigned long entries);
15b244a8	40	extern long mm_iommu_ua_to_hpa(struct mm_iommu_table_group_mem_t *mem,
76fa4975	41	unsigned long ua, unsigned int pageshift, unsigned long *hpa);
6b5c19c5	42	extern long mm_iommu_ua_to_hpa_rm(struct mm_iommu_table_group_mem_t *mem,
76fa4975	43	unsigned long ua, unsigned int pageshift, unsigned long *hpa);
425333bf	44	extern void mm_iommu_ua_mark_dirty_rm(struct mm_struct *mm, unsigned long ua);
c10c21ef AK	45	extern bool mm_iommu_is_devmem(struct mm_struct *mm, unsigned long hpa,
c10c21ef AK	46	unsigned int pageshift, unsigned long *size);
15b244a8 AK	47	extern long mm_iommu_mapped_inc(struct mm_iommu_table_group_mem_t *mem);
15b244a8 AK	48	extern void mm_iommu_mapped_dec(struct mm_iommu_table_group_mem_t *mem);
c10c21ef AK	49	#else
	50	static inline bool mm_iommu_is_devmem(struct mm_struct *mm, unsigned long hpa,
	51	unsigned int pageshift, unsigned long *size)
	52	{
	53	return false;
	54	}
15b244a8	55	#endif
1da177e4	56	extern void switch_slb(struct task_struct tsk, struct mm_struct mm);
5e696617	57	extern void set_context(unsigned long id, pgd_t *pgd);
1da177e4	58
6f0ef0f5	59	#ifdef CONFIG_PPC_BOOK3S_64
7e381c0f	60	extern void radix__switch_mmu_context(struct mm_struct *prev,
a25bd72b	61	struct mm_struct *next);
d2adba3f AK	62	static inline void switch_mmu_context(struct mm_struct *prev,
	63	struct mm_struct *next,
	64	struct task_struct *tsk)
	65	{
7e381c0f AK	66	if (radix_enabled())
7e381c0f AK	67	return radix__switch_mmu_context(prev, next);
d2adba3f AK	68	return switch_slb(tsk, next);
	69	}
	70
a336f2f5	71	extern int hash__alloc_context_id(void);
82228e36	72	extern void hash__reserve_context_id(int id);
e85a4710	73	extern void __destroy_context(int context_id);
6f0ef0f5	74	static inline void mmu_context_init(void) { }
f384796c AK	75
	76	static inline int alloc_extended_context(struct mm_struct *mm,
	77	unsigned long ea)
	78	{
	79	int context_id;
	80
	81	int index = ea >> MAX_EA_BITS_PER_CONTEXT;
	82
	83	context_id = hash__alloc_context_id();
	84	if (context_id < 0)
	85	return context_id;
	86
	87	VM_WARN_ON(mm->context.extended_id[index]);
	88	mm->context.extended_id[index] = context_id;
	89	return context_id;
	90	}
	91
	92	static inline bool need_extra_context(struct mm_struct *mm, unsigned long ea)
	93	{
	94	int context_id;
	95
c9f80734	96	context_id = get_user_context(&mm->context, ea);
f384796c AK	97	if (!context_id)
	98	return true;
	99	return false;
	100	}
	101
6f0ef0f5	102	#else
d2adba3f AK	103	extern void switch_mmu_context(struct mm_struct prev, struct mm_struct next,
d2adba3f AK	104	struct task_struct *tsk);
c83ec269 AG	105	extern unsigned long __init_new_context(void);
c83ec269 AG	106	extern void __destroy_context(unsigned long context_id);
6f0ef0f5	107	extern void mmu_context_init(void);
f384796c AK	108	static inline int alloc_extended_context(struct mm_struct *mm,
	109	unsigned long ea)
	110	{
	111	/* non book3s_64 should never find this called */
	112	WARN_ON(1);
	113	return -ENOMEM;
	114	}
	115
	116	static inline bool need_extra_context(struct mm_struct *mm, unsigned long ea)
	117	{
	118	return false;
	119	}
6f0ef0f5 BH	120	#endif
6f0ef0f5 BH	121
a25bd72b BH	122	#if defined(CONFIG_KVM_BOOK3S_HV_POSSIBLE) && defined(CONFIG_PPC_RADIX_MMU)
	123	extern void radix_kvm_prefetch_workaround(struct mm_struct *mm);
	124	#else
	125	static inline void radix_kvm_prefetch_workaround(struct mm_struct *mm) { }
	126	#endif
	127
851d2e2f THFL	128	extern void switch_cop(struct mm_struct *next);
	129	extern int use_cop(unsigned long acop, struct mm_struct *mm);
	130	extern void drop_cop(unsigned long acop, struct mm_struct *mm);
	131
03b8abed FB	132	#ifdef CONFIG_PPC_BOOK3S_64
	133	static inline void inc_mm_active_cpus(struct mm_struct *mm)
	134	{
	135	atomic_inc(&mm->context.active_cpus);
	136	}
	137
	138	static inline void dec_mm_active_cpus(struct mm_struct *mm)
	139	{
	140	atomic_dec(&mm->context.active_cpus);
	141	}
	142
	143	static inline void mm_context_add_copro(struct mm_struct *mm)
	144	{
	145	/*
aff6f8cb BH	146	* If any copro is in use, increment the active CPU count
	147	* in order to force TLB invalidations to be global as to
	148	* propagate to the Nest MMU.
03b8abed	149	*/
aff6f8cb BH	150	if (atomic_inc_return(&mm->context.copros) == 1)
aff6f8cb BH	151	inc_mm_active_cpus(mm);
03b8abed FB	152	}
	153
	154	static inline void mm_context_remove_copro(struct mm_struct *mm)
	155	{
aff6f8cb BH	156	int c;
aff6f8cb BH	157
03b8abed	158	/*
cca19f0b FB	159	* When removing the last copro, we need to broadcast a global
	160	* flush of the full mm, as the next TLBI may be local and the
	161	* nMMU and/or PSL need to be cleaned up.
	162	*
	163	* Both the 'copros' and 'active_cpus' counts are looked at in
	164	* flush_all_mm() to determine the scope (local/global) of the
	165	* TLBIs, so we need to flush first before decrementing
	166	* 'copros'. If this API is used by several callers for the
	167	* same context, it can lead to over-flushing. It's hopefully
	168	* not common enough to be a problem.
03b8abed FB	169	*
	170	* Skip on hash, as we don't know how to do the proper flush
	171	* for the time being. Invalidations will remain global if
cca19f0b FB	172	* used on hash. Note that we can't drop 'copros' either, as
	173	* it could make some invalidations local with no flush
	174	* in-between.
03b8abed	175	*/
cca19f0b	176	if (radix_enabled()) {
03b8abed	177	flush_all_mm(mm);
cca19f0b FB	178
	179	c = atomic_dec_if_positive(&mm->context.copros);
	180	/* Detect imbalance between add and remove */
	181	WARN_ON(c < 0);
	182
	183	if (c == 0)
	184	dec_mm_active_cpus(mm);
03b8abed FB	185	}
	186	}
	187	#else
	188	static inline void inc_mm_active_cpus(struct mm_struct *mm) { }
	189	static inline void dec_mm_active_cpus(struct mm_struct *mm) { }
	190	static inline void mm_context_add_copro(struct mm_struct *mm) { }
	191	static inline void mm_context_remove_copro(struct mm_struct *mm) { }
	192	#endif
	193
	194
3a2df379 BH	195	extern void switch_mm_irqs_off(struct mm_struct prev, struct mm_struct next,
3a2df379 BH	196	struct task_struct *tsk);
1da177e4	197
9765ad13 DG	198	static inline void switch_mm(struct mm_struct prev, struct mm_struct next,
	199	struct task_struct *tsk)
	200	{
	201	unsigned long flags;
	202
	203	local_irq_save(flags);
	204	switch_mm_irqs_off(prev, next, tsk);
	205	local_irq_restore(flags);
	206	}
	207	#define switch_mm_irqs_off switch_mm_irqs_off
	208
	209
1da177e4 LT	210	#define deactivate_mm(tsk,mm) do { } while (0)
	211
	212	/*
	213	* After we have set current->mm to a new value, this activates
	214	* the context for the new mm so we see the new mappings.
	215	*/
	216	static inline void activate_mm(struct mm_struct prev, struct mm_struct next)
	217	{
1da177e4	218	switch_mm(prev, next, current);
1da177e4 LT	219	}
1da177e4 LT	220
5e696617 BH	221	/* We don't currently use enter_lazy_tlb() for anything */
	222	static inline void enter_lazy_tlb(struct mm_struct *mm,
	223	struct task_struct *tsk)
	224	{
25d21ad6 BH	225	/* 64-bit Book3E keeps track of current PGD in the PACA */
	226	#ifdef CONFIG_PPC_BOOK3E_64
	227	get_paca()->pgd = NULL;
	228	#endif
5e696617 BH	229	}
5e696617 BH	230
32ea4c14	231	#ifdef CONFIG_PPC_BOOK3E_64
83d3f0e9 LD	232	static inline void arch_exit_mmap(struct mm_struct *mm)
	233	{
	234	}
30b49ec7 NP	235	#else
	236	extern void arch_exit_mmap(struct mm_struct *mm);
	237	#endif
83d3f0e9 LD	238
	239	static inline void arch_unmap(struct mm_struct *mm,
	240	struct vm_area_struct *vma,
	241	unsigned long start, unsigned long end)
	242	{
	243	if (start <= mm->context.vdso_base && mm->context.vdso_base < end)
	244	mm->context.vdso_base = 0;
	245	}
	246
	247	static inline void arch_bprm_mm_init(struct mm_struct *mm,
	248	struct vm_area_struct *vma)
	249	{
	250	}
	251
1137573a RP	252	#ifdef CONFIG_PPC_MEM_KEYS
	253	bool arch_vma_access_permitted(struct vm_area_struct *vma, bool write,
	254	bool execute, bool foreign);
2cd4bd19	255	void arch_dup_pkeys(struct mm_struct oldmm, struct mm_struct mm);
1137573a	256	#else /* CONFIG_PPC_MEM_KEYS */
1b2ee126	257	static inline bool arch_vma_access_permitted(struct vm_area_struct *vma,
d61172b4	258	bool write, bool execute, bool foreign)
33a709b2 DH	259	{
	260	/* by default, allow everything */
	261	return true;
	262	}
92e3da3c	263
4fb158f6	264	#define pkey_mm_init(mm)
06bb53b3 RP	265	#define thread_pkey_regs_save(thread)
	266	#define thread_pkey_regs_restore(new_thread, old_thread)
	267	#define thread_pkey_regs_init(thread)
2cd4bd19	268	#define arch_dup_pkeys(oldmm, mm)
87bbabbe	269
a6590ca5 RP	270	static inline u64 pte_to_hpte_pkey_bits(u64 pteflags)
	271	{
	272	return 0x0UL;
	273	}
	274
4fb158f6 RP	275	#endif /* CONFIG_PPC_MEM_KEYS */
4fb158f6 RP	276
2cd4bd19 RP	277	static inline int arch_dup_mmap(struct mm_struct *oldmm,
	278	struct mm_struct *mm)
	279	{
	280	arch_dup_pkeys(oldmm, mm);
	281	return 0;
	282	}
	283
88ced031	284	#endif /* __KERNEL__ */
047ea784	285	#endif /* __ASM_POWERPC_MMU_CONTEXT_H */