[linux-block.git] / arch / arm64 / include / asm / fpsimd.h

/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * Copyright (C) 2012 ARM Ltd.
 */
#ifndef __ASM_FP_H
#define __ASM_FP_H

#include <asm/errno.h>
#include <asm/ptrace.h>
#include <asm/processor.h>
#include <asm/sigcontext.h>
#include <asm/sysreg.h>

#ifndef __ASSEMBLY__

#include <linux/bitmap.h>
#include <linux/build_bug.h>
#include <linux/bug.h>
#include <linux/cache.h>
#include <linux/init.h>
#include <linux/stddef.h>
#include <linux/types.h>

#ifdef CONFIG_COMPAT
/* Masks for extracting the FPSR and FPCR from the FPSCR */
#define VFP_FPSCR_STAT_MASK	0xf800009f
#define VFP_FPSCR_CTRL_MASK	0x07f79f00
/*
 * The VFP state has 32x64-bit registers and a single 32-bit
 * control/status register.
 */
#define VFP_STATE_SIZE		((32 * 8) + 4)
#endif

static inline unsigned long cpacr_save_enable_kernel_sve(void)
{
	unsigned long old = read_sysreg(cpacr_el1);
	unsigned long set = CPACR_EL1_FPEN_EL1EN | CPACR_EL1_ZEN_EL1EN;

	write_sysreg(old | set, cpacr_el1);
	isb();
	return old;
}

static inline unsigned long cpacr_save_enable_kernel_sme(void)
{
	unsigned long old = read_sysreg(cpacr_el1);
	unsigned long set = CPACR_EL1_FPEN_EL1EN | CPACR_EL1_SMEN_EL1EN;

	write_sysreg(old | set, cpacr_el1);
	isb();
	return old;
}

static inline void cpacr_restore(unsigned long cpacr)
{
	write_sysreg(cpacr, cpacr_el1);
	isb();
}

/*
 * When we defined the maximum SVE vector length we defined the ABI so
 * that the maximum vector length included all the reserved for future
 * expansion bits in ZCR rather than those just currently defined by
 * the architecture.  Using this length to allocate worst size buffers
 * results in excessively large allocations, and this effect is even
 * more pronounced for SME due to ZA.  Define more suitable VLs for
 * these situations.
 */
#define ARCH_SVE_VQ_MAX ((ZCR_ELx_LEN_MASK >> ZCR_ELx_LEN_SHIFT) + 1)
#define SME_VQ_MAX	((SMCR_ELx_LEN_MASK >> SMCR_ELx_LEN_SHIFT) + 1)

struct task_struct;

extern void fpsimd_save_state(struct user_fpsimd_state *state);
extern void fpsimd_load_state(struct user_fpsimd_state *state);

extern void fpsimd_thread_switch(struct task_struct *next);
extern void fpsimd_flush_thread(void);

extern void fpsimd_signal_preserve_current_state(void);
extern void fpsimd_preserve_current_state(void);
extern void fpsimd_restore_current_state(void);
extern void fpsimd_update_current_state(struct user_fpsimd_state const *state);
extern void fpsimd_kvm_prepare(void);

struct cpu_fp_state {
	struct user_fpsimd_state *st;
	void *sve_state;
	void *sme_state;
	u64 *svcr;
	unsigned int sve_vl;
	unsigned int sme_vl;
	enum fp_type *fp_type;
	enum fp_type to_save;
};

extern void fpsimd_bind_state_to_cpu(struct cpu_fp_state *fp_state);

extern void fpsimd_flush_task_state(struct task_struct *target);
extern void fpsimd_save_and_flush_cpu_state(void);

static inline bool thread_sm_enabled(struct thread_struct *thread)
{
	return system_supports_sme() && (thread->svcr & SVCR_SM_MASK);
}

static inline bool thread_za_enabled(struct thread_struct *thread)
{
	return system_supports_sme() && (thread->svcr & SVCR_ZA_MASK);
}

/* Maximum VL that SVE/SME VL-agnostic software can transparently support */
#define VL_ARCH_MAX 0x100

/* Offset of FFR in the SVE register dump */
static inline size_t sve_ffr_offset(int vl)
{
	return SVE_SIG_FFR_OFFSET(sve_vq_from_vl(vl)) - SVE_SIG_REGS_OFFSET;
}

static inline void *sve_pffr(struct thread_struct *thread)
{
	unsigned int vl;

	if (system_supports_sme() && thread_sm_enabled(thread))
		vl = thread_get_sme_vl(thread);
	else
		vl = thread_get_sve_vl(thread);

	return (char *)thread->sve_state + sve_ffr_offset(vl);
}

static inline void *thread_zt_state(struct thread_struct *thread)
{
	/* The ZT register state is stored immediately after the ZA state */
	unsigned int sme_vq = sve_vq_from_vl(thread_get_sme_vl(thread));
	return thread->sme_state + ZA_SIG_REGS_SIZE(sme_vq);
}

extern void sve_save_state(void *state, u32 *pfpsr, int save_ffr);
extern void sve_load_state(void const *state, u32 const *pfpsr,
			   int restore_ffr);
extern void sve_flush_live(bool flush_ffr, unsigned long vq_minus_1);
extern unsigned int sve_get_vl(void);
extern void sve_set_vq(unsigned long vq_minus_1);
extern void sme_set_vq(unsigned long vq_minus_1);
extern void sme_save_state(void *state, int zt);
extern void sme_load_state(void const *state, int zt);

struct arm64_cpu_capabilities;
extern void cpu_enable_fpsimd(const struct arm64_cpu_capabilities *__unused);
extern void cpu_enable_sve(const struct arm64_cpu_capabilities *__unused);
extern void cpu_enable_sme(const struct arm64_cpu_capabilities *__unused);
extern void cpu_enable_sme2(const struct arm64_cpu_capabilities *__unused);
extern void cpu_enable_fa64(const struct arm64_cpu_capabilities *__unused);

extern u64 read_smcr_features(void);

/*
 * Helpers to translate bit indices in sve_vq_map to VQ values (and
 * vice versa).  This allows find_next_bit() to be used to find the
 * _maximum_ VQ not exceeding a certain value.
 */
static inline unsigned int __vq_to_bit(unsigned int vq)
{
	return SVE_VQ_MAX - vq;
}

static inline unsigned int __bit_to_vq(unsigned int bit)
{
	return SVE_VQ_MAX - bit;
}


struct vl_info {
	enum vec_type type;
	const char *name;		/* For display purposes */

	/* Minimum supported vector length across all CPUs */
	int min_vl;

	/* Maximum supported vector length across all CPUs */
	int max_vl;
	int max_virtualisable_vl;

	/*
	 * Set of available vector lengths,
	 * where length vq encoded as bit __vq_to_bit(vq):
	 */
	DECLARE_BITMAP(vq_map, SVE_VQ_MAX);

	/* Set of vector lengths present on at least one cpu: */
	DECLARE_BITMAP(vq_partial_map, SVE_VQ_MAX);
};

#ifdef CONFIG_ARM64_SVE

extern void sve_alloc(struct task_struct *task, bool flush);
extern void fpsimd_release_task(struct task_struct *task);
extern void fpsimd_sync_to_sve(struct task_struct *task);
extern void fpsimd_force_sync_to_sve(struct task_struct *task);
extern void sve_sync_to_fpsimd(struct task_struct *task);
extern void sve_sync_from_fpsimd_zeropad(struct task_struct *task);

extern int vec_set_vector_length(struct task_struct *task, enum vec_type type,
				 unsigned long vl, unsigned long flags);

extern int sve_set_current_vl(unsigned long arg);
extern int sve_get_current_vl(void);

static inline void sve_user_disable(void)
{
	sysreg_clear_set(cpacr_el1, CPACR_EL1_ZEN_EL0EN, 0);
}

static inline void sve_user_enable(void)
{
	sysreg_clear_set(cpacr_el1, 0, CPACR_EL1_ZEN_EL0EN);
}

#define sve_cond_update_zcr_vq(val, reg)		\
	do {						\
		u64 __zcr = read_sysreg_s((reg));	\
		u64 __new = __zcr & ~ZCR_ELx_LEN_MASK;	\
		__new |= (val) & ZCR_ELx_LEN_MASK;	\
		if (__zcr != __new)			\
			write_sysreg_s(__new, (reg));	\
	} while (0)

/*
 * Probing and setup functions.
 * Calls to these functions must be serialised with one another.
 */
enum vec_type;

extern void __init vec_init_vq_map(enum vec_type type);
extern void vec_update_vq_map(enum vec_type type);
extern int vec_verify_vq_map(enum vec_type type);
extern void __init sve_setup(void);

extern __ro_after_init struct vl_info vl_info[ARM64_VEC_MAX];

static inline void write_vl(enum vec_type type, u64 val)
{
	u64 tmp;

	switch (type) {
#ifdef CONFIG_ARM64_SVE
	case ARM64_VEC_SVE:
		tmp = read_sysreg_s(SYS_ZCR_EL1) & ~ZCR_ELx_LEN_MASK;
		write_sysreg_s(tmp | val, SYS_ZCR_EL1);
		break;
#endif
#ifdef CONFIG_ARM64_SME
	case ARM64_VEC_SME:
		tmp = read_sysreg_s(SYS_SMCR_EL1) & ~SMCR_ELx_LEN_MASK;
		write_sysreg_s(tmp | val, SYS_SMCR_EL1);
		break;
#endif
	default:
		WARN_ON_ONCE(1);
		break;
	}
}

static inline int vec_max_vl(enum vec_type type)
{
	return vl_info[type].max_vl;
}

static inline int vec_max_virtualisable_vl(enum vec_type type)
{
	return vl_info[type].max_virtualisable_vl;
}

static inline int sve_max_vl(void)
{
	return vec_max_vl(ARM64_VEC_SVE);
}

static inline int sve_max_virtualisable_vl(void)
{
	return vec_max_virtualisable_vl(ARM64_VEC_SVE);
}

/* Ensure vq >= SVE_VQ_MIN && vq <= SVE_VQ_MAX before calling this function */
static inline bool vq_available(enum vec_type type, unsigned int vq)
{
	return test_bit(__vq_to_bit(vq), vl_info[type].vq_map);
}

static inline bool sve_vq_available(unsigned int vq)
{
	return vq_available(ARM64_VEC_SVE, vq);
}

size_t sve_state_size(struct task_struct const *task);

#else /* ! CONFIG_ARM64_SVE */

static inline void sve_alloc(struct task_struct *task, bool flush) { }
static inline void fpsimd_release_task(struct task_struct *task) { }
static inline void sve_sync_to_fpsimd(struct task_struct *task) { }
static inline void sve_sync_from_fpsimd_zeropad(struct task_struct *task) { }

static inline int sve_max_virtualisable_vl(void)
{
	return 0;
}

static inline int sve_set_current_vl(unsigned long arg)
{
	return -EINVAL;
}

static inline int sve_get_current_vl(void)
{
	return -EINVAL;
}

static inline int sve_max_vl(void)
{
	return -EINVAL;
}

static inline bool sve_vq_available(unsigned int vq) { return false; }

static inline void sve_user_disable(void) { BUILD_BUG(); }
static inline void sve_user_enable(void) { BUILD_BUG(); }

#define sve_cond_update_zcr_vq(val, reg) do { } while (0)

static inline void vec_init_vq_map(enum vec_type t) { }
static inline void vec_update_vq_map(enum vec_type t) { }
static inline int vec_verify_vq_map(enum vec_type t) { return 0; }
static inline void sve_setup(void) { }

static inline size_t sve_state_size(struct task_struct const *task)
{
	return 0;
}

#endif /* ! CONFIG_ARM64_SVE */

#ifdef CONFIG_ARM64_SME

static inline void sme_user_disable(void)
{
	sysreg_clear_set(cpacr_el1, CPACR_EL1_SMEN_EL0EN, 0);
}

static inline void sme_user_enable(void)
{
	sysreg_clear_set(cpacr_el1, 0, CPACR_EL1_SMEN_EL0EN);
}

static inline void sme_smstart_sm(void)
{
	asm volatile(__msr_s(SYS_SVCR_SMSTART_SM_EL0, "xzr"));
}

static inline void sme_smstop_sm(void)
{
	asm volatile(__msr_s(SYS_SVCR_SMSTOP_SM_EL0, "xzr"));
}

static inline void sme_smstop(void)
{
	asm volatile(__msr_s(SYS_SVCR_SMSTOP_SMZA_EL0, "xzr"));
}

extern void __init sme_setup(void);

static inline int sme_max_vl(void)
{
	return vec_max_vl(ARM64_VEC_SME);
}

static inline int sme_max_virtualisable_vl(void)
{
	return vec_max_virtualisable_vl(ARM64_VEC_SME);
}

extern void sme_alloc(struct task_struct *task, bool flush);
extern unsigned int sme_get_vl(void);
extern int sme_set_current_vl(unsigned long arg);
extern int sme_get_current_vl(void);
extern void sme_suspend_exit(void);

/*
 * Return how many bytes of memory are required to store the full SME
 * specific state for task, given task's currently configured vector
 * length.
 */
static inline size_t sme_state_size(struct task_struct const *task)
{
	unsigned int vl = task_get_sme_vl(task);
	size_t size;

	size = ZA_SIG_REGS_SIZE(sve_vq_from_vl(vl));

	if (system_supports_sme2())
		size += ZT_SIG_REG_SIZE;

	return size;
}

#else

static inline void sme_user_disable(void) { BUILD_BUG(); }
static inline void sme_user_enable(void) { BUILD_BUG(); }

static inline void sme_smstart_sm(void) { }
static inline void sme_smstop_sm(void) { }
static inline void sme_smstop(void) { }

static inline void sme_alloc(struct task_struct *task, bool flush) { }
static inline void sme_setup(void) { }
static inline unsigned int sme_get_vl(void) { return 0; }
static inline int sme_max_vl(void) { return 0; }
static inline int sme_max_virtualisable_vl(void) { return 0; }
static inline int sme_set_current_vl(unsigned long arg) { return -EINVAL; }
static inline int sme_get_current_vl(void) { return -EINVAL; }
static inline void sme_suspend_exit(void) { }

static inline size_t sme_state_size(struct task_struct const *task)
{
	return 0;
}

#endif /* ! CONFIG_ARM64_SME */

/* For use by EFI runtime services calls only */
extern void __efi_fpsimd_begin(void);
extern void __efi_fpsimd_end(void);

#endif

#endif
Commit	Line	Data
caab277b	1	/* SPDX-License-Identifier: GPL-2.0-only */
53631b54 CM	2	/*
53631b54 CM	3	* Copyright (C) 2012 ARM Ltd.
53631b54 CM	4	*/
	5	#ifndef __ASM_FP_H
	6	#define __ASM_FP_H
	7
2d2123bc	8	#include <asm/errno.h>
f9209e26	9	#include <asm/ptrace.h>
9a6e5948 DM	10	#include <asm/processor.h>
9a6e5948 DM	11	#include <asm/sigcontext.h>
f9209e26	12	#include <asm/sysreg.h>
53631b54 CM	13
	14	#ifndef __ASSEMBLY__
	15
ead9e430	16	#include <linux/bitmap.h>
f9209e26	17	#include <linux/build_bug.h>
ead9e430	18	#include <linux/bug.h>
7582e220	19	#include <linux/cache.h>
b4f9b390	20	#include <linux/init.h>
bc0ee476	21	#include <linux/stddef.h>
ead9e430	22	#include <linux/types.h>
bc0ee476	23
b907b80d	24	#ifdef CONFIG_COMPAT
53631b54 CM	25	/* Masks for extracting the FPSR and FPCR from the FPSCR */
	26	#define VFP_FPSCR_STAT_MASK 0xf800009f
	27	#define VFP_FPSCR_CTRL_MASK 0x07f79f00
	28	/*
	29	* The VFP state has 32x64-bit registers and a single 32-bit
	30	* control/status register.
	31	*/
	32	#define VFP_STATE_SIZE ((32 * 8) + 4)
	33	#endif
	34
bc9bbb78 MR	35	static inline unsigned long cpacr_save_enable_kernel_sve(void)
	36	{
	37	unsigned long old = read_sysreg(cpacr_el1);
	38	unsigned long set = CPACR_EL1_FPEN_EL1EN \| CPACR_EL1_ZEN_EL1EN;
	39
	40	write_sysreg(old \| set, cpacr_el1);
	41	isb();
	42	return old;
	43	}
	44
	45	static inline unsigned long cpacr_save_enable_kernel_sme(void)
	46	{
	47	unsigned long old = read_sysreg(cpacr_el1);
	48	unsigned long set = CPACR_EL1_FPEN_EL1EN \| CPACR_EL1_SMEN_EL1EN;
	49
	50	write_sysreg(old \| set, cpacr_el1);
	51	isb();
	52	return old;
	53	}
	54
	55	static inline void cpacr_restore(unsigned long cpacr)
	56	{
	57	write_sysreg(cpacr, cpacr_el1);
	58	isb();
	59	}
	60
d158a060 MB	61	/*
	62	* When we defined the maximum SVE vector length we defined the ABI so
	63	* that the maximum vector length included all the reserved for future
	64	* expansion bits in ZCR rather than those just currently defined by
28139262 MB	65	* the architecture. Using this length to allocate worst size buffers
	66	* results in excessively large allocations, and this effect is even
	67	* more pronounced for SME due to ZA. Define more suitable VLs for
	68	* these situations.
d158a060	69	*/
28139262 MB	70	#define ARCH_SVE_VQ_MAX ((ZCR_ELx_LEN_MASK >> ZCR_ELx_LEN_SHIFT) + 1)
28139262 MB	71	#define SME_VQ_MAX ((SMCR_ELx_LEN_MASK >> SMCR_ELx_LEN_SHIFT) + 1)
d158a060	72
53631b54 CM	73	struct task_struct;
53631b54 CM	74
20b85472 DM	75	extern void fpsimd_save_state(struct user_fpsimd_state *state);
20b85472 DM	76	extern void fpsimd_load_state(struct user_fpsimd_state *state);
53631b54 CM	77
	78	extern void fpsimd_thread_switch(struct task_struct *next);
	79	extern void fpsimd_flush_thread(void);
	80
8cd969d2	81	extern void fpsimd_signal_preserve_current_state(void);
c51f9269	82	extern void fpsimd_preserve_current_state(void);
005f78cd	83	extern void fpsimd_restore_current_state(void);
0abdeff5	84	extern void fpsimd_update_current_state(struct user_fpsimd_state const *state);
93ae6b01	85	extern void fpsimd_kvm_prepare(void);
c51f9269	86
1192b93b MB	87	struct cpu_fp_state {
	88	struct user_fpsimd_state *st;
	89	void *sve_state;
ce514000	90	void *sme_state;
1192b93b MB	91	u64 *svcr;
	92	unsigned int sve_vl;
	93	unsigned int sme_vl;
	94	enum fp_type *fp_type;
	95	enum fp_type to_save;
	96	};
	97
	98	extern void fpsimd_bind_state_to_cpu(struct cpu_fp_state *fp_state);
e6b673b7	99
005f78cd	100	extern void fpsimd_flush_task_state(struct task_struct *target);
54b8c7cb	101	extern void fpsimd_save_and_flush_cpu_state(void);
005f78cd	102
af7167d6 MB	103	static inline bool thread_sm_enabled(struct thread_struct *thread)
af7167d6 MB	104	{
ec0067a6	105	return system_supports_sme() && (thread->svcr & SVCR_SM_MASK);
af7167d6 MB	106	}
	107
	108	static inline bool thread_za_enabled(struct thread_struct *thread)
	109	{
ec0067a6	110	return system_supports_sme() && (thread->svcr & SVCR_ZA_MASK);
af7167d6 MB	111	}
af7167d6 MB	112
30c43e73 MB	113	/* Maximum VL that SVE/SME VL-agnostic software can transparently support */
30c43e73 MB	114	#define VL_ARCH_MAX 0x100
7582e220	115
9a6e5948 DM	116	/* Offset of FFR in the SVE register dump */
	117	static inline size_t sve_ffr_offset(int vl)
	118	{
	119	return SVE_SIG_FFR_OFFSET(sve_vq_from_vl(vl)) - SVE_SIG_REGS_OFFSET;
	120	}
	121
	122	static inline void sve_pffr(struct thread_struct thread)
	123	{
af7167d6 MB	124	unsigned int vl;
	125
	126	if (system_supports_sme() && thread_sm_enabled(thread))
	127	vl = thread_get_sme_vl(thread);
	128	else
	129	vl = thread_get_sve_vl(thread);
	130
	131	return (char *)thread->sve_state + sve_ffr_offset(vl);
9a6e5948 DM	132	}
9a6e5948 DM	133
d6138b4a MB	134	static inline void thread_zt_state(struct thread_struct thread)
	135	{
	136	/* The ZT register state is stored immediately after the ZA state */
	137	unsigned int sme_vq = sve_vq_from_vl(thread_get_sme_vl(thread));
	138	return thread->sme_state + ZA_SIG_REGS_SIZE(sme_vq);
	139	}
	140
9f584866	141	extern void sve_save_state(void state, u32 pfpsr, int save_ffr);
1fc5dce7	142	extern void sve_load_state(void const state, u32 const pfpsr,
ddc806b5	143	int restore_ffr);
9f584866	144	extern void sve_flush_live(bool flush_ffr, unsigned long vq_minus_1);
1fc5dce7	145	extern unsigned int sve_get_vl(void);
cccb78ce	146	extern void sve_set_vq(unsigned long vq_minus_1);
af7167d6	147	extern void sme_set_vq(unsigned long vq_minus_1);
95fcec71 MB	148	extern void sme_save_state(void *state, int zt);
95fcec71 MB	149	extern void sme_load_state(void const *state, int zt);
c0cda3b8 DM	150
c0cda3b8 DM	151	struct arm64_cpu_capabilities;
34f66c4c	152	extern void cpu_enable_fpsimd(const struct arm64_cpu_capabilities *__unused);
14567ba4 MR	153	extern void cpu_enable_sve(const struct arm64_cpu_capabilities *__unused);
	154	extern void cpu_enable_sme(const struct arm64_cpu_capabilities *__unused);
	155	extern void cpu_enable_sme2(const struct arm64_cpu_capabilities *__unused);
	156	extern void cpu_enable_fa64(const struct arm64_cpu_capabilities *__unused);
1fc5dce7	157
b42990d3	158	extern u64 read_smcr_features(void);
31dc52b3	159
ead9e430 DM	160	/*
	161	* Helpers to translate bit indices in sve_vq_map to VQ values (and
	162	* vice versa). This allows find_next_bit() to be used to find the
	163	* _maximum_ VQ not exceeding a certain value.
	164	*/
	165	static inline unsigned int __vq_to_bit(unsigned int vq)
	166	{
	167	return SVE_VQ_MAX - vq;
	168	}
	169
	170	static inline unsigned int __bit_to_vq(unsigned int bit)
	171	{
ead9e430 DM	172	return SVE_VQ_MAX - bit;
	173	}
	174
b5bc00ff MB	175
	176	struct vl_info {
	177	enum vec_type type;
	178	const char name; / For display purposes */
	179
	180	/* Minimum supported vector length across all CPUs */
	181	int min_vl;
	182
	183	/* Maximum supported vector length across all CPUs */
	184	int max_vl;
	185	int max_virtualisable_vl;
	186
	187	/*
	188	* Set of available vector lengths,
	189	* where length vq encoded as bit __vq_to_bit(vq):
	190	*/
	191	DECLARE_BITMAP(vq_map, SVE_VQ_MAX);
	192
	193	/* Set of vector lengths present on at least one cpu: */
	194	DECLARE_BITMAP(vq_partial_map, SVE_VQ_MAX);
	195	};
7582e220	196
bc0ee476 DM	197	#ifdef CONFIG_ARM64_SVE
bc0ee476 DM	198
826a4fdd	199	extern void sve_alloc(struct task_struct *task, bool flush);
bc0ee476	200	extern void fpsimd_release_task(struct task_struct *task);
43d4da2c	201	extern void fpsimd_sync_to_sve(struct task_struct *task);
e12310a0	202	extern void fpsimd_force_sync_to_sve(struct task_struct *task);
43d4da2c DM	203	extern void sve_sync_to_fpsimd(struct task_struct *task);
	204	extern void sve_sync_from_fpsimd_zeropad(struct task_struct *task);
	205
30c43e73	206	extern int vec_set_vector_length(struct task_struct *task, enum vec_type type,
7582e220	207	unsigned long vl, unsigned long flags);
bc0ee476	208
2d2123bc DM	209	extern int sve_set_current_vl(unsigned long arg);
	210	extern int sve_get_current_vl(void);
	211
f9209e26 MR	212	static inline void sve_user_disable(void)
	213	{
	214	sysreg_clear_set(cpacr_el1, CPACR_EL1_ZEN_EL0EN, 0);
	215	}
	216
	217	static inline void sve_user_enable(void)
	218	{
	219	sysreg_clear_set(cpacr_el1, 0, CPACR_EL1_ZEN_EL0EN);
	220	}
	221
71ce1ae5 MZ	222	#define sve_cond_update_zcr_vq(val, reg) \
	223	do { \
	224	u64 __zcr = read_sysreg_s((reg)); \
	225	u64 __new = __zcr & ~ZCR_ELx_LEN_MASK; \
	226	__new \|= (val) & ZCR_ELx_LEN_MASK; \
	227	if (__zcr != __new) \
	228	write_sysreg_s(__new, (reg)); \
	229	} while (0)
	230
2e0f2478 DM	231	/*
	232	* Probing and setup functions.
	233	* Calls to these functions must be serialised with one another.
	234	*/
b5bc00ff MB	235	enum vec_type;
	236
	237	extern void __init vec_init_vq_map(enum vec_type type);
	238	extern void vec_update_vq_map(enum vec_type type);
	239	extern int vec_verify_vq_map(enum vec_type type);
2e0f2478 DM	240	extern void __init sve_setup(void);
2e0f2478 DM	241
b5bc00ff MB	242	extern __ro_after_init struct vl_info vl_info[ARM64_VEC_MAX];
	243
	244	static inline void write_vl(enum vec_type type, u64 val)
	245	{
	246	u64 tmp;
	247
	248	switch (type) {
	249	#ifdef CONFIG_ARM64_SVE
	250	case ARM64_VEC_SVE:
	251	tmp = read_sysreg_s(SYS_ZCR_EL1) & ~ZCR_ELx_LEN_MASK;
	252	write_sysreg_s(tmp \| val, SYS_ZCR_EL1);
	253	break;
b42990d3 MB	254	#endif
	255	#ifdef CONFIG_ARM64_SME
	256	case ARM64_VEC_SME:
	257	tmp = read_sysreg_s(SYS_SMCR_EL1) & ~SMCR_ELx_LEN_MASK;
	258	write_sysreg_s(tmp \| val, SYS_SMCR_EL1);
	259	break;
b5bc00ff MB	260	#endif
	261	default:
	262	WARN_ON_ONCE(1);
	263	break;
	264	}
	265	}
	266
	267	static inline int vec_max_vl(enum vec_type type)
	268	{
	269	return vl_info[type].max_vl;
	270	}
	271
	272	static inline int vec_max_virtualisable_vl(enum vec_type type)
	273	{
	274	return vl_info[type].max_virtualisable_vl;
	275	}
	276
	277	static inline int sve_max_vl(void)
	278	{
	279	return vec_max_vl(ARM64_VEC_SVE);
	280	}
	281
	282	static inline int sve_max_virtualisable_vl(void)
	283	{
	284	return vec_max_virtualisable_vl(ARM64_VEC_SVE);
	285	}
	286
	287	/* Ensure vq >= SVE_VQ_MIN && vq <= SVE_VQ_MAX before calling this function */
	288	static inline bool vq_available(enum vec_type type, unsigned int vq)
	289	{
	290	return test_bit(__vq_to_bit(vq), vl_info[type].vq_map);
	291	}
	292
	293	static inline bool sve_vq_available(unsigned int vq)
	294	{
	295	return vq_available(ARM64_VEC_SVE, vq);
	296	}
	297
8bd7f91c MB	298	size_t sve_state_size(struct task_struct const *task);
8bd7f91c MB	299
bc0ee476 DM	300	#else /* ! CONFIG_ARM64_SVE */
bc0ee476 DM	301
826a4fdd	302	static inline void sve_alloc(struct task_struct *task, bool flush) { }
bc0ee476	303	static inline void fpsimd_release_task(struct task_struct *task) { }
43d4da2c DM	304	static inline void sve_sync_to_fpsimd(struct task_struct *task) { }
	305	static inline void sve_sync_from_fpsimd_zeropad(struct task_struct *task) { }
	306
49ed9204 MB	307	static inline int sve_max_virtualisable_vl(void)
	308	{
	309	return 0;
	310	}
	311
2d2123bc DM	312	static inline int sve_set_current_vl(unsigned long arg)
	313	{
	314	return -EINVAL;
	315	}
	316
	317	static inline int sve_get_current_vl(void)
	318	{
	319	return -EINVAL;
	320	}
	321
b5bc00ff MB	322	static inline int sve_max_vl(void)
	323	{
	324	return -EINVAL;
	325	}
	326
	327	static inline bool sve_vq_available(unsigned int vq) { return false; }
	328
f9209e26 MR	329	static inline void sve_user_disable(void) { BUILD_BUG(); }
	330	static inline void sve_user_enable(void) { BUILD_BUG(); }
	331
a9f8696d XT	332	#define sve_cond_update_zcr_vq(val, reg) do { } while (0)
a9f8696d XT	333
b5bc00ff MB	334	static inline void vec_init_vq_map(enum vec_type t) { }
	335	static inline void vec_update_vq_map(enum vec_type t) { }
	336	static inline int vec_verify_vq_map(enum vec_type t) { return 0; }
2e0f2478	337	static inline void sve_setup(void) { }
bc0ee476	338
8bd7f91c MB	339	static inline size_t sve_state_size(struct task_struct const *task)
	340	{
	341	return 0;
	342	}
	343
bc0ee476 DM	344	#endif /* ! CONFIG_ARM64_SVE */
bc0ee476 DM	345
ca8a4ebc MB	346	#ifdef CONFIG_ARM64_SME
ca8a4ebc MB	347
8bd7f91c MB	348	static inline void sme_user_disable(void)
	349	{
	350	sysreg_clear_set(cpacr_el1, CPACR_EL1_SMEN_EL0EN, 0);
	351	}
	352
	353	static inline void sme_user_enable(void)
	354	{
	355	sysreg_clear_set(cpacr_el1, 0, CPACR_EL1_SMEN_EL0EN);
	356	}
	357
ca8a4ebc MB	358	static inline void sme_smstart_sm(void)
	359	{
	360	asm volatile(__msr_s(SYS_SVCR_SMSTART_SM_EL0, "xzr"));
	361	}
	362
	363	static inline void sme_smstop_sm(void)
	364	{
	365	asm volatile(__msr_s(SYS_SVCR_SMSTOP_SM_EL0, "xzr"));
	366	}
	367
	368	static inline void sme_smstop(void)
	369	{
	370	asm volatile(__msr_s(SYS_SVCR_SMSTOP_SMZA_EL0, "xzr"));
	371	}
	372
b42990d3 MB	373	extern void __init sme_setup(void);
	374
	375	static inline int sme_max_vl(void)
	376	{
	377	return vec_max_vl(ARM64_VEC_SME);
	378	}
	379
	380	static inline int sme_max_virtualisable_vl(void)
	381	{
	382	return vec_max_virtualisable_vl(ARM64_VEC_SME);
	383	}
	384
5d0a8d2f	385	extern void sme_alloc(struct task_struct *task, bool flush);
b42990d3	386	extern unsigned int sme_get_vl(void);
9e4ab6c8 MB	387	extern int sme_set_current_vl(unsigned long arg);
9e4ab6c8 MB	388	extern int sme_get_current_vl(void);
95338648	389	extern void sme_suspend_exit(void);
b42990d3	390
8bd7f91c MB	391	/*
8bd7f91c MB	392	* Return how many bytes of memory are required to store the full SME
ce514000 MB	393	* specific state for task, given task's currently configured vector
ce514000 MB	394	* length.
8bd7f91c	395	*/
ce514000	396	static inline size_t sme_state_size(struct task_struct const *task)
8bd7f91c MB	397	{
8bd7f91c MB	398	unsigned int vl = task_get_sme_vl(task);
ce514000	399	size_t size;
8bd7f91c	400
ce514000 MB	401	size = ZA_SIG_REGS_SIZE(sve_vq_from_vl(vl));
ce514000 MB	402
d6138b4a MB	403	if (system_supports_sme2())
	404	size += ZT_SIG_REG_SIZE;
	405
ce514000	406	return size;
8bd7f91c MB	407	}
8bd7f91c MB	408
ca8a4ebc MB	409	#else
ca8a4ebc MB	410
8bd7f91c MB	411	static inline void sme_user_disable(void) { BUILD_BUG(); }
	412	static inline void sme_user_enable(void) { BUILD_BUG(); }
	413
ca8a4ebc MB	414	static inline void sme_smstart_sm(void) { }
	415	static inline void sme_smstop_sm(void) { }
	416	static inline void sme_smstop(void) { }
	417
5d0a8d2f	418	static inline void sme_alloc(struct task_struct *task, bool flush) { }
b42990d3 MB	419	static inline void sme_setup(void) { }
	420	static inline unsigned int sme_get_vl(void) { return 0; }
	421	static inline int sme_max_vl(void) { return 0; }
	422	static inline int sme_max_virtualisable_vl(void) { return 0; }
9e4ab6c8 MB	423	static inline int sme_set_current_vl(unsigned long arg) { return -EINVAL; }
9e4ab6c8 MB	424	static inline int sme_get_current_vl(void) { return -EINVAL; }
95338648	425	static inline void sme_suspend_exit(void) { }
b42990d3	426
ce514000	427	static inline size_t sme_state_size(struct task_struct const *task)
8bd7f91c MB	428	{
	429	return 0;
	430	}
	431
ca8a4ebc MB	432	#endif /* ! CONFIG_ARM64_SME */
ca8a4ebc MB	433
4328825d DM	434	/* For use by EFI runtime services calls only */
	435	extern void __efi_fpsimd_begin(void);
	436	extern void __efi_fpsimd_end(void);
	437
53631b54 CM	438	#endif
	439
	440	#endif