[linux-block.git] / arch / arm64 / include / asm / fpsimd.h

/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * Copyright (C) 2012 ARM Ltd.
 */
#ifndef __ASM_FP_H
#define __ASM_FP_H

#include <asm/errno.h>
#include <asm/ptrace.h>
#include <asm/processor.h>
#include <asm/sigcontext.h>
#include <asm/sysreg.h>

#ifndef __ASSEMBLY__

#include <linux/bitmap.h>
#include <linux/build_bug.h>
#include <linux/bug.h>
#include <linux/cache.h>
#include <linux/init.h>
#include <linux/stddef.h>
#include <linux/types.h>

#ifdef CONFIG_COMPAT
/* Masks for extracting the FPSR and FPCR from the FPSCR */
#define VFP_FPSCR_STAT_MASK	0xf800009f
#define VFP_FPSCR_CTRL_MASK	0x07f79f00
/*
 * The VFP state has 32x64-bit registers and a single 32-bit
 * control/status register.
 */
#define VFP_STATE_SIZE		((32 * 8) + 4)
#endif

/*
 * When we defined the maximum SVE vector length we defined the ABI so
 * that the maximum vector length included all the reserved for future
 * expansion bits in ZCR rather than those just currently defined by
 * the architecture. While SME follows a similar pattern the fact that
 * it includes a square matrix means that any allocations that attempt
 * to cover the maximum potential vector length (such as happen with
 * the regset used for ptrace) end up being extremely large. Define
 * the much lower actual limit for use in such situations.
 */
#define SME_VQ_MAX	16

struct task_struct;

extern void fpsimd_save_state(struct user_fpsimd_state *state);
extern void fpsimd_load_state(struct user_fpsimd_state *state);

extern void fpsimd_thread_switch(struct task_struct *next);
extern void fpsimd_flush_thread(void);

extern void fpsimd_signal_preserve_current_state(void);
extern void fpsimd_preserve_current_state(void);
extern void fpsimd_restore_current_state(void);
extern void fpsimd_update_current_state(struct user_fpsimd_state const *state);

extern void fpsimd_bind_state_to_cpu(struct user_fpsimd_state *state,
				     void *sve_state, unsigned int sve_vl,
				     void *za_state, unsigned int sme_vl,
				     u64 *svcr);

extern void fpsimd_flush_task_state(struct task_struct *target);
extern void fpsimd_save_and_flush_cpu_state(void);

static inline bool thread_sm_enabled(struct thread_struct *thread)
{
	return system_supports_sme() && (thread->svcr & SVCR_SM_MASK);
}

static inline bool thread_za_enabled(struct thread_struct *thread)
{
	return system_supports_sme() && (thread->svcr & SVCR_ZA_MASK);
}

/* Maximum VL that SVE/SME VL-agnostic software can transparently support */
#define VL_ARCH_MAX 0x100

/* Offset of FFR in the SVE register dump */
static inline size_t sve_ffr_offset(int vl)
{
	return SVE_SIG_FFR_OFFSET(sve_vq_from_vl(vl)) - SVE_SIG_REGS_OFFSET;
}

static inline void *sve_pffr(struct thread_struct *thread)
{
	unsigned int vl;

	if (system_supports_sme() && thread_sm_enabled(thread))
		vl = thread_get_sme_vl(thread);
	else
		vl = thread_get_sve_vl(thread);

	return (char *)thread->sve_state + sve_ffr_offset(vl);
}

extern void sve_save_state(void *state, u32 *pfpsr, int save_ffr);
extern void sve_load_state(void const *state, u32 const *pfpsr,
			   int restore_ffr);
extern void sve_flush_live(bool flush_ffr, unsigned long vq_minus_1);
extern unsigned int sve_get_vl(void);
extern void sve_set_vq(unsigned long vq_minus_1);
extern void sme_set_vq(unsigned long vq_minus_1);
extern void za_save_state(void *state);
extern void za_load_state(void const *state);

struct arm64_cpu_capabilities;
extern void sve_kernel_enable(const struct arm64_cpu_capabilities *__unused);
extern void sme_kernel_enable(const struct arm64_cpu_capabilities *__unused);
extern void fa64_kernel_enable(const struct arm64_cpu_capabilities *__unused);

extern u64 read_zcr_features(void);
extern u64 read_smcr_features(void);

/*
 * Helpers to translate bit indices in sve_vq_map to VQ values (and
 * vice versa).  This allows find_next_bit() to be used to find the
 * _maximum_ VQ not exceeding a certain value.
 */
static inline unsigned int __vq_to_bit(unsigned int vq)
{
	return SVE_VQ_MAX - vq;
}

static inline unsigned int __bit_to_vq(unsigned int bit)
{
	return SVE_VQ_MAX - bit;
}


struct vl_info {
	enum vec_type type;
	const char *name;		/* For display purposes */

	/* Minimum supported vector length across all CPUs */
	int min_vl;

	/* Maximum supported vector length across all CPUs */
	int max_vl;
	int max_virtualisable_vl;

	/*
	 * Set of available vector lengths,
	 * where length vq encoded as bit __vq_to_bit(vq):
	 */
	DECLARE_BITMAP(vq_map, SVE_VQ_MAX);

	/* Set of vector lengths present on at least one cpu: */
	DECLARE_BITMAP(vq_partial_map, SVE_VQ_MAX);
};

#ifdef CONFIG_ARM64_SVE

extern void sve_alloc(struct task_struct *task);
extern void fpsimd_release_task(struct task_struct *task);
extern void fpsimd_sync_to_sve(struct task_struct *task);
extern void fpsimd_force_sync_to_sve(struct task_struct *task);
extern void sve_sync_to_fpsimd(struct task_struct *task);
extern void sve_sync_from_fpsimd_zeropad(struct task_struct *task);

extern int vec_set_vector_length(struct task_struct *task, enum vec_type type,
				 unsigned long vl, unsigned long flags);

extern int sve_set_current_vl(unsigned long arg);
extern int sve_get_current_vl(void);

static inline void sve_user_disable(void)
{
	sysreg_clear_set(cpacr_el1, CPACR_EL1_ZEN_EL0EN, 0);
}

static inline void sve_user_enable(void)
{
	sysreg_clear_set(cpacr_el1, 0, CPACR_EL1_ZEN_EL0EN);
}

#define sve_cond_update_zcr_vq(val, reg)		\
	do {						\
		u64 __zcr = read_sysreg_s((reg));	\
		u64 __new = __zcr & ~ZCR_ELx_LEN_MASK;	\
		__new |= (val) & ZCR_ELx_LEN_MASK;	\
		if (__zcr != __new)			\
			write_sysreg_s(__new, (reg));	\
	} while (0)

/*
 * Probing and setup functions.
 * Calls to these functions must be serialised with one another.
 */
enum vec_type;

extern void __init vec_init_vq_map(enum vec_type type);
extern void vec_update_vq_map(enum vec_type type);
extern int vec_verify_vq_map(enum vec_type type);
extern void __init sve_setup(void);

extern __ro_after_init struct vl_info vl_info[ARM64_VEC_MAX];

static inline void write_vl(enum vec_type type, u64 val)
{
	u64 tmp;

	switch (type) {
#ifdef CONFIG_ARM64_SVE
	case ARM64_VEC_SVE:
		tmp = read_sysreg_s(SYS_ZCR_EL1) & ~ZCR_ELx_LEN_MASK;
		write_sysreg_s(tmp | val, SYS_ZCR_EL1);
		break;
#endif
#ifdef CONFIG_ARM64_SME
	case ARM64_VEC_SME:
		tmp = read_sysreg_s(SYS_SMCR_EL1) & ~SMCR_ELx_LEN_MASK;
		write_sysreg_s(tmp | val, SYS_SMCR_EL1);
		break;
#endif
	default:
		WARN_ON_ONCE(1);
		break;
	}
}

static inline int vec_max_vl(enum vec_type type)
{
	return vl_info[type].max_vl;
}

static inline int vec_max_virtualisable_vl(enum vec_type type)
{
	return vl_info[type].max_virtualisable_vl;
}

static inline int sve_max_vl(void)
{
	return vec_max_vl(ARM64_VEC_SVE);
}

static inline int sve_max_virtualisable_vl(void)
{
	return vec_max_virtualisable_vl(ARM64_VEC_SVE);
}

/* Ensure vq >= SVE_VQ_MIN && vq <= SVE_VQ_MAX before calling this function */
static inline bool vq_available(enum vec_type type, unsigned int vq)
{
	return test_bit(__vq_to_bit(vq), vl_info[type].vq_map);
}

static inline bool sve_vq_available(unsigned int vq)
{
	return vq_available(ARM64_VEC_SVE, vq);
}

size_t sve_state_size(struct task_struct const *task);

#else /* ! CONFIG_ARM64_SVE */

static inline void sve_alloc(struct task_struct *task) { }
static inline void fpsimd_release_task(struct task_struct *task) { }
static inline void sve_sync_to_fpsimd(struct task_struct *task) { }
static inline void sve_sync_from_fpsimd_zeropad(struct task_struct *task) { }

static inline int sve_max_virtualisable_vl(void)
{
	return 0;
}

static inline int sve_set_current_vl(unsigned long arg)
{
	return -EINVAL;
}

static inline int sve_get_current_vl(void)
{
	return -EINVAL;
}

static inline int sve_max_vl(void)
{
	return -EINVAL;
}

static inline bool sve_vq_available(unsigned int vq) { return false; }

static inline void sve_user_disable(void) { BUILD_BUG(); }
static inline void sve_user_enable(void) { BUILD_BUG(); }

#define sve_cond_update_zcr_vq(val, reg) do { } while (0)

static inline void vec_init_vq_map(enum vec_type t) { }
static inline void vec_update_vq_map(enum vec_type t) { }
static inline int vec_verify_vq_map(enum vec_type t) { return 0; }
static inline void sve_setup(void) { }

static inline size_t sve_state_size(struct task_struct const *task)
{
	return 0;
}

#endif /* ! CONFIG_ARM64_SVE */

#ifdef CONFIG_ARM64_SME

static inline void sme_user_disable(void)
{
	sysreg_clear_set(cpacr_el1, CPACR_EL1_SMEN_EL0EN, 0);
}

static inline void sme_user_enable(void)
{
	sysreg_clear_set(cpacr_el1, 0, CPACR_EL1_SMEN_EL0EN);
}

static inline void sme_smstart_sm(void)
{
	asm volatile(__msr_s(SYS_SVCR_SMSTART_SM_EL0, "xzr"));
}

static inline void sme_smstop_sm(void)
{
	asm volatile(__msr_s(SYS_SVCR_SMSTOP_SM_EL0, "xzr"));
}

static inline void sme_smstop(void)
{
	asm volatile(__msr_s(SYS_SVCR_SMSTOP_SMZA_EL0, "xzr"));
}

extern void __init sme_setup(void);

static inline int sme_max_vl(void)
{
	return vec_max_vl(ARM64_VEC_SME);
}

static inline int sme_max_virtualisable_vl(void)
{
	return vec_max_virtualisable_vl(ARM64_VEC_SME);
}

extern void sme_alloc(struct task_struct *task);
extern unsigned int sme_get_vl(void);
extern int sme_set_current_vl(unsigned long arg);
extern int sme_get_current_vl(void);

/*
 * Return how many bytes of memory are required to store the full SME
 * specific state (currently just ZA) for task, given task's currently
 * configured vector length.
 */
static inline size_t za_state_size(struct task_struct const *task)
{
	unsigned int vl = task_get_sme_vl(task);

	return ZA_SIG_REGS_SIZE(sve_vq_from_vl(vl));
}

#else

static inline void sme_user_disable(void) { BUILD_BUG(); }
static inline void sme_user_enable(void) { BUILD_BUG(); }

static inline void sme_smstart_sm(void) { }
static inline void sme_smstop_sm(void) { }
static inline void sme_smstop(void) { }

static inline void sme_alloc(struct task_struct *task) { }
static inline void sme_setup(void) { }
static inline unsigned int sme_get_vl(void) { return 0; }
static inline int sme_max_vl(void) { return 0; }
static inline int sme_max_virtualisable_vl(void) { return 0; }
static inline int sme_set_current_vl(unsigned long arg) { return -EINVAL; }
static inline int sme_get_current_vl(void) { return -EINVAL; }

static inline size_t za_state_size(struct task_struct const *task)
{
	return 0;
}

#endif /* ! CONFIG_ARM64_SME */

/* For use by EFI runtime services calls only */
extern void __efi_fpsimd_begin(void);
extern void __efi_fpsimd_end(void);

#endif

#endif
Commit	Line	Data
caab277b	1	/* SPDX-License-Identifier: GPL-2.0-only */
53631b54 CM	2	/*
53631b54 CM	3	* Copyright (C) 2012 ARM Ltd.
53631b54 CM	4	*/
	5	#ifndef __ASM_FP_H
	6	#define __ASM_FP_H
	7
2d2123bc	8	#include <asm/errno.h>
f9209e26	9	#include <asm/ptrace.h>
9a6e5948 DM	10	#include <asm/processor.h>
9a6e5948 DM	11	#include <asm/sigcontext.h>
f9209e26	12	#include <asm/sysreg.h>
53631b54 CM	13
	14	#ifndef __ASSEMBLY__
	15
ead9e430	16	#include <linux/bitmap.h>
f9209e26	17	#include <linux/build_bug.h>
ead9e430	18	#include <linux/bug.h>
7582e220	19	#include <linux/cache.h>
b4f9b390	20	#include <linux/init.h>
bc0ee476	21	#include <linux/stddef.h>
ead9e430	22	#include <linux/types.h>
bc0ee476	23
b907b80d	24	#ifdef CONFIG_COMPAT
53631b54 CM	25	/* Masks for extracting the FPSR and FPCR from the FPSCR */
	26	#define VFP_FPSCR_STAT_MASK 0xf800009f
	27	#define VFP_FPSCR_CTRL_MASK 0x07f79f00
	28	/*
	29	* The VFP state has 32x64-bit registers and a single 32-bit
	30	* control/status register.
	31	*/
	32	#define VFP_STATE_SIZE ((32 * 8) + 4)
	33	#endif
	34
d158a060 MB	35	/*
	36	* When we defined the maximum SVE vector length we defined the ABI so
	37	* that the maximum vector length included all the reserved for future
	38	* expansion bits in ZCR rather than those just currently defined by
	39	* the architecture. While SME follows a similar pattern the fact that
	40	* it includes a square matrix means that any allocations that attempt
	41	* to cover the maximum potential vector length (such as happen with
	42	* the regset used for ptrace) end up being extremely large. Define
	43	* the much lower actual limit for use in such situations.
	44	*/
	45	#define SME_VQ_MAX 16
	46
53631b54 CM	47	struct task_struct;
53631b54 CM	48
20b85472 DM	49	extern void fpsimd_save_state(struct user_fpsimd_state *state);
20b85472 DM	50	extern void fpsimd_load_state(struct user_fpsimd_state *state);
53631b54 CM	51
	52	extern void fpsimd_thread_switch(struct task_struct *next);
	53	extern void fpsimd_flush_thread(void);
	54
8cd969d2	55	extern void fpsimd_signal_preserve_current_state(void);
c51f9269	56	extern void fpsimd_preserve_current_state(void);
005f78cd	57	extern void fpsimd_restore_current_state(void);
0abdeff5	58	extern void fpsimd_update_current_state(struct user_fpsimd_state const *state);
c51f9269	59
04950674	60	extern void fpsimd_bind_state_to_cpu(struct user_fpsimd_state *state,
b40c559b	61	void *sve_state, unsigned int sve_vl,
0033cd93 MB	62	void *za_state, unsigned int sme_vl,
0033cd93 MB	63	u64 *svcr);
e6b673b7	64
005f78cd	65	extern void fpsimd_flush_task_state(struct task_struct *target);
54b8c7cb	66	extern void fpsimd_save_and_flush_cpu_state(void);
005f78cd	67
af7167d6 MB	68	static inline bool thread_sm_enabled(struct thread_struct *thread)
af7167d6 MB	69	{
ec0067a6	70	return system_supports_sme() && (thread->svcr & SVCR_SM_MASK);
af7167d6 MB	71	}
	72
	73	static inline bool thread_za_enabled(struct thread_struct *thread)
	74	{
ec0067a6	75	return system_supports_sme() && (thread->svcr & SVCR_ZA_MASK);
af7167d6 MB	76	}
af7167d6 MB	77
30c43e73 MB	78	/* Maximum VL that SVE/SME VL-agnostic software can transparently support */
30c43e73 MB	79	#define VL_ARCH_MAX 0x100
7582e220	80
9a6e5948 DM	81	/* Offset of FFR in the SVE register dump */
	82	static inline size_t sve_ffr_offset(int vl)
	83	{
	84	return SVE_SIG_FFR_OFFSET(sve_vq_from_vl(vl)) - SVE_SIG_REGS_OFFSET;
	85	}
	86
	87	static inline void sve_pffr(struct thread_struct thread)
	88	{
af7167d6 MB	89	unsigned int vl;
	90
	91	if (system_supports_sme() && thread_sm_enabled(thread))
	92	vl = thread_get_sme_vl(thread);
	93	else
	94	vl = thread_get_sve_vl(thread);
	95
	96	return (char *)thread->sve_state + sve_ffr_offset(vl);
9a6e5948 DM	97	}
9a6e5948 DM	98
9f584866	99	extern void sve_save_state(void state, u32 pfpsr, int save_ffr);
1fc5dce7	100	extern void sve_load_state(void const state, u32 const pfpsr,
ddc806b5	101	int restore_ffr);
9f584866	102	extern void sve_flush_live(bool flush_ffr, unsigned long vq_minus_1);
1fc5dce7	103	extern unsigned int sve_get_vl(void);
cccb78ce	104	extern void sve_set_vq(unsigned long vq_minus_1);
af7167d6	105	extern void sme_set_vq(unsigned long vq_minus_1);
0033cd93 MB	106	extern void za_save_state(void *state);
0033cd93 MB	107	extern void za_load_state(void const *state);
c0cda3b8 DM	108
	109	struct arm64_cpu_capabilities;
	110	extern void sve_kernel_enable(const struct arm64_cpu_capabilities *__unused);
5e64b862 MB	111	extern void sme_kernel_enable(const struct arm64_cpu_capabilities *__unused);
5e64b862 MB	112	extern void fa64_kernel_enable(const struct arm64_cpu_capabilities *__unused);
1fc5dce7	113
31dc52b3	114	extern u64 read_zcr_features(void);
b42990d3	115	extern u64 read_smcr_features(void);
31dc52b3	116
ead9e430 DM	117	/*
	118	* Helpers to translate bit indices in sve_vq_map to VQ values (and
	119	* vice versa). This allows find_next_bit() to be used to find the
	120	* _maximum_ VQ not exceeding a certain value.
	121	*/
	122	static inline unsigned int __vq_to_bit(unsigned int vq)
	123	{
	124	return SVE_VQ_MAX - vq;
	125	}
	126
	127	static inline unsigned int __bit_to_vq(unsigned int bit)
	128	{
ead9e430 DM	129	return SVE_VQ_MAX - bit;
	130	}
	131
b5bc00ff MB	132
	133	struct vl_info {
	134	enum vec_type type;
	135	const char name; / For display purposes */
	136
	137	/* Minimum supported vector length across all CPUs */
	138	int min_vl;
	139
	140	/* Maximum supported vector length across all CPUs */
	141	int max_vl;
	142	int max_virtualisable_vl;
	143
	144	/*
	145	* Set of available vector lengths,
	146	* where length vq encoded as bit __vq_to_bit(vq):
	147	*/
	148	DECLARE_BITMAP(vq_map, SVE_VQ_MAX);
	149
	150	/* Set of vector lengths present on at least one cpu: */
	151	DECLARE_BITMAP(vq_partial_map, SVE_VQ_MAX);
	152	};
7582e220	153
bc0ee476 DM	154	#ifdef CONFIG_ARM64_SVE
bc0ee476 DM	155
bc0ee476 DM	156	extern void sve_alloc(struct task_struct *task);
bc0ee476 DM	157	extern void fpsimd_release_task(struct task_struct *task);
43d4da2c	158	extern void fpsimd_sync_to_sve(struct task_struct *task);
e12310a0	159	extern void fpsimd_force_sync_to_sve(struct task_struct *task);
43d4da2c DM	160	extern void sve_sync_to_fpsimd(struct task_struct *task);
	161	extern void sve_sync_from_fpsimd_zeropad(struct task_struct *task);
	162
30c43e73	163	extern int vec_set_vector_length(struct task_struct *task, enum vec_type type,
7582e220	164	unsigned long vl, unsigned long flags);
bc0ee476	165
2d2123bc DM	166	extern int sve_set_current_vl(unsigned long arg);
	167	extern int sve_get_current_vl(void);
	168
f9209e26 MR	169	static inline void sve_user_disable(void)
	170	{
	171	sysreg_clear_set(cpacr_el1, CPACR_EL1_ZEN_EL0EN, 0);
	172	}
	173
	174	static inline void sve_user_enable(void)
	175	{
	176	sysreg_clear_set(cpacr_el1, 0, CPACR_EL1_ZEN_EL0EN);
	177	}
	178
71ce1ae5 MZ	179	#define sve_cond_update_zcr_vq(val, reg) \
	180	do { \
	181	u64 __zcr = read_sysreg_s((reg)); \
	182	u64 __new = __zcr & ~ZCR_ELx_LEN_MASK; \
	183	__new \|= (val) & ZCR_ELx_LEN_MASK; \
	184	if (__zcr != __new) \
	185	write_sysreg_s(__new, (reg)); \
	186	} while (0)
	187
2e0f2478 DM	188	/*
	189	* Probing and setup functions.
	190	* Calls to these functions must be serialised with one another.
	191	*/
b5bc00ff MB	192	enum vec_type;
	193
	194	extern void __init vec_init_vq_map(enum vec_type type);
	195	extern void vec_update_vq_map(enum vec_type type);
	196	extern int vec_verify_vq_map(enum vec_type type);
2e0f2478 DM	197	extern void __init sve_setup(void);
2e0f2478 DM	198
b5bc00ff MB	199	extern __ro_after_init struct vl_info vl_info[ARM64_VEC_MAX];
	200
	201	static inline void write_vl(enum vec_type type, u64 val)
	202	{
	203	u64 tmp;
	204
	205	switch (type) {
	206	#ifdef CONFIG_ARM64_SVE
	207	case ARM64_VEC_SVE:
	208	tmp = read_sysreg_s(SYS_ZCR_EL1) & ~ZCR_ELx_LEN_MASK;
	209	write_sysreg_s(tmp \| val, SYS_ZCR_EL1);
	210	break;
b42990d3 MB	211	#endif
	212	#ifdef CONFIG_ARM64_SME
	213	case ARM64_VEC_SME:
	214	tmp = read_sysreg_s(SYS_SMCR_EL1) & ~SMCR_ELx_LEN_MASK;
	215	write_sysreg_s(tmp \| val, SYS_SMCR_EL1);
	216	break;
b5bc00ff MB	217	#endif
	218	default:
	219	WARN_ON_ONCE(1);
	220	break;
	221	}
	222	}
	223
	224	static inline int vec_max_vl(enum vec_type type)
	225	{
	226	return vl_info[type].max_vl;
	227	}
	228
	229	static inline int vec_max_virtualisable_vl(enum vec_type type)
	230	{
	231	return vl_info[type].max_virtualisable_vl;
	232	}
	233
	234	static inline int sve_max_vl(void)
	235	{
	236	return vec_max_vl(ARM64_VEC_SVE);
	237	}
	238
	239	static inline int sve_max_virtualisable_vl(void)
	240	{
	241	return vec_max_virtualisable_vl(ARM64_VEC_SVE);
	242	}
	243
	244	/* Ensure vq >= SVE_VQ_MIN && vq <= SVE_VQ_MAX before calling this function */
	245	static inline bool vq_available(enum vec_type type, unsigned int vq)
	246	{
	247	return test_bit(__vq_to_bit(vq), vl_info[type].vq_map);
	248	}
	249
	250	static inline bool sve_vq_available(unsigned int vq)
	251	{
	252	return vq_available(ARM64_VEC_SVE, vq);
	253	}
	254
8bd7f91c MB	255	size_t sve_state_size(struct task_struct const *task);
8bd7f91c MB	256
bc0ee476 DM	257	#else /* ! CONFIG_ARM64_SVE */
	258
	259	static inline void sve_alloc(struct task_struct *task) { }
	260	static inline void fpsimd_release_task(struct task_struct *task) { }
43d4da2c DM	261	static inline void sve_sync_to_fpsimd(struct task_struct *task) { }
	262	static inline void sve_sync_from_fpsimd_zeropad(struct task_struct *task) { }
	263
49ed9204 MB	264	static inline int sve_max_virtualisable_vl(void)
	265	{
	266	return 0;
	267	}
	268
2d2123bc DM	269	static inline int sve_set_current_vl(unsigned long arg)
	270	{
	271	return -EINVAL;
	272	}
	273
	274	static inline int sve_get_current_vl(void)
	275	{
	276	return -EINVAL;
	277	}
	278
b5bc00ff MB	279	static inline int sve_max_vl(void)
	280	{
	281	return -EINVAL;
	282	}
	283
	284	static inline bool sve_vq_available(unsigned int vq) { return false; }
	285
f9209e26 MR	286	static inline void sve_user_disable(void) { BUILD_BUG(); }
	287	static inline void sve_user_enable(void) { BUILD_BUG(); }
	288
a9f8696d XT	289	#define sve_cond_update_zcr_vq(val, reg) do { } while (0)
a9f8696d XT	290
b5bc00ff MB	291	static inline void vec_init_vq_map(enum vec_type t) { }
	292	static inline void vec_update_vq_map(enum vec_type t) { }
	293	static inline int vec_verify_vq_map(enum vec_type t) { return 0; }
2e0f2478	294	static inline void sve_setup(void) { }
bc0ee476	295
8bd7f91c MB	296	static inline size_t sve_state_size(struct task_struct const *task)
	297	{
	298	return 0;
	299	}
	300
bc0ee476 DM	301	#endif /* ! CONFIG_ARM64_SVE */
bc0ee476 DM	302
ca8a4ebc MB	303	#ifdef CONFIG_ARM64_SME
ca8a4ebc MB	304
8bd7f91c MB	305	static inline void sme_user_disable(void)
	306	{
	307	sysreg_clear_set(cpacr_el1, CPACR_EL1_SMEN_EL0EN, 0);
	308	}
	309
	310	static inline void sme_user_enable(void)
	311	{
	312	sysreg_clear_set(cpacr_el1, 0, CPACR_EL1_SMEN_EL0EN);
	313	}
	314
ca8a4ebc MB	315	static inline void sme_smstart_sm(void)
	316	{
	317	asm volatile(__msr_s(SYS_SVCR_SMSTART_SM_EL0, "xzr"));
	318	}
	319
	320	static inline void sme_smstop_sm(void)
	321	{
	322	asm volatile(__msr_s(SYS_SVCR_SMSTOP_SM_EL0, "xzr"));
	323	}
	324
	325	static inline void sme_smstop(void)
	326	{
	327	asm volatile(__msr_s(SYS_SVCR_SMSTOP_SMZA_EL0, "xzr"));
	328	}
	329
b42990d3 MB	330	extern void __init sme_setup(void);
	331
	332	static inline int sme_max_vl(void)
	333	{
	334	return vec_max_vl(ARM64_VEC_SME);
	335	}
	336
	337	static inline int sme_max_virtualisable_vl(void)
	338	{
	339	return vec_max_virtualisable_vl(ARM64_VEC_SME);
	340	}
	341
8bd7f91c	342	extern void sme_alloc(struct task_struct *task);
b42990d3	343	extern unsigned int sme_get_vl(void);
9e4ab6c8 MB	344	extern int sme_set_current_vl(unsigned long arg);
9e4ab6c8 MB	345	extern int sme_get_current_vl(void);
b42990d3	346
8bd7f91c MB	347	/*
	348	* Return how many bytes of memory are required to store the full SME
	349	* specific state (currently just ZA) for task, given task's currently
	350	* configured vector length.
	351	*/
	352	static inline size_t za_state_size(struct task_struct const *task)
	353	{
	354	unsigned int vl = task_get_sme_vl(task);
	355
	356	return ZA_SIG_REGS_SIZE(sve_vq_from_vl(vl));
	357	}
	358
ca8a4ebc MB	359	#else
ca8a4ebc MB	360
8bd7f91c MB	361	static inline void sme_user_disable(void) { BUILD_BUG(); }
	362	static inline void sme_user_enable(void) { BUILD_BUG(); }
	363
ca8a4ebc MB	364	static inline void sme_smstart_sm(void) { }
	365	static inline void sme_smstop_sm(void) { }
	366	static inline void sme_smstop(void) { }
	367
8bd7f91c	368	static inline void sme_alloc(struct task_struct *task) { }
b42990d3 MB	369	static inline void sme_setup(void) { }
	370	static inline unsigned int sme_get_vl(void) { return 0; }
	371	static inline int sme_max_vl(void) { return 0; }
	372	static inline int sme_max_virtualisable_vl(void) { return 0; }
9e4ab6c8 MB	373	static inline int sme_set_current_vl(unsigned long arg) { return -EINVAL; }
9e4ab6c8 MB	374	static inline int sme_get_current_vl(void) { return -EINVAL; }
b42990d3	375
8bd7f91c MB	376	static inline size_t za_state_size(struct task_struct const *task)
	377	{
	378	return 0;
	379	}
	380
ca8a4ebc MB	381	#endif /* ! CONFIG_ARM64_SME */
ca8a4ebc MB	382
4328825d DM	383	/* For use by EFI runtime services calls only */
	384	extern void __efi_fpsimd_begin(void);
	385	extern void __efi_fpsimd_end(void);
	386
53631b54 CM	387	#endif
	388
	389	#endif