[linux-block.git] / arch / arm64 / include / asm / fpsimd.h

/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * Copyright (C) 2012 ARM Ltd.
 */
#ifndef __ASM_FP_H
#define __ASM_FP_H

#include <asm/errno.h>
#include <asm/ptrace.h>
#include <asm/processor.h>
#include <asm/sigcontext.h>
#include <asm/sysreg.h>

#ifndef __ASSEMBLY__

#include <linux/bitmap.h>
#include <linux/build_bug.h>
#include <linux/bug.h>
#include <linux/cache.h>
#include <linux/init.h>
#include <linux/stddef.h>
#include <linux/types.h>

#ifdef CONFIG_COMPAT
/* Masks for extracting the FPSR and FPCR from the FPSCR */
#define VFP_FPSCR_STAT_MASK	0xf800009f
#define VFP_FPSCR_CTRL_MASK	0x07f79f00
/*
 * The VFP state has 32x64-bit registers and a single 32-bit
 * control/status register.
 */
#define VFP_STATE_SIZE		((32 * 8) + 4)
#endif

/*
 * When we defined the maximum SVE vector length we defined the ABI so
 * that the maximum vector length included all the reserved for future
 * expansion bits in ZCR rather than those just currently defined by
 * the architecture. While SME follows a similar pattern the fact that
 * it includes a square matrix means that any allocations that attempt
 * to cover the maximum potential vector length (such as happen with
 * the regset used for ptrace) end up being extremely large. Define
 * the much lower actual limit for use in such situations.
 */
#define SME_VQ_MAX	16

struct task_struct;

extern void fpsimd_save_state(struct user_fpsimd_state *state);
extern void fpsimd_load_state(struct user_fpsimd_state *state);

extern void fpsimd_thread_switch(struct task_struct *next);
extern void fpsimd_flush_thread(void);

extern void fpsimd_signal_preserve_current_state(void);
extern void fpsimd_preserve_current_state(void);
extern void fpsimd_restore_current_state(void);
extern void fpsimd_update_current_state(struct user_fpsimd_state const *state);
extern void fpsimd_kvm_prepare(void);

struct cpu_fp_state {
	struct user_fpsimd_state *st;
	void *sve_state;
	void *sme_state;
	u64 *svcr;
	unsigned int sve_vl;
	unsigned int sme_vl;
	enum fp_type *fp_type;
	enum fp_type to_save;
};

extern void fpsimd_bind_state_to_cpu(struct cpu_fp_state *fp_state);

extern void fpsimd_flush_task_state(struct task_struct *target);
extern void fpsimd_save_and_flush_cpu_state(void);

static inline bool thread_sm_enabled(struct thread_struct *thread)
{
	return system_supports_sme() && (thread->svcr & SVCR_SM_MASK);
}

static inline bool thread_za_enabled(struct thread_struct *thread)
{
	return system_supports_sme() && (thread->svcr & SVCR_ZA_MASK);
}

/* Maximum VL that SVE/SME VL-agnostic software can transparently support */
#define VL_ARCH_MAX 0x100

/* Offset of FFR in the SVE register dump */
static inline size_t sve_ffr_offset(int vl)
{
	return SVE_SIG_FFR_OFFSET(sve_vq_from_vl(vl)) - SVE_SIG_REGS_OFFSET;
}

static inline void *sve_pffr(struct thread_struct *thread)
{
	unsigned int vl;

	if (system_supports_sme() && thread_sm_enabled(thread))
		vl = thread_get_sme_vl(thread);
	else
		vl = thread_get_sve_vl(thread);

	return (char *)thread->sve_state + sve_ffr_offset(vl);
}

static inline void *thread_zt_state(struct thread_struct *thread)
{
	/* The ZT register state is stored immediately after the ZA state */
	unsigned int sme_vq = sve_vq_from_vl(thread_get_sme_vl(thread));
	return thread->sme_state + ZA_SIG_REGS_SIZE(sme_vq);
}

extern void sve_save_state(void *state, u32 *pfpsr, int save_ffr);
extern void sve_load_state(void const *state, u32 const *pfpsr,
			   int restore_ffr);
extern void sve_flush_live(bool flush_ffr, unsigned long vq_minus_1);
extern unsigned int sve_get_vl(void);
extern void sve_set_vq(unsigned long vq_minus_1);
extern void sme_set_vq(unsigned long vq_minus_1);
extern void sme_save_state(void *state, int zt);
extern void sme_load_state(void const *state, int zt);

struct arm64_cpu_capabilities;
extern void sve_kernel_enable(const struct arm64_cpu_capabilities *__unused);
extern void sme_kernel_enable(const struct arm64_cpu_capabilities *__unused);
extern void sme2_kernel_enable(const struct arm64_cpu_capabilities *__unused);
extern void fa64_kernel_enable(const struct arm64_cpu_capabilities *__unused);

extern u64 read_zcr_features(void);
extern u64 read_smcr_features(void);

/*
 * Helpers to translate bit indices in sve_vq_map to VQ values (and
 * vice versa).  This allows find_next_bit() to be used to find the
 * _maximum_ VQ not exceeding a certain value.
 */
static inline unsigned int __vq_to_bit(unsigned int vq)
{
	return SVE_VQ_MAX - vq;
}

static inline unsigned int __bit_to_vq(unsigned int bit)
{
	return SVE_VQ_MAX - bit;
}


struct vl_info {
	enum vec_type type;
	const char *name;		/* For display purposes */

	/* Minimum supported vector length across all CPUs */
	int min_vl;

	/* Maximum supported vector length across all CPUs */
	int max_vl;
	int max_virtualisable_vl;

	/*
	 * Set of available vector lengths,
	 * where length vq encoded as bit __vq_to_bit(vq):
	 */
	DECLARE_BITMAP(vq_map, SVE_VQ_MAX);

	/* Set of vector lengths present on at least one cpu: */
	DECLARE_BITMAP(vq_partial_map, SVE_VQ_MAX);
};

#ifdef CONFIG_ARM64_SVE

extern void sve_alloc(struct task_struct *task, bool flush);
extern void fpsimd_release_task(struct task_struct *task);
extern void fpsimd_sync_to_sve(struct task_struct *task);
extern void fpsimd_force_sync_to_sve(struct task_struct *task);
extern void sve_sync_to_fpsimd(struct task_struct *task);
extern void sve_sync_from_fpsimd_zeropad(struct task_struct *task);

extern int vec_set_vector_length(struct task_struct *task, enum vec_type type,
				 unsigned long vl, unsigned long flags);

extern int sve_set_current_vl(unsigned long arg);
extern int sve_get_current_vl(void);

static inline void sve_user_disable(void)
{
	sysreg_clear_set(cpacr_el1, CPACR_EL1_ZEN_EL0EN, 0);
}

static inline void sve_user_enable(void)
{
	sysreg_clear_set(cpacr_el1, 0, CPACR_EL1_ZEN_EL0EN);
}

#define sve_cond_update_zcr_vq(val, reg)		\
	do {						\
		u64 __zcr = read_sysreg_s((reg));	\
		u64 __new = __zcr & ~ZCR_ELx_LEN_MASK;	\
		__new |= (val) & ZCR_ELx_LEN_MASK;	\
		if (__zcr != __new)			\
			write_sysreg_s(__new, (reg));	\
	} while (0)

/*
 * Probing and setup functions.
 * Calls to these functions must be serialised with one another.
 */
enum vec_type;

extern void __init vec_init_vq_map(enum vec_type type);
extern void vec_update_vq_map(enum vec_type type);
extern int vec_verify_vq_map(enum vec_type type);
extern void __init sve_setup(void);

extern __ro_after_init struct vl_info vl_info[ARM64_VEC_MAX];

static inline void write_vl(enum vec_type type, u64 val)
{
	u64 tmp;

	switch (type) {
#ifdef CONFIG_ARM64_SVE
	case ARM64_VEC_SVE:
		tmp = read_sysreg_s(SYS_ZCR_EL1) & ~ZCR_ELx_LEN_MASK;
		write_sysreg_s(tmp | val, SYS_ZCR_EL1);
		break;
#endif
#ifdef CONFIG_ARM64_SME
	case ARM64_VEC_SME:
		tmp = read_sysreg_s(SYS_SMCR_EL1) & ~SMCR_ELx_LEN_MASK;
		write_sysreg_s(tmp | val, SYS_SMCR_EL1);
		break;
#endif
	default:
		WARN_ON_ONCE(1);
		break;
	}
}

static inline int vec_max_vl(enum vec_type type)
{
	return vl_info[type].max_vl;
}

static inline int vec_max_virtualisable_vl(enum vec_type type)
{
	return vl_info[type].max_virtualisable_vl;
}

static inline int sve_max_vl(void)
{
	return vec_max_vl(ARM64_VEC_SVE);
}

static inline int sve_max_virtualisable_vl(void)
{
	return vec_max_virtualisable_vl(ARM64_VEC_SVE);
}

/* Ensure vq >= SVE_VQ_MIN && vq <= SVE_VQ_MAX before calling this function */
static inline bool vq_available(enum vec_type type, unsigned int vq)
{
	return test_bit(__vq_to_bit(vq), vl_info[type].vq_map);
}

static inline bool sve_vq_available(unsigned int vq)
{
	return vq_available(ARM64_VEC_SVE, vq);
}

size_t sve_state_size(struct task_struct const *task);

#else /* ! CONFIG_ARM64_SVE */

static inline void sve_alloc(struct task_struct *task, bool flush) { }
static inline void fpsimd_release_task(struct task_struct *task) { }
static inline void sve_sync_to_fpsimd(struct task_struct *task) { }
static inline void sve_sync_from_fpsimd_zeropad(struct task_struct *task) { }

static inline int sve_max_virtualisable_vl(void)
{
	return 0;
}

static inline int sve_set_current_vl(unsigned long arg)
{
	return -EINVAL;
}

static inline int sve_get_current_vl(void)
{
	return -EINVAL;
}

static inline int sve_max_vl(void)
{
	return -EINVAL;
}

static inline bool sve_vq_available(unsigned int vq) { return false; }

static inline void sve_user_disable(void) { BUILD_BUG(); }
static inline void sve_user_enable(void) { BUILD_BUG(); }

#define sve_cond_update_zcr_vq(val, reg) do { } while (0)

static inline void vec_init_vq_map(enum vec_type t) { }
static inline void vec_update_vq_map(enum vec_type t) { }
static inline int vec_verify_vq_map(enum vec_type t) { return 0; }
static inline void sve_setup(void) { }

static inline size_t sve_state_size(struct task_struct const *task)
{
	return 0;
}

#endif /* ! CONFIG_ARM64_SVE */

#ifdef CONFIG_ARM64_SME

static inline void sme_user_disable(void)
{
	sysreg_clear_set(cpacr_el1, CPACR_EL1_SMEN_EL0EN, 0);
}

static inline void sme_user_enable(void)
{
	sysreg_clear_set(cpacr_el1, 0, CPACR_EL1_SMEN_EL0EN);
}

static inline void sme_smstart_sm(void)
{
	asm volatile(__msr_s(SYS_SVCR_SMSTART_SM_EL0, "xzr"));
}

static inline void sme_smstop_sm(void)
{
	asm volatile(__msr_s(SYS_SVCR_SMSTOP_SM_EL0, "xzr"));
}

static inline void sme_smstop(void)
{
	asm volatile(__msr_s(SYS_SVCR_SMSTOP_SMZA_EL0, "xzr"));
}

extern void __init sme_setup(void);

static inline int sme_max_vl(void)
{
	return vec_max_vl(ARM64_VEC_SME);
}

static inline int sme_max_virtualisable_vl(void)
{
	return vec_max_virtualisable_vl(ARM64_VEC_SME);
}

extern void sme_alloc(struct task_struct *task, bool flush);
extern unsigned int sme_get_vl(void);
extern int sme_set_current_vl(unsigned long arg);
extern int sme_get_current_vl(void);

/*
 * Return how many bytes of memory are required to store the full SME
 * specific state for task, given task's currently configured vector
 * length.
 */
static inline size_t sme_state_size(struct task_struct const *task)
{
	unsigned int vl = task_get_sme_vl(task);
	size_t size;

	size = ZA_SIG_REGS_SIZE(sve_vq_from_vl(vl));

	if (system_supports_sme2())
		size += ZT_SIG_REG_SIZE;

	return size;
}

#else

static inline void sme_user_disable(void) { BUILD_BUG(); }
static inline void sme_user_enable(void) { BUILD_BUG(); }

static inline void sme_smstart_sm(void) { }
static inline void sme_smstop_sm(void) { }
static inline void sme_smstop(void) { }

static inline void sme_alloc(struct task_struct *task, bool flush) { }
static inline void sme_setup(void) { }
static inline unsigned int sme_get_vl(void) { return 0; }
static inline int sme_max_vl(void) { return 0; }
static inline int sme_max_virtualisable_vl(void) { return 0; }
static inline int sme_set_current_vl(unsigned long arg) { return -EINVAL; }
static inline int sme_get_current_vl(void) { return -EINVAL; }

static inline size_t sme_state_size(struct task_struct const *task)
{
	return 0;
}

#endif /* ! CONFIG_ARM64_SME */

/* For use by EFI runtime services calls only */
extern void __efi_fpsimd_begin(void);
extern void __efi_fpsimd_end(void);

#endif

#endif
Commit	Line	Data
caab277b	1	/* SPDX-License-Identifier: GPL-2.0-only */
53631b54 CM	2	/*
53631b54 CM	3	* Copyright (C) 2012 ARM Ltd.
53631b54 CM	4	*/
	5	#ifndef __ASM_FP_H
	6	#define __ASM_FP_H
	7
2d2123bc	8	#include <asm/errno.h>
f9209e26	9	#include <asm/ptrace.h>
9a6e5948 DM	10	#include <asm/processor.h>
9a6e5948 DM	11	#include <asm/sigcontext.h>
f9209e26	12	#include <asm/sysreg.h>
53631b54 CM	13
	14	#ifndef __ASSEMBLY__
	15
ead9e430	16	#include <linux/bitmap.h>
f9209e26	17	#include <linux/build_bug.h>
ead9e430	18	#include <linux/bug.h>
7582e220	19	#include <linux/cache.h>
b4f9b390	20	#include <linux/init.h>
bc0ee476	21	#include <linux/stddef.h>
ead9e430	22	#include <linux/types.h>
bc0ee476	23
b907b80d	24	#ifdef CONFIG_COMPAT
53631b54 CM	25	/* Masks for extracting the FPSR and FPCR from the FPSCR */
	26	#define VFP_FPSCR_STAT_MASK 0xf800009f
	27	#define VFP_FPSCR_CTRL_MASK 0x07f79f00
	28	/*
	29	* The VFP state has 32x64-bit registers and a single 32-bit
	30	* control/status register.
	31	*/
	32	#define VFP_STATE_SIZE ((32 * 8) + 4)
	33	#endif
	34
d158a060 MB	35	/*
	36	* When we defined the maximum SVE vector length we defined the ABI so
	37	* that the maximum vector length included all the reserved for future
	38	* expansion bits in ZCR rather than those just currently defined by
	39	* the architecture. While SME follows a similar pattern the fact that
	40	* it includes a square matrix means that any allocations that attempt
	41	* to cover the maximum potential vector length (such as happen with
	42	* the regset used for ptrace) end up being extremely large. Define
	43	* the much lower actual limit for use in such situations.
	44	*/
	45	#define SME_VQ_MAX 16
	46
53631b54 CM	47	struct task_struct;
53631b54 CM	48
20b85472 DM	49	extern void fpsimd_save_state(struct user_fpsimd_state *state);
20b85472 DM	50	extern void fpsimd_load_state(struct user_fpsimd_state *state);
53631b54 CM	51
	52	extern void fpsimd_thread_switch(struct task_struct *next);
	53	extern void fpsimd_flush_thread(void);
	54
8cd969d2	55	extern void fpsimd_signal_preserve_current_state(void);
c51f9269	56	extern void fpsimd_preserve_current_state(void);
005f78cd	57	extern void fpsimd_restore_current_state(void);
0abdeff5	58	extern void fpsimd_update_current_state(struct user_fpsimd_state const *state);
93ae6b01	59	extern void fpsimd_kvm_prepare(void);
c51f9269	60
1192b93b MB	61	struct cpu_fp_state {
	62	struct user_fpsimd_state *st;
	63	void *sve_state;
ce514000	64	void *sme_state;
1192b93b MB	65	u64 *svcr;
	66	unsigned int sve_vl;
	67	unsigned int sme_vl;
	68	enum fp_type *fp_type;
	69	enum fp_type to_save;
	70	};
	71
	72	extern void fpsimd_bind_state_to_cpu(struct cpu_fp_state *fp_state);
e6b673b7	73
005f78cd	74	extern void fpsimd_flush_task_state(struct task_struct *target);
54b8c7cb	75	extern void fpsimd_save_and_flush_cpu_state(void);
005f78cd	76
af7167d6 MB	77	static inline bool thread_sm_enabled(struct thread_struct *thread)
af7167d6 MB	78	{
ec0067a6	79	return system_supports_sme() && (thread->svcr & SVCR_SM_MASK);
af7167d6 MB	80	}
	81
	82	static inline bool thread_za_enabled(struct thread_struct *thread)
	83	{
ec0067a6	84	return system_supports_sme() && (thread->svcr & SVCR_ZA_MASK);
af7167d6 MB	85	}
af7167d6 MB	86
30c43e73 MB	87	/* Maximum VL that SVE/SME VL-agnostic software can transparently support */
30c43e73 MB	88	#define VL_ARCH_MAX 0x100
7582e220	89
9a6e5948 DM	90	/* Offset of FFR in the SVE register dump */
	91	static inline size_t sve_ffr_offset(int vl)
	92	{
	93	return SVE_SIG_FFR_OFFSET(sve_vq_from_vl(vl)) - SVE_SIG_REGS_OFFSET;
	94	}
	95
	96	static inline void sve_pffr(struct thread_struct thread)
	97	{
af7167d6 MB	98	unsigned int vl;
	99
	100	if (system_supports_sme() && thread_sm_enabled(thread))
	101	vl = thread_get_sme_vl(thread);
	102	else
	103	vl = thread_get_sve_vl(thread);
	104
	105	return (char *)thread->sve_state + sve_ffr_offset(vl);
9a6e5948 DM	106	}
9a6e5948 DM	107
d6138b4a MB	108	static inline void thread_zt_state(struct thread_struct thread)
	109	{
	110	/* The ZT register state is stored immediately after the ZA state */
	111	unsigned int sme_vq = sve_vq_from_vl(thread_get_sme_vl(thread));
	112	return thread->sme_state + ZA_SIG_REGS_SIZE(sme_vq);
	113	}
	114
9f584866	115	extern void sve_save_state(void state, u32 pfpsr, int save_ffr);
1fc5dce7	116	extern void sve_load_state(void const state, u32 const pfpsr,
ddc806b5	117	int restore_ffr);
9f584866	118	extern void sve_flush_live(bool flush_ffr, unsigned long vq_minus_1);
1fc5dce7	119	extern unsigned int sve_get_vl(void);
cccb78ce	120	extern void sve_set_vq(unsigned long vq_minus_1);
af7167d6	121	extern void sme_set_vq(unsigned long vq_minus_1);
95fcec71 MB	122	extern void sme_save_state(void *state, int zt);
95fcec71 MB	123	extern void sme_load_state(void const *state, int zt);
c0cda3b8 DM	124
	125	struct arm64_cpu_capabilities;
	126	extern void sve_kernel_enable(const struct arm64_cpu_capabilities *__unused);
5e64b862	127	extern void sme_kernel_enable(const struct arm64_cpu_capabilities *__unused);
d4913eee	128	extern void sme2_kernel_enable(const struct arm64_cpu_capabilities *__unused);
5e64b862	129	extern void fa64_kernel_enable(const struct arm64_cpu_capabilities *__unused);
1fc5dce7	130
31dc52b3	131	extern u64 read_zcr_features(void);
b42990d3	132	extern u64 read_smcr_features(void);
31dc52b3	133
ead9e430 DM	134	/*
	135	* Helpers to translate bit indices in sve_vq_map to VQ values (and
	136	* vice versa). This allows find_next_bit() to be used to find the
	137	* _maximum_ VQ not exceeding a certain value.
	138	*/
	139	static inline unsigned int __vq_to_bit(unsigned int vq)
	140	{
	141	return SVE_VQ_MAX - vq;
	142	}
	143
	144	static inline unsigned int __bit_to_vq(unsigned int bit)
	145	{
ead9e430 DM	146	return SVE_VQ_MAX - bit;
	147	}
	148
b5bc00ff MB	149
	150	struct vl_info {
	151	enum vec_type type;
	152	const char name; / For display purposes */
	153
	154	/* Minimum supported vector length across all CPUs */
	155	int min_vl;
	156
	157	/* Maximum supported vector length across all CPUs */
	158	int max_vl;
	159	int max_virtualisable_vl;
	160
	161	/*
	162	* Set of available vector lengths,
	163	* where length vq encoded as bit __vq_to_bit(vq):
	164	*/
	165	DECLARE_BITMAP(vq_map, SVE_VQ_MAX);
	166
	167	/* Set of vector lengths present on at least one cpu: */
	168	DECLARE_BITMAP(vq_partial_map, SVE_VQ_MAX);
	169	};
7582e220	170
bc0ee476 DM	171	#ifdef CONFIG_ARM64_SVE
bc0ee476 DM	172
826a4fdd	173	extern void sve_alloc(struct task_struct *task, bool flush);
bc0ee476	174	extern void fpsimd_release_task(struct task_struct *task);
43d4da2c	175	extern void fpsimd_sync_to_sve(struct task_struct *task);
e12310a0	176	extern void fpsimd_force_sync_to_sve(struct task_struct *task);
43d4da2c DM	177	extern void sve_sync_to_fpsimd(struct task_struct *task);
	178	extern void sve_sync_from_fpsimd_zeropad(struct task_struct *task);
	179
30c43e73	180	extern int vec_set_vector_length(struct task_struct *task, enum vec_type type,
7582e220	181	unsigned long vl, unsigned long flags);
bc0ee476	182
2d2123bc DM	183	extern int sve_set_current_vl(unsigned long arg);
	184	extern int sve_get_current_vl(void);
	185
f9209e26 MR	186	static inline void sve_user_disable(void)
	187	{
	188	sysreg_clear_set(cpacr_el1, CPACR_EL1_ZEN_EL0EN, 0);
	189	}
	190
	191	static inline void sve_user_enable(void)
	192	{
	193	sysreg_clear_set(cpacr_el1, 0, CPACR_EL1_ZEN_EL0EN);
	194	}
	195
71ce1ae5 MZ	196	#define sve_cond_update_zcr_vq(val, reg) \
	197	do { \
	198	u64 __zcr = read_sysreg_s((reg)); \
	199	u64 __new = __zcr & ~ZCR_ELx_LEN_MASK; \
	200	__new \|= (val) & ZCR_ELx_LEN_MASK; \
	201	if (__zcr != __new) \
	202	write_sysreg_s(__new, (reg)); \
	203	} while (0)
	204
2e0f2478 DM	205	/*
	206	* Probing and setup functions.
	207	* Calls to these functions must be serialised with one another.
	208	*/
b5bc00ff MB	209	enum vec_type;
	210
	211	extern void __init vec_init_vq_map(enum vec_type type);
	212	extern void vec_update_vq_map(enum vec_type type);
	213	extern int vec_verify_vq_map(enum vec_type type);
2e0f2478 DM	214	extern void __init sve_setup(void);
2e0f2478 DM	215
b5bc00ff MB	216	extern __ro_after_init struct vl_info vl_info[ARM64_VEC_MAX];
	217
	218	static inline void write_vl(enum vec_type type, u64 val)
	219	{
	220	u64 tmp;
	221
	222	switch (type) {
	223	#ifdef CONFIG_ARM64_SVE
	224	case ARM64_VEC_SVE:
	225	tmp = read_sysreg_s(SYS_ZCR_EL1) & ~ZCR_ELx_LEN_MASK;
	226	write_sysreg_s(tmp \| val, SYS_ZCR_EL1);
	227	break;
b42990d3 MB	228	#endif
	229	#ifdef CONFIG_ARM64_SME
	230	case ARM64_VEC_SME:
	231	tmp = read_sysreg_s(SYS_SMCR_EL1) & ~SMCR_ELx_LEN_MASK;
	232	write_sysreg_s(tmp \| val, SYS_SMCR_EL1);
	233	break;
b5bc00ff MB	234	#endif
	235	default:
	236	WARN_ON_ONCE(1);
	237	break;
	238	}
	239	}
	240
	241	static inline int vec_max_vl(enum vec_type type)
	242	{
	243	return vl_info[type].max_vl;
	244	}
	245
	246	static inline int vec_max_virtualisable_vl(enum vec_type type)
	247	{
	248	return vl_info[type].max_virtualisable_vl;
	249	}
	250
	251	static inline int sve_max_vl(void)
	252	{
	253	return vec_max_vl(ARM64_VEC_SVE);
	254	}
	255
	256	static inline int sve_max_virtualisable_vl(void)
	257	{
	258	return vec_max_virtualisable_vl(ARM64_VEC_SVE);
	259	}
	260
	261	/* Ensure vq >= SVE_VQ_MIN && vq <= SVE_VQ_MAX before calling this function */
	262	static inline bool vq_available(enum vec_type type, unsigned int vq)
	263	{
	264	return test_bit(__vq_to_bit(vq), vl_info[type].vq_map);
	265	}
	266
	267	static inline bool sve_vq_available(unsigned int vq)
	268	{
	269	return vq_available(ARM64_VEC_SVE, vq);
	270	}
	271
8bd7f91c MB	272	size_t sve_state_size(struct task_struct const *task);
8bd7f91c MB	273
bc0ee476 DM	274	#else /* ! CONFIG_ARM64_SVE */
bc0ee476 DM	275
826a4fdd	276	static inline void sve_alloc(struct task_struct *task, bool flush) { }
bc0ee476	277	static inline void fpsimd_release_task(struct task_struct *task) { }
43d4da2c DM	278	static inline void sve_sync_to_fpsimd(struct task_struct *task) { }
	279	static inline void sve_sync_from_fpsimd_zeropad(struct task_struct *task) { }
	280
49ed9204 MB	281	static inline int sve_max_virtualisable_vl(void)
	282	{
	283	return 0;
	284	}
	285
2d2123bc DM	286	static inline int sve_set_current_vl(unsigned long arg)
	287	{
	288	return -EINVAL;
	289	}
	290
	291	static inline int sve_get_current_vl(void)
	292	{
	293	return -EINVAL;
	294	}
	295
b5bc00ff MB	296	static inline int sve_max_vl(void)
	297	{
	298	return -EINVAL;
	299	}
	300
	301	static inline bool sve_vq_available(unsigned int vq) { return false; }
	302
f9209e26 MR	303	static inline void sve_user_disable(void) { BUILD_BUG(); }
	304	static inline void sve_user_enable(void) { BUILD_BUG(); }
	305
a9f8696d XT	306	#define sve_cond_update_zcr_vq(val, reg) do { } while (0)
a9f8696d XT	307
b5bc00ff MB	308	static inline void vec_init_vq_map(enum vec_type t) { }
	309	static inline void vec_update_vq_map(enum vec_type t) { }
	310	static inline int vec_verify_vq_map(enum vec_type t) { return 0; }
2e0f2478	311	static inline void sve_setup(void) { }
bc0ee476	312
8bd7f91c MB	313	static inline size_t sve_state_size(struct task_struct const *task)
	314	{
	315	return 0;
	316	}
	317
bc0ee476 DM	318	#endif /* ! CONFIG_ARM64_SVE */
bc0ee476 DM	319
ca8a4ebc MB	320	#ifdef CONFIG_ARM64_SME
ca8a4ebc MB	321
8bd7f91c MB	322	static inline void sme_user_disable(void)
	323	{
	324	sysreg_clear_set(cpacr_el1, CPACR_EL1_SMEN_EL0EN, 0);
	325	}
	326
	327	static inline void sme_user_enable(void)
	328	{
	329	sysreg_clear_set(cpacr_el1, 0, CPACR_EL1_SMEN_EL0EN);
	330	}
	331
ca8a4ebc MB	332	static inline void sme_smstart_sm(void)
	333	{
	334	asm volatile(__msr_s(SYS_SVCR_SMSTART_SM_EL0, "xzr"));
	335	}
	336
	337	static inline void sme_smstop_sm(void)
	338	{
	339	asm volatile(__msr_s(SYS_SVCR_SMSTOP_SM_EL0, "xzr"));
	340	}
	341
	342	static inline void sme_smstop(void)
	343	{
	344	asm volatile(__msr_s(SYS_SVCR_SMSTOP_SMZA_EL0, "xzr"));
	345	}
	346
b42990d3 MB	347	extern void __init sme_setup(void);
	348
	349	static inline int sme_max_vl(void)
	350	{
	351	return vec_max_vl(ARM64_VEC_SME);
	352	}
	353
	354	static inline int sme_max_virtualisable_vl(void)
	355	{
	356	return vec_max_virtualisable_vl(ARM64_VEC_SME);
	357	}
	358
5d0a8d2f	359	extern void sme_alloc(struct task_struct *task, bool flush);
b42990d3	360	extern unsigned int sme_get_vl(void);
9e4ab6c8 MB	361	extern int sme_set_current_vl(unsigned long arg);
9e4ab6c8 MB	362	extern int sme_get_current_vl(void);
b42990d3	363
8bd7f91c MB	364	/*
8bd7f91c MB	365	* Return how many bytes of memory are required to store the full SME
ce514000 MB	366	* specific state for task, given task's currently configured vector
ce514000 MB	367	* length.
8bd7f91c	368	*/
ce514000	369	static inline size_t sme_state_size(struct task_struct const *task)
8bd7f91c MB	370	{
8bd7f91c MB	371	unsigned int vl = task_get_sme_vl(task);
ce514000	372	size_t size;
8bd7f91c	373
ce514000 MB	374	size = ZA_SIG_REGS_SIZE(sve_vq_from_vl(vl));
ce514000 MB	375
d6138b4a MB	376	if (system_supports_sme2())
	377	size += ZT_SIG_REG_SIZE;
	378
ce514000	379	return size;
8bd7f91c MB	380	}
8bd7f91c MB	381
ca8a4ebc MB	382	#else
ca8a4ebc MB	383
8bd7f91c MB	384	static inline void sme_user_disable(void) { BUILD_BUG(); }
	385	static inline void sme_user_enable(void) { BUILD_BUG(); }
	386
ca8a4ebc MB	387	static inline void sme_smstart_sm(void) { }
	388	static inline void sme_smstop_sm(void) { }
	389	static inline void sme_smstop(void) { }
	390
5d0a8d2f	391	static inline void sme_alloc(struct task_struct *task, bool flush) { }
b42990d3 MB	392	static inline void sme_setup(void) { }
	393	static inline unsigned int sme_get_vl(void) { return 0; }
	394	static inline int sme_max_vl(void) { return 0; }
	395	static inline int sme_max_virtualisable_vl(void) { return 0; }
9e4ab6c8 MB	396	static inline int sme_set_current_vl(unsigned long arg) { return -EINVAL; }
9e4ab6c8 MB	397	static inline int sme_get_current_vl(void) { return -EINVAL; }
b42990d3	398
ce514000	399	static inline size_t sme_state_size(struct task_struct const *task)
8bd7f91c MB	400	{
	401	return 0;
	402	}
	403
ca8a4ebc MB	404	#endif /* ! CONFIG_ARM64_SME */
ca8a4ebc MB	405
4328825d DM	406	/* For use by EFI runtime services calls only */
	407	extern void __efi_fpsimd_begin(void);
	408	extern void __efi_fpsimd_end(void);
	409
53631b54 CM	410	#endif
	411
	412	#endif