[linux-2.6-block.git] / arch / arm64 / lib / mte.S

/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * Copyright (C) 2020 ARM Ltd.
 */
#include <linux/linkage.h>

#include <asm/asm-uaccess.h>
#include <asm/assembler.h>
#include <asm/mte.h>
#include <asm/page.h>
#include <asm/sysreg.h>

	.arch	armv8.5-a+memtag

/*
 * multitag_transfer_size - set \reg to the block size that is accessed by the
 * LDGM/STGM instructions.
 */
	.macro	multitag_transfer_size, reg, tmp
	mrs_s	\reg, SYS_GMID_EL1
	ubfx	\reg, \reg, #SYS_GMID_EL1_BS_SHIFT, #SYS_GMID_EL1_BS_SIZE
	mov	\tmp, #4
	lsl	\reg, \tmp, \reg
	.endm

/*
 * Clear the tags in a page
 *   x0 - address of the page to be cleared
 */
SYM_FUNC_START(mte_clear_page_tags)
	multitag_transfer_size x1, x2
1:	stgm	xzr, [x0]
	add	x0, x0, x1
	tst	x0, #(PAGE_SIZE - 1)
	b.ne	1b
	ret
SYM_FUNC_END(mte_clear_page_tags)

/*
 * Copy the tags from the source page to the destination one
 *   x0 - address of the destination page
 *   x1 - address of the source page
 */
SYM_FUNC_START(mte_copy_page_tags)
	mov	x2, x0
	mov	x3, x1
	multitag_transfer_size x5, x6
1:	ldgm	x4, [x3]
	stgm	x4, [x2]
	add	x2, x2, x5
	add	x3, x3, x5
	tst	x2, #(PAGE_SIZE - 1)
	b.ne	1b
	ret
SYM_FUNC_END(mte_copy_page_tags)

/*
 * Read tags from a user buffer (one tag per byte) and set the corresponding
 * tags at the given kernel address. Used by PTRACE_POKEMTETAGS.
 *   x0 - kernel address (to)
 *   x1 - user buffer (from)
 *   x2 - number of tags/bytes (n)
 * Returns:
 *   x0 - number of tags read/set
 */
SYM_FUNC_START(mte_copy_tags_from_user)
	mov	x3, x1
	cbz	x2, 2f
1:
	user_ldst 2f, ldtrb, w4, x1, 0
	lsl	x4, x4, #MTE_TAG_SHIFT
	stg	x4, [x0], #MTE_GRANULE_SIZE
	add	x1, x1, #1
	subs	x2, x2, #1
	b.ne	1b

	// exception handling and function return
2:	sub	x0, x1, x3		// update the number of tags set
	ret
SYM_FUNC_END(mte_copy_tags_from_user)

/*
 * Get the tags from a kernel address range and write the tag values to the
 * given user buffer (one tag per byte). Used by PTRACE_PEEKMTETAGS.
 *   x0 - user buffer (to)
 *   x1 - kernel address (from)
 *   x2 - number of tags/bytes (n)
 * Returns:
 *   x0 - number of tags read/set
 */
SYM_FUNC_START(mte_copy_tags_to_user)
	mov	x3, x0
	cbz	x2, 2f
1:
	ldg	x4, [x1]
	ubfx	x4, x4, #MTE_TAG_SHIFT, #MTE_TAG_SIZE
	user_ldst 2f, sttrb, w4, x0, 0
	add	x0, x0, #1
	add	x1, x1, #MTE_GRANULE_SIZE
	subs	x2, x2, #1
	b.ne	1b

	// exception handling and function return
2:	sub	x0, x0, x3		// update the number of tags copied
	ret
SYM_FUNC_END(mte_copy_tags_to_user)

/*
 * Save the tags in a page
 *   x0 - page address
 *   x1 - tag storage
 */
SYM_FUNC_START(mte_save_page_tags)
	multitag_transfer_size x7, x5
1:
	mov	x2, #0
2:
	ldgm	x5, [x0]
	orr	x2, x2, x5
	add	x0, x0, x7
	tst	x0, #0xFF		// 16 tag values fit in a register,
	b.ne	2b			// which is 16*16=256 bytes

	str	x2, [x1], #8

	tst	x0, #(PAGE_SIZE - 1)
	b.ne	1b

	ret
SYM_FUNC_END(mte_save_page_tags)

/*
 * Restore the tags in a page
 *   x0 - page address
 *   x1 - tag storage
 */
SYM_FUNC_START(mte_restore_page_tags)
	multitag_transfer_size x7, x5
1:
	ldr	x2, [x1], #8
2:
	stgm	x2, [x0]
	add	x0, x0, x7
	tst	x0, #0xFF
	b.ne	2b

	tst	x0, #(PAGE_SIZE - 1)
	b.ne	1b

	ret
SYM_FUNC_END(mte_restore_page_tags)

/*
 * Assign allocation tags for a region of memory based on the pointer tag
 *   x0 - source pointer
 *   x1 - size
 *
 * Note: The address must be non-NULL and MTE_GRANULE_SIZE aligned and
 * size must be non-zero and MTE_GRANULE_SIZE aligned.
 */
SYM_FUNC_START(mte_assign_mem_tag_range)
1:	stg	x0, [x0]
	add	x0, x0, #MTE_GRANULE_SIZE
	subs	x1, x1, #MTE_GRANULE_SIZE
	b.gt	1b
	ret
SYM_FUNC_END(mte_assign_mem_tag_range)
Commit	Line	Data
34bfeea4 CM	1	/* SPDX-License-Identifier: GPL-2.0-only */
	2	/*
	3	* Copyright (C) 2020 ARM Ltd.
	4	*/
	5	#include <linux/linkage.h>
	6
e2a2190a	7	#include <asm/asm-uaccess.h>
34bfeea4	8	#include <asm/assembler.h>
18ddbaa0	9	#include <asm/mte.h>
2563776b	10	#include <asm/page.h>
34bfeea4 CM	11	#include <asm/sysreg.h>
	12
	13	.arch armv8.5-a+memtag
	14
	15	/*
	16	* multitag_transfer_size - set \reg to the block size that is accessed by the
	17	* LDGM/STGM instructions.
	18	*/
	19	.macro multitag_transfer_size, reg, tmp
	20	mrs_s \reg, SYS_GMID_EL1
	21	ubfx \reg, \reg, #SYS_GMID_EL1_BS_SHIFT, #SYS_GMID_EL1_BS_SIZE
	22	mov \tmp, #4
	23	lsl \reg, \tmp, \reg
	24	.endm
	25
	26	/*
	27	* Clear the tags in a page
	28	* x0 - address of the page to be cleared
	29	*/
	30	SYM_FUNC_START(mte_clear_page_tags)
	31	multitag_transfer_size x1, x2
	32	1: stgm xzr, [x0]
	33	add x0, x0, x1
	34	tst x0, #(PAGE_SIZE - 1)
	35	b.ne 1b
	36	ret
	37	SYM_FUNC_END(mte_clear_page_tags)
2563776b VF	38
	39	/*
	40	* Copy the tags from the source page to the destination one
	41	* x0 - address of the destination page
	42	* x1 - address of the source page
	43	*/
	44	SYM_FUNC_START(mte_copy_page_tags)
	45	mov x2, x0
	46	mov x3, x1
	47	multitag_transfer_size x5, x6
	48	1: ldgm x4, [x3]
	49	stgm x4, [x2]
	50	add x2, x2, x5
	51	add x3, x3, x5
	52	tst x2, #(PAGE_SIZE - 1)
	53	b.ne 1b
	54	ret
	55	SYM_FUNC_END(mte_copy_page_tags)
18ddbaa0 CM	56
	57	/*
	58	* Read tags from a user buffer (one tag per byte) and set the corresponding
	59	* tags at the given kernel address. Used by PTRACE_POKEMTETAGS.
	60	* x0 - kernel address (to)
	61	* x1 - user buffer (from)
	62	* x2 - number of tags/bytes (n)
	63	* Returns:
	64	* x0 - number of tags read/set
	65	*/
	66	SYM_FUNC_START(mte_copy_tags_from_user)
	67	mov x3, x1
	68	cbz x2, 2f
	69	1:
7b90dc40	70	user_ldst 2f, ldtrb, w4, x1, 0
18ddbaa0 CM	71	lsl x4, x4, #MTE_TAG_SHIFT
	72	stg x4, [x0], #MTE_GRANULE_SIZE
	73	add x1, x1, #1
	74	subs x2, x2, #1
	75	b.ne 1b
	76
	77	// exception handling and function return
	78	2: sub x0, x1, x3 // update the number of tags set
	79	ret
	80	SYM_FUNC_END(mte_copy_tags_from_user)
	81
	82	/*
	83	* Get the tags from a kernel address range and write the tag values to the
	84	* given user buffer (one tag per byte). Used by PTRACE_PEEKMTETAGS.
	85	* x0 - user buffer (to)
	86	* x1 - kernel address (from)
	87	* x2 - number of tags/bytes (n)
	88	* Returns:
	89	* x0 - number of tags read/set
	90	*/
	91	SYM_FUNC_START(mte_copy_tags_to_user)
	92	mov x3, x0
	93	cbz x2, 2f
	94	1:
	95	ldg x4, [x1]
	96	ubfx x4, x4, #MTE_TAG_SHIFT, #MTE_TAG_SIZE
7b90dc40	97	user_ldst 2f, sttrb, w4, x0, 0
18ddbaa0 CM	98	add x0, x0, #1
	99	add x1, x1, #MTE_GRANULE_SIZE
	100	subs x2, x2, #1
	101	b.ne 1b
	102
	103	// exception handling and function return
	104	2: sub x0, x0, x3 // update the number of tags copied
	105	ret
	106	SYM_FUNC_END(mte_copy_tags_to_user)
36943aba SP	107
	108	/*
	109	* Save the tags in a page
	110	* x0 - page address
	111	* x1 - tag storage
	112	*/
	113	SYM_FUNC_START(mte_save_page_tags)
	114	multitag_transfer_size x7, x5
	115	1:
	116	mov x2, #0
	117	2:
	118	ldgm x5, [x0]
	119	orr x2, x2, x5
	120	add x0, x0, x7
	121	tst x0, #0xFF // 16 tag values fit in a register,
	122	b.ne 2b // which is 16*16=256 bytes
	123
	124	str x2, [x1], #8
	125
	126	tst x0, #(PAGE_SIZE - 1)
	127	b.ne 1b
	128
	129	ret
	130	SYM_FUNC_END(mte_save_page_tags)
	131
	132	/*
	133	* Restore the tags in a page
	134	* x0 - page address
	135	* x1 - tag storage
	136	*/
	137	SYM_FUNC_START(mte_restore_page_tags)
	138	multitag_transfer_size x7, x5
	139	1:
	140	ldr x2, [x1], #8
	141	2:
	142	stgm x2, [x0]
	143	add x0, x0, x7
	144	tst x0, #0xFF
	145	b.ne 2b
	146
	147	tst x0, #(PAGE_SIZE - 1)
	148	b.ne 1b
	149
	150	ret
	151	SYM_FUNC_END(mte_restore_page_tags)
85f49cae VF	152
	153	/*
	154	* Assign allocation tags for a region of memory based on the pointer tag
	155	* x0 - source pointer
	156	* x1 - size
	157	*
	158	* Note: The address must be non-NULL and MTE_GRANULE_SIZE aligned and
	159	* size must be non-zero and MTE_GRANULE_SIZE aligned.
	160	*/
	161	SYM_FUNC_START(mte_assign_mem_tag_range)
	162	1: stg x0, [x0]
	163	add x0, x0, #MTE_GRANULE_SIZE
	164	subs x1, x1, #MTE_GRANULE_SIZE
	165	b.gt 1b
	166	ret
	167	SYM_FUNC_END(mte_assign_mem_tag_range)