Commit | Line | Data |
---|---|---|
34bfeea4 CM |
1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
2 | /* | |
3 | * Copyright (C) 2020 ARM Ltd. | |
4 | */ | |
5 | #include <linux/linkage.h> | |
6 | ||
e2a2190a | 7 | #include <asm/asm-uaccess.h> |
34bfeea4 | 8 | #include <asm/assembler.h> |
18ddbaa0 | 9 | #include <asm/mte.h> |
2563776b | 10 | #include <asm/page.h> |
34bfeea4 CM |
11 | #include <asm/sysreg.h> |
12 | ||
13 | .arch armv8.5-a+memtag | |
14 | ||
15 | /* | |
16 | * multitag_transfer_size - set \reg to the block size that is accessed by the | |
17 | * LDGM/STGM instructions. | |
18 | */ | |
19 | .macro multitag_transfer_size, reg, tmp | |
20 | mrs_s \reg, SYS_GMID_EL1 | |
21 | ubfx \reg, \reg, #SYS_GMID_EL1_BS_SHIFT, #SYS_GMID_EL1_BS_SIZE | |
22 | mov \tmp, #4 | |
23 | lsl \reg, \tmp, \reg | |
24 | .endm | |
25 | ||
26 | /* | |
27 | * Clear the tags in a page | |
28 | * x0 - address of the page to be cleared | |
29 | */ | |
30 | SYM_FUNC_START(mte_clear_page_tags) | |
31 | multitag_transfer_size x1, x2 | |
32 | 1: stgm xzr, [x0] | |
33 | add x0, x0, x1 | |
34 | tst x0, #(PAGE_SIZE - 1) | |
35 | b.ne 1b | |
36 | ret | |
37 | SYM_FUNC_END(mte_clear_page_tags) | |
2563776b | 38 | |
013bb59d PC |
39 | /* |
40 | * Zero the page and tags at the same time | |
41 | * | |
42 | * Parameters: | |
43 | * x0 - address to the beginning of the page | |
44 | */ | |
45 | SYM_FUNC_START(mte_zero_clear_page_tags) | |
685e2564 | 46 | and x0, x0, #(1 << MTE_TAG_SHIFT) - 1 // clear the tag |
013bb59d | 47 | mrs x1, dczid_el0 |
685e2564 | 48 | tbnz x1, #4, 2f // Branch if DC GZVA is prohibited |
013bb59d PC |
49 | and w1, w1, #0xf |
50 | mov x2, #4 | |
51 | lsl x1, x2, x1 | |
013bb59d PC |
52 | |
53 | 1: dc gzva, x0 | |
54 | add x0, x0, x1 | |
55 | tst x0, #(PAGE_SIZE - 1) | |
56 | b.ne 1b | |
57 | ret | |
685e2564 RW |
58 | |
59 | 2: stz2g x0, [x0], #(MTE_GRANULE_SIZE * 2) | |
60 | tst x0, #(PAGE_SIZE - 1) | |
61 | b.ne 2b | |
62 | ret | |
013bb59d PC |
63 | SYM_FUNC_END(mte_zero_clear_page_tags) |
64 | ||
2563776b VF |
65 | /* |
66 | * Copy the tags from the source page to the destination one | |
67 | * x0 - address of the destination page | |
68 | * x1 - address of the source page | |
69 | */ | |
70 | SYM_FUNC_START(mte_copy_page_tags) | |
71 | mov x2, x0 | |
72 | mov x3, x1 | |
73 | multitag_transfer_size x5, x6 | |
74 | 1: ldgm x4, [x3] | |
75 | stgm x4, [x2] | |
76 | add x2, x2, x5 | |
77 | add x3, x3, x5 | |
78 | tst x2, #(PAGE_SIZE - 1) | |
79 | b.ne 1b | |
80 | ret | |
81 | SYM_FUNC_END(mte_copy_page_tags) | |
18ddbaa0 CM |
82 | |
83 | /* | |
84 | * Read tags from a user buffer (one tag per byte) and set the corresponding | |
85 | * tags at the given kernel address. Used by PTRACE_POKEMTETAGS. | |
86 | * x0 - kernel address (to) | |
87 | * x1 - user buffer (from) | |
88 | * x2 - number of tags/bytes (n) | |
89 | * Returns: | |
90 | * x0 - number of tags read/set | |
91 | */ | |
92 | SYM_FUNC_START(mte_copy_tags_from_user) | |
93 | mov x3, x1 | |
94 | cbz x2, 2f | |
95 | 1: | |
7b90dc40 | 96 | user_ldst 2f, ldtrb, w4, x1, 0 |
18ddbaa0 CM |
97 | lsl x4, x4, #MTE_TAG_SHIFT |
98 | stg x4, [x0], #MTE_GRANULE_SIZE | |
99 | add x1, x1, #1 | |
100 | subs x2, x2, #1 | |
101 | b.ne 1b | |
102 | ||
103 | // exception handling and function return | |
104 | 2: sub x0, x1, x3 // update the number of tags set | |
105 | ret | |
106 | SYM_FUNC_END(mte_copy_tags_from_user) | |
107 | ||
108 | /* | |
109 | * Get the tags from a kernel address range and write the tag values to the | |
110 | * given user buffer (one tag per byte). Used by PTRACE_PEEKMTETAGS. | |
111 | * x0 - user buffer (to) | |
112 | * x1 - kernel address (from) | |
113 | * x2 - number of tags/bytes (n) | |
114 | * Returns: | |
115 | * x0 - number of tags read/set | |
116 | */ | |
117 | SYM_FUNC_START(mte_copy_tags_to_user) | |
118 | mov x3, x0 | |
119 | cbz x2, 2f | |
120 | 1: | |
121 | ldg x4, [x1] | |
122 | ubfx x4, x4, #MTE_TAG_SHIFT, #MTE_TAG_SIZE | |
7b90dc40 | 123 | user_ldst 2f, sttrb, w4, x0, 0 |
18ddbaa0 CM |
124 | add x0, x0, #1 |
125 | add x1, x1, #MTE_GRANULE_SIZE | |
126 | subs x2, x2, #1 | |
127 | b.ne 1b | |
128 | ||
129 | // exception handling and function return | |
130 | 2: sub x0, x0, x3 // update the number of tags copied | |
131 | ret | |
132 | SYM_FUNC_END(mte_copy_tags_to_user) | |
36943aba SP |
133 | |
134 | /* | |
135 | * Save the tags in a page | |
136 | * x0 - page address | |
ab1e435c | 137 | * x1 - tag storage, MTE_PAGE_TAG_STORAGE bytes |
36943aba SP |
138 | */ |
139 | SYM_FUNC_START(mte_save_page_tags) | |
140 | multitag_transfer_size x7, x5 | |
141 | 1: | |
142 | mov x2, #0 | |
143 | 2: | |
144 | ldgm x5, [x0] | |
145 | orr x2, x2, x5 | |
146 | add x0, x0, x7 | |
147 | tst x0, #0xFF // 16 tag values fit in a register, | |
148 | b.ne 2b // which is 16*16=256 bytes | |
149 | ||
150 | str x2, [x1], #8 | |
151 | ||
152 | tst x0, #(PAGE_SIZE - 1) | |
153 | b.ne 1b | |
154 | ||
155 | ret | |
156 | SYM_FUNC_END(mte_save_page_tags) | |
157 | ||
158 | /* | |
159 | * Restore the tags in a page | |
160 | * x0 - page address | |
ab1e435c | 161 | * x1 - tag storage, MTE_PAGE_TAG_STORAGE bytes |
36943aba SP |
162 | */ |
163 | SYM_FUNC_START(mte_restore_page_tags) | |
164 | multitag_transfer_size x7, x5 | |
165 | 1: | |
166 | ldr x2, [x1], #8 | |
167 | 2: | |
168 | stgm x2, [x0] | |
169 | add x0, x0, x7 | |
170 | tst x0, #0xFF | |
171 | b.ne 2b | |
172 | ||
173 | tst x0, #(PAGE_SIZE - 1) | |
174 | b.ne 1b | |
175 | ||
176 | ret | |
177 | SYM_FUNC_END(mte_restore_page_tags) |