Commit | Line | Data |
---|---|---|
92e3da3c RP |
1 | // SPDX-License-Identifier: GPL-2.0+ |
2 | /* | |
3 | * PowerPC Memory Protection Keys management | |
4 | * | |
5 | * Copyright 2017, Ram Pai, IBM Corporation. | |
6 | */ | |
7 | ||
2ddc53f3 | 8 | #include <asm/mman.h> |
71432ce2 | 9 | #include <asm/mmu_context.h> |
cf43d3b2 | 10 | #include <asm/setup.h> |
92e3da3c | 11 | #include <linux/pkeys.h> |
cf43d3b2 | 12 | #include <linux/of_device.h> |
92e3da3c RP |
13 | |
14 | DEFINE_STATIC_KEY_TRUE(pkey_disabled); | |
4fb158f6 | 15 | int pkeys_total; /* Total pkeys as per device tree */ |
07f522d2 RP |
16 | u32 initial_allocation_mask; /* Bits set for the initially allocated keys */ |
17 | u32 reserved_allocation_mask; /* Bits set for reserved keys */ | |
71432ce2 BL |
18 | static bool pkey_execute_disable_supported; |
19 | static bool pkeys_devtree_defined; /* property exported by device tree */ | |
20 | static u64 pkey_amr_mask; /* Bits in AMR not to be touched */ | |
21 | static u64 pkey_iamr_mask; /* Bits in AMR not to be touched */ | |
22 | static u64 pkey_uamor_mask; /* Bits in UMOR not to be touched */ | |
23 | static int execute_only_key = 2; | |
92e3da3c | 24 | |
4d70b698 | 25 | #define AMR_BITS_PER_PKEY 2 |
2ddc53f3 RP |
26 | #define AMR_RD_BIT 0x1UL |
27 | #define AMR_WR_BIT 0x2UL | |
28 | #define IAMR_EX_BIT 0x1UL | |
4d70b698 RP |
29 | #define PKEY_REG_BITS (sizeof(u64)*8) |
30 | #define pkeyshift(pkey) (PKEY_REG_BITS - ((pkey+1) * AMR_BITS_PER_PKEY)) | |
31 | ||
cf43d3b2 RP |
32 | static void scan_pkey_feature(void) |
33 | { | |
34 | u32 vals[2]; | |
35 | struct device_node *cpu; | |
36 | ||
37 | cpu = of_find_node_by_type(NULL, "cpu"); | |
38 | if (!cpu) | |
39 | return; | |
40 | ||
41 | if (of_property_read_u32_array(cpu, | |
42 | "ibm,processor-storage-keys", vals, 2)) | |
43 | return; | |
44 | ||
45 | /* | |
46 | * Since any pkey can be used for data or execute, we will just treat | |
47 | * all keys as equal and track them as one entity. | |
48 | */ | |
c716a25b | 49 | pkeys_total = vals[0]; |
cf43d3b2 RP |
50 | pkeys_devtree_defined = true; |
51 | } | |
52 | ||
53 | static inline bool pkey_mmu_enabled(void) | |
54 | { | |
55 | if (firmware_has_feature(FW_FEATURE_LPAR)) | |
56 | return pkeys_total; | |
57 | else | |
58 | return cpu_has_feature(CPU_FTR_PKEY); | |
59 | } | |
60 | ||
71432ce2 | 61 | static int pkey_initialize(void) |
92e3da3c | 62 | { |
4fb158f6 RP |
63 | int os_reserved, i; |
64 | ||
dcf87295 RP |
65 | /* |
66 | * We define PKEY_DISABLE_EXECUTE in addition to the arch-neutral | |
67 | * generic defines for PKEY_DISABLE_ACCESS and PKEY_DISABLE_WRITE. | |
68 | * Ensure that the bits a distinct. | |
69 | */ | |
70 | BUILD_BUG_ON(PKEY_DISABLE_EXECUTE & | |
71 | (PKEY_DISABLE_ACCESS | PKEY_DISABLE_WRITE)); | |
72 | ||
013a91b3 RP |
73 | /* |
74 | * pkey_to_vmflag_bits() assumes that the pkey bits are contiguous | |
75 | * in the vmaflag. Make sure that is really the case. | |
76 | */ | |
77 | BUILD_BUG_ON(__builtin_clzl(ARCH_VM_PKEY_FLAGS >> VM_PKEY_SHIFT) + | |
78 | __builtin_popcountl(ARCH_VM_PKEY_FLAGS >> VM_PKEY_SHIFT) | |
79 | != (sizeof(u64) * BITS_PER_BYTE)); | |
80 | ||
cf43d3b2 RP |
81 | /* scan the device tree for pkey feature */ |
82 | scan_pkey_feature(); | |
83 | ||
92e3da3c | 84 | /* |
cf43d3b2 RP |
85 | * Let's assume 32 pkeys on P8 bare metal, if its not defined by device |
86 | * tree. We make this exception since skiboot forgot to expose this | |
87 | * property on power8. | |
92e3da3c | 88 | */ |
cf43d3b2 RP |
89 | if (!pkeys_devtree_defined && !firmware_has_feature(FW_FEATURE_LPAR) && |
90 | cpu_has_feature(CPU_FTRS_POWER8)) | |
91 | pkeys_total = 32; | |
4fb158f6 RP |
92 | |
93 | /* | |
94 | * Adjust the upper limit, based on the number of bits supported by | |
95 | * arch-neutral code. | |
96 | */ | |
97 | pkeys_total = min_t(int, pkeys_total, | |
fe6a2804 | 98 | ((ARCH_VM_PKEY_FLAGS >> VM_PKEY_SHIFT)+1)); |
4fb158f6 | 99 | |
cf43d3b2 RP |
100 | if (!pkey_mmu_enabled() || radix_enabled() || !pkeys_total) |
101 | static_branch_enable(&pkey_disabled); | |
102 | else | |
103 | static_branch_disable(&pkey_disabled); | |
104 | ||
105 | if (static_branch_likely(&pkey_disabled)) | |
106 | return 0; | |
107 | ||
92e3da3c | 108 | /* |
cf43d3b2 RP |
109 | * The device tree cannot be relied to indicate support for |
110 | * execute_disable support. Instead we use a PVR check. | |
92e3da3c | 111 | */ |
cf43d3b2 RP |
112 | if (pvr_version_is(PVR_POWER7) || pvr_version_is(PVR_POWER7p)) |
113 | pkey_execute_disable_supported = false; | |
114 | else | |
115 | pkey_execute_disable_supported = true; | |
4fb158f6 RP |
116 | |
117 | #ifdef CONFIG_PPC_4K_PAGES | |
118 | /* | |
119 | * The OS can manage only 8 pkeys due to its inability to represent them | |
120 | * in the Linux 4K PTE. | |
121 | */ | |
122 | os_reserved = pkeys_total - 8; | |
123 | #else | |
124 | os_reserved = 0; | |
125 | #endif | |
07f522d2 RP |
126 | /* Bits are in LE format. */ |
127 | reserved_allocation_mask = (0x1 << 1) | (0x1 << execute_only_key); | |
a57a04c7 RP |
128 | |
129 | /* register mask is in BE format */ | |
130 | pkey_amr_mask = ~0x0ul; | |
de113256 | 131 | pkey_amr_mask &= ~(0x3ul << pkeyshift(0)); |
a57a04c7 | 132 | |
de113256 RP |
133 | pkey_iamr_mask = ~0x0ul; |
134 | pkey_iamr_mask &= ~(0x3ul << pkeyshift(0)); | |
a4fcc877 | 135 | pkey_iamr_mask &= ~(0x3ul << pkeyshift(execute_only_key)); |
a57a04c7 RP |
136 | |
137 | pkey_uamor_mask = ~0x0ul; | |
138 | pkey_uamor_mask &= ~(0x3ul << pkeyshift(0)); | |
a4fcc877 | 139 | pkey_uamor_mask &= ~(0x3ul << pkeyshift(execute_only_key)); |
a57a04c7 RP |
140 | |
141 | /* mark the rest of the keys as reserved and hence unavailable */ | |
142 | for (i = (pkeys_total - os_reserved); i < pkeys_total; i++) { | |
07f522d2 | 143 | reserved_allocation_mask |= (0x1 << i); |
a57a04c7 RP |
144 | pkey_uamor_mask &= ~(0x3ul << pkeyshift(i)); |
145 | } | |
07f522d2 | 146 | initial_allocation_mask = reserved_allocation_mask | (0x1 << 0); |
a57a04c7 | 147 | |
a4fcc877 RP |
148 | if (unlikely((pkeys_total - os_reserved) <= execute_only_key)) { |
149 | /* | |
150 | * Insufficient number of keys to support | |
151 | * execute only key. Mark it unavailable. | |
152 | * Any AMR, UAMOR, IAMR bit set for | |
153 | * this key is irrelevant since this key | |
154 | * can never be allocated. | |
155 | */ | |
156 | execute_only_key = -1; | |
157 | } | |
158 | ||
92e3da3c RP |
159 | return 0; |
160 | } | |
161 | ||
162 | arch_initcall(pkey_initialize); | |
4fb158f6 RP |
163 | |
164 | void pkey_mm_init(struct mm_struct *mm) | |
165 | { | |
166 | if (static_branch_likely(&pkey_disabled)) | |
167 | return; | |
168 | mm_pkey_allocation_map(mm) = initial_allocation_mask; | |
a4fcc877 | 169 | mm->context.execute_only_pkey = execute_only_key; |
4fb158f6 | 170 | } |
1b4037de RP |
171 | |
172 | static inline u64 read_amr(void) | |
173 | { | |
174 | return mfspr(SPRN_AMR); | |
175 | } | |
176 | ||
177 | static inline void write_amr(u64 value) | |
178 | { | |
179 | mtspr(SPRN_AMR, value); | |
180 | } | |
181 | ||
182 | static inline u64 read_iamr(void) | |
183 | { | |
184 | if (!likely(pkey_execute_disable_supported)) | |
185 | return 0x0UL; | |
186 | ||
187 | return mfspr(SPRN_IAMR); | |
188 | } | |
189 | ||
190 | static inline void write_iamr(u64 value) | |
191 | { | |
192 | if (!likely(pkey_execute_disable_supported)) | |
193 | return; | |
194 | ||
195 | mtspr(SPRN_IAMR, value); | |
196 | } | |
197 | ||
198 | static inline u64 read_uamor(void) | |
199 | { | |
200 | return mfspr(SPRN_UAMOR); | |
201 | } | |
202 | ||
203 | static inline void write_uamor(u64 value) | |
204 | { | |
205 | mtspr(SPRN_UAMOR, value); | |
206 | } | |
4d70b698 | 207 | |
2ddc53f3 RP |
208 | static bool is_pkey_enabled(int pkey) |
209 | { | |
210 | u64 uamor = read_uamor(); | |
211 | u64 pkey_bits = 0x3ul << pkeyshift(pkey); | |
212 | u64 uamor_pkey_bits = (uamor & pkey_bits); | |
213 | ||
214 | /* | |
215 | * Both the bits in UAMOR corresponding to the key should be set or | |
216 | * reset. | |
217 | */ | |
218 | WARN_ON(uamor_pkey_bits && (uamor_pkey_bits != pkey_bits)); | |
219 | return !!(uamor_pkey_bits); | |
220 | } | |
221 | ||
4d70b698 RP |
222 | static inline void init_amr(int pkey, u8 init_bits) |
223 | { | |
224 | u64 new_amr_bits = (((u64)init_bits & 0x3UL) << pkeyshift(pkey)); | |
225 | u64 old_amr = read_amr() & ~((u64)(0x3ul) << pkeyshift(pkey)); | |
226 | ||
227 | write_amr(old_amr | new_amr_bits); | |
228 | } | |
229 | ||
230 | static inline void init_iamr(int pkey, u8 init_bits) | |
231 | { | |
232 | u64 new_iamr_bits = (((u64)init_bits & 0x1UL) << pkeyshift(pkey)); | |
233 | u64 old_iamr = read_iamr() & ~((u64)(0x1ul) << pkeyshift(pkey)); | |
234 | ||
235 | write_iamr(old_iamr | new_iamr_bits); | |
236 | } | |
237 | ||
2ddc53f3 RP |
238 | /* |
239 | * Set the access rights in AMR IAMR and UAMOR registers for @pkey to that | |
240 | * specified in @init_val. | |
241 | */ | |
242 | int __arch_set_user_pkey_access(struct task_struct *tsk, int pkey, | |
243 | unsigned long init_val) | |
244 | { | |
245 | u64 new_amr_bits = 0x0ul; | |
dcf87295 | 246 | u64 new_iamr_bits = 0x0ul; |
2ddc53f3 RP |
247 | |
248 | if (!is_pkey_enabled(pkey)) | |
249 | return -EINVAL; | |
250 | ||
dcf87295 RP |
251 | if (init_val & PKEY_DISABLE_EXECUTE) { |
252 | if (!pkey_execute_disable_supported) | |
253 | return -EINVAL; | |
254 | new_iamr_bits |= IAMR_EX_BIT; | |
255 | } | |
256 | init_iamr(pkey, new_iamr_bits); | |
257 | ||
2ddc53f3 RP |
258 | /* Set the bits we need in AMR: */ |
259 | if (init_val & PKEY_DISABLE_ACCESS) | |
260 | new_amr_bits |= AMR_RD_BIT | AMR_WR_BIT; | |
261 | else if (init_val & PKEY_DISABLE_WRITE) | |
262 | new_amr_bits |= AMR_WR_BIT; | |
263 | ||
264 | init_amr(pkey, new_amr_bits); | |
265 | return 0; | |
266 | } | |
06bb53b3 RP |
267 | |
268 | void thread_pkey_regs_save(struct thread_struct *thread) | |
269 | { | |
270 | if (static_branch_likely(&pkey_disabled)) | |
271 | return; | |
272 | ||
273 | /* | |
274 | * TODO: Skip saving registers if @thread hasn't used any keys yet. | |
275 | */ | |
276 | thread->amr = read_amr(); | |
277 | thread->iamr = read_iamr(); | |
278 | thread->uamor = read_uamor(); | |
279 | } | |
280 | ||
281 | void thread_pkey_regs_restore(struct thread_struct *new_thread, | |
282 | struct thread_struct *old_thread) | |
283 | { | |
284 | if (static_branch_likely(&pkey_disabled)) | |
285 | return; | |
286 | ||
06bb53b3 RP |
287 | if (old_thread->amr != new_thread->amr) |
288 | write_amr(new_thread->amr); | |
289 | if (old_thread->iamr != new_thread->iamr) | |
290 | write_iamr(new_thread->iamr); | |
291 | if (old_thread->uamor != new_thread->uamor) | |
292 | write_uamor(new_thread->uamor); | |
293 | } | |
294 | ||
295 | void thread_pkey_regs_init(struct thread_struct *thread) | |
296 | { | |
297 | if (static_branch_likely(&pkey_disabled)) | |
298 | return; | |
299 | ||
a57a04c7 RP |
300 | thread->amr = pkey_amr_mask; |
301 | thread->iamr = pkey_iamr_mask; | |
302 | thread->uamor = pkey_uamor_mask; | |
303 | ||
304 | write_uamor(pkey_uamor_mask); | |
305 | write_amr(pkey_amr_mask); | |
306 | write_iamr(pkey_iamr_mask); | |
06bb53b3 | 307 | } |
5586cf61 RP |
308 | |
309 | static inline bool pkey_allows_readwrite(int pkey) | |
310 | { | |
311 | int pkey_shift = pkeyshift(pkey); | |
312 | ||
313 | if (!is_pkey_enabled(pkey)) | |
314 | return true; | |
315 | ||
316 | return !(read_amr() & ((AMR_RD_BIT|AMR_WR_BIT) << pkey_shift)); | |
317 | } | |
318 | ||
319 | int __execute_only_pkey(struct mm_struct *mm) | |
320 | { | |
a4fcc877 | 321 | return mm->context.execute_only_pkey; |
5586cf61 | 322 | } |
87bbabbe RP |
323 | |
324 | static inline bool vma_is_pkey_exec_only(struct vm_area_struct *vma) | |
325 | { | |
326 | /* Do this check first since the vm_flags should be hot */ | |
327 | if ((vma->vm_flags & (VM_READ | VM_WRITE | VM_EXEC)) != VM_EXEC) | |
328 | return false; | |
329 | ||
330 | return (vma_pkey(vma) == vma->vm_mm->context.execute_only_pkey); | |
331 | } | |
332 | ||
333 | /* | |
334 | * This should only be called for *plain* mprotect calls. | |
335 | */ | |
336 | int __arch_override_mprotect_pkey(struct vm_area_struct *vma, int prot, | |
337 | int pkey) | |
338 | { | |
339 | /* | |
340 | * If the currently associated pkey is execute-only, but the requested | |
eabdb8ca | 341 | * protection is not execute-only, move it back to the default pkey. |
87bbabbe | 342 | */ |
eabdb8ca | 343 | if (vma_is_pkey_exec_only(vma) && (prot != PROT_EXEC)) |
87bbabbe RP |
344 | return 0; |
345 | ||
346 | /* | |
347 | * The requested protection is execute-only. Hence let's use an | |
348 | * execute-only pkey. | |
349 | */ | |
350 | if (prot == PROT_EXEC) { | |
351 | pkey = execute_only_pkey(vma->vm_mm); | |
352 | if (pkey > 0) | |
353 | return pkey; | |
354 | } | |
355 | ||
356 | /* Nothing to override. */ | |
357 | return vma_pkey(vma); | |
358 | } | |
f2407ef3 RP |
359 | |
360 | static bool pkey_access_permitted(int pkey, bool write, bool execute) | |
361 | { | |
362 | int pkey_shift; | |
363 | u64 amr; | |
364 | ||
f2407ef3 RP |
365 | if (!is_pkey_enabled(pkey)) |
366 | return true; | |
367 | ||
368 | pkey_shift = pkeyshift(pkey); | |
369 | if (execute && !(read_iamr() & (IAMR_EX_BIT << pkey_shift))) | |
370 | return true; | |
371 | ||
372 | amr = read_amr(); /* Delay reading amr until absolutely needed */ | |
373 | return ((!write && !(amr & (AMR_RD_BIT << pkey_shift))) || | |
374 | (write && !(amr & (AMR_WR_BIT << pkey_shift)))); | |
375 | } | |
376 | ||
377 | bool arch_pte_access_permitted(u64 pte, bool write, bool execute) | |
378 | { | |
379 | if (static_branch_likely(&pkey_disabled)) | |
380 | return true; | |
381 | ||
382 | return pkey_access_permitted(pte_to_pkey_bits(pte), write, execute); | |
383 | } | |
1137573a RP |
384 | |
385 | /* | |
386 | * We only want to enforce protection keys on the current thread because we | |
387 | * effectively have no access to AMR/IAMR for other threads or any way to tell | |
388 | * which AMR/IAMR in a threaded process we could use. | |
389 | * | |
390 | * So do not enforce things if the VMA is not from the current mm, or if we are | |
391 | * in a kernel thread. | |
392 | */ | |
393 | static inline bool vma_is_foreign(struct vm_area_struct *vma) | |
394 | { | |
395 | if (!current->mm) | |
396 | return true; | |
397 | ||
398 | /* if it is not our ->mm, it has to be foreign */ | |
399 | if (current->mm != vma->vm_mm) | |
400 | return true; | |
401 | ||
402 | return false; | |
403 | } | |
404 | ||
405 | bool arch_vma_access_permitted(struct vm_area_struct *vma, bool write, | |
406 | bool execute, bool foreign) | |
407 | { | |
408 | if (static_branch_likely(&pkey_disabled)) | |
409 | return true; | |
410 | /* | |
411 | * Do not enforce our key-permissions on a foreign vma. | |
412 | */ | |
413 | if (foreign || vma_is_foreign(vma)) | |
414 | return true; | |
415 | ||
416 | return pkey_access_permitted(vma_pkey(vma), write, execute); | |
417 | } |