Commit | Line | Data |
---|---|---|
6ebcb060 TL |
1 | /* |
2 | * AMD Memory Encryption Support | |
3 | * | |
4 | * Copyright (C) 2016 Advanced Micro Devices, Inc. | |
5 | * | |
6 | * Author: Tom Lendacky <thomas.lendacky@amd.com> | |
7 | * | |
8 | * This program is free software; you can redistribute it and/or modify | |
9 | * it under the terms of the GNU General Public License version 2 as | |
10 | * published by the Free Software Foundation. | |
11 | */ | |
12 | ||
13 | #include <linux/linkage.h> | |
14 | #include <asm/pgtable.h> | |
15 | #include <asm/page.h> | |
16 | #include <asm/processor-flags.h> | |
17 | #include <asm/msr-index.h> | |
18 | #include <asm/frame.h> | |
19 | ||
20 | .text | |
21 | .code64 | |
22 | ENTRY(sme_encrypt_execute) | |
23 | ||
24 | /* | |
25 | * Entry parameters: | |
26 | * RDI - virtual address for the encrypted kernel mapping | |
27 | * RSI - virtual address for the decrypted kernel mapping | |
28 | * RDX - length of kernel | |
29 | * RCX - virtual address of the encryption workarea, including: | |
30 | * - stack page (PAGE_SIZE) | |
31 | * - encryption routine page (PAGE_SIZE) | |
32 | * - intermediate copy buffer (PMD_PAGE_SIZE) | |
33 | * R8 - physcial address of the pagetables to use for encryption | |
34 | */ | |
35 | ||
36 | FRAME_BEGIN /* RBP now has original stack pointer */ | |
37 | ||
38 | /* Set up a one page stack in the non-encrypted memory area */ | |
39 | movq %rcx, %rax /* Workarea stack page */ | |
40 | leaq PAGE_SIZE(%rax), %rsp /* Set new stack pointer */ | |
41 | addq $PAGE_SIZE, %rax /* Workarea encryption routine */ | |
42 | ||
43 | push %r12 | |
44 | movq %rdi, %r10 /* Encrypted kernel */ | |
45 | movq %rsi, %r11 /* Decrypted kernel */ | |
46 | movq %rdx, %r12 /* Kernel length */ | |
47 | ||
48 | /* Copy encryption routine into the workarea */ | |
49 | movq %rax, %rdi /* Workarea encryption routine */ | |
50 | leaq __enc_copy(%rip), %rsi /* Encryption routine */ | |
51 | movq $(.L__enc_copy_end - __enc_copy), %rcx /* Encryption routine length */ | |
52 | rep movsb | |
53 | ||
54 | /* Setup registers for call */ | |
55 | movq %r10, %rdi /* Encrypted kernel */ | |
56 | movq %r11, %rsi /* Decrypted kernel */ | |
57 | movq %r8, %rdx /* Pagetables used for encryption */ | |
58 | movq %r12, %rcx /* Kernel length */ | |
59 | movq %rax, %r8 /* Workarea encryption routine */ | |
60 | addq $PAGE_SIZE, %r8 /* Workarea intermediate copy buffer */ | |
61 | ||
62 | call *%rax /* Call the encryption routine */ | |
63 | ||
64 | pop %r12 | |
65 | ||
66 | movq %rbp, %rsp /* Restore original stack pointer */ | |
67 | FRAME_END | |
68 | ||
69 | ret | |
70 | ENDPROC(sme_encrypt_execute) | |
71 | ||
72 | ENTRY(__enc_copy) | |
73 | /* | |
74 | * Routine used to encrypt kernel. | |
75 | * This routine must be run outside of the kernel proper since | |
76 | * the kernel will be encrypted during the process. So this | |
77 | * routine is defined here and then copied to an area outside | |
78 | * of the kernel where it will remain and run decrypted | |
79 | * during execution. | |
80 | * | |
81 | * On entry the registers must be: | |
82 | * RDI - virtual address for the encrypted kernel mapping | |
83 | * RSI - virtual address for the decrypted kernel mapping | |
84 | * RDX - address of the pagetables to use for encryption | |
85 | * RCX - length of kernel | |
86 | * R8 - intermediate copy buffer | |
87 | * | |
88 | * RAX - points to this routine | |
89 | * | |
90 | * The kernel will be encrypted by copying from the non-encrypted | |
91 | * kernel space to an intermediate buffer and then copying from the | |
92 | * intermediate buffer back to the encrypted kernel space. The physical | |
93 | * addresses of the two kernel space mappings are the same which | |
94 | * results in the kernel being encrypted "in place". | |
95 | */ | |
96 | /* Enable the new page tables */ | |
97 | mov %rdx, %cr3 | |
98 | ||
99 | /* Flush any global TLBs */ | |
100 | mov %cr4, %rdx | |
101 | andq $~X86_CR4_PGE, %rdx | |
102 | mov %rdx, %cr4 | |
103 | orq $X86_CR4_PGE, %rdx | |
104 | mov %rdx, %cr4 | |
105 | ||
106 | /* Set the PAT register PA5 entry to write-protect */ | |
107 | push %rcx | |
108 | movl $MSR_IA32_CR_PAT, %ecx | |
109 | rdmsr | |
110 | push %rdx /* Save original PAT value */ | |
111 | andl $0xffff00ff, %edx /* Clear PA5 */ | |
112 | orl $0x00000500, %edx /* Set PA5 to WP */ | |
113 | wrmsr | |
114 | pop %rdx /* RDX contains original PAT value */ | |
115 | pop %rcx | |
116 | ||
117 | movq %rcx, %r9 /* Save kernel length */ | |
118 | movq %rdi, %r10 /* Save encrypted kernel address */ | |
119 | movq %rsi, %r11 /* Save decrypted kernel address */ | |
120 | ||
121 | wbinvd /* Invalidate any cache entries */ | |
122 | ||
123 | /* Copy/encrypt 2MB at a time */ | |
124 | 1: | |
125 | movq %r11, %rsi /* Source - decrypted kernel */ | |
126 | movq %r8, %rdi /* Dest - intermediate copy buffer */ | |
127 | movq $PMD_PAGE_SIZE, %rcx /* 2MB length */ | |
128 | rep movsb | |
129 | ||
130 | movq %r8, %rsi /* Source - intermediate copy buffer */ | |
131 | movq %r10, %rdi /* Dest - encrypted kernel */ | |
132 | movq $PMD_PAGE_SIZE, %rcx /* 2MB length */ | |
133 | rep movsb | |
134 | ||
135 | addq $PMD_PAGE_SIZE, %r11 | |
136 | addq $PMD_PAGE_SIZE, %r10 | |
137 | subq $PMD_PAGE_SIZE, %r9 /* Kernel length decrement */ | |
138 | jnz 1b /* Kernel length not zero? */ | |
139 | ||
140 | /* Restore PAT register */ | |
141 | push %rdx /* Save original PAT value */ | |
142 | movl $MSR_IA32_CR_PAT, %ecx | |
143 | rdmsr | |
144 | pop %rdx /* Restore original PAT value */ | |
145 | wrmsr | |
146 | ||
147 | ret | |
148 | .L__enc_copy_end: | |
149 | ENDPROC(__enc_copy) |