| 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
| 2 | /* |
| 3 | * AMD Encrypted Register State Support |
| 4 | * |
| 5 | * Author: Joerg Roedel <jroedel@suse.de> |
| 6 | */ |
| 7 | |
| 8 | #ifndef __ASM_ENCRYPTED_STATE_H |
| 9 | #define __ASM_ENCRYPTED_STATE_H |
| 10 | |
| 11 | #include <linux/types.h> |
| 12 | #include <linux/sev-guest.h> |
| 13 | |
| 14 | #include <asm/insn.h> |
| 15 | #include <asm/sev-common.h> |
| 16 | #include <asm/coco.h> |
| 17 | |
| 18 | #define GHCB_PROTOCOL_MIN 1ULL |
| 19 | #define GHCB_PROTOCOL_MAX 2ULL |
| 20 | #define GHCB_DEFAULT_USAGE 0ULL |
| 21 | |
| 22 | #define VMGEXIT() { asm volatile("rep; vmmcall\n\r"); } |
| 23 | |
| 24 | struct boot_params; |
| 25 | |
| 26 | enum es_result { |
| 27 | ES_OK, /* All good */ |
| 28 | ES_UNSUPPORTED, /* Requested operation not supported */ |
| 29 | ES_VMM_ERROR, /* Unexpected state from the VMM */ |
| 30 | ES_DECODE_FAILED, /* Instruction decoding failed */ |
| 31 | ES_EXCEPTION, /* Instruction caused exception */ |
| 32 | ES_RETRY, /* Retry instruction emulation */ |
| 33 | }; |
| 34 | |
| 35 | struct es_fault_info { |
| 36 | unsigned long vector; |
| 37 | unsigned long error_code; |
| 38 | unsigned long cr2; |
| 39 | }; |
| 40 | |
| 41 | struct pt_regs; |
| 42 | |
| 43 | /* ES instruction emulation context */ |
| 44 | struct es_em_ctxt { |
| 45 | struct pt_regs *regs; |
| 46 | struct insn insn; |
| 47 | struct es_fault_info fi; |
| 48 | }; |
| 49 | |
| 50 | /* |
| 51 | * AMD SEV Confidential computing blob structure. The structure is |
| 52 | * defined in OVMF UEFI firmware header: |
| 53 | * https://github.com/tianocore/edk2/blob/master/OvmfPkg/Include/Guid/ConfidentialComputingSevSnpBlob.h |
| 54 | */ |
| 55 | #define CC_BLOB_SEV_HDR_MAGIC 0x45444d41 |
| 56 | struct cc_blob_sev_info { |
| 57 | u32 magic; |
| 58 | u16 version; |
| 59 | u16 reserved; |
| 60 | u64 secrets_phys; |
| 61 | u32 secrets_len; |
| 62 | u32 rsvd1; |
| 63 | u64 cpuid_phys; |
| 64 | u32 cpuid_len; |
| 65 | u32 rsvd2; |
| 66 | } __packed; |
| 67 | |
| 68 | void do_vc_no_ghcb(struct pt_regs *regs, unsigned long exit_code); |
| 69 | |
| 70 | static inline u64 lower_bits(u64 val, unsigned int bits) |
| 71 | { |
| 72 | u64 mask = (1ULL << bits) - 1; |
| 73 | |
| 74 | return (val & mask); |
| 75 | } |
| 76 | |
| 77 | struct real_mode_header; |
| 78 | enum stack_type; |
| 79 | |
| 80 | /* Early IDT entry points for #VC handler */ |
| 81 | extern void vc_no_ghcb(void); |
| 82 | extern void vc_boot_ghcb(void); |
| 83 | extern bool handle_vc_boot_ghcb(struct pt_regs *regs); |
| 84 | |
| 85 | /* PVALIDATE return codes */ |
| 86 | #define PVALIDATE_FAIL_SIZEMISMATCH 6 |
| 87 | |
| 88 | /* Software defined (when rFlags.CF = 1) */ |
| 89 | #define PVALIDATE_FAIL_NOUPDATE 255 |
| 90 | |
| 91 | /* RMUPDATE detected 4K page and 2MB page overlap. */ |
| 92 | #define RMPUPDATE_FAIL_OVERLAP 4 |
| 93 | |
| 94 | /* PSMASH failed due to concurrent access by another CPU */ |
| 95 | #define PSMASH_FAIL_INUSE 3 |
| 96 | |
| 97 | /* RMP page size */ |
| 98 | #define RMP_PG_SIZE_4K 0 |
| 99 | #define RMP_PG_SIZE_2M 1 |
| 100 | #define RMP_TO_PG_LEVEL(level) (((level) == RMP_PG_SIZE_4K) ? PG_LEVEL_4K : PG_LEVEL_2M) |
| 101 | #define PG_LEVEL_TO_RMP(level) (((level) == PG_LEVEL_4K) ? RMP_PG_SIZE_4K : RMP_PG_SIZE_2M) |
| 102 | |
| 103 | struct rmp_state { |
| 104 | u64 gpa; |
| 105 | u8 assigned; |
| 106 | u8 pagesize; |
| 107 | u8 immutable; |
| 108 | u8 rsvd; |
| 109 | u32 asid; |
| 110 | } __packed; |
| 111 | |
| 112 | #define RMPADJUST_VMSA_PAGE_BIT BIT(16) |
| 113 | |
| 114 | /* SNP Guest message request */ |
| 115 | struct snp_req_data { |
| 116 | unsigned long req_gpa; |
| 117 | unsigned long resp_gpa; |
| 118 | unsigned long data_gpa; |
| 119 | unsigned int data_npages; |
| 120 | }; |
| 121 | |
| 122 | #define MAX_AUTHTAG_LEN 32 |
| 123 | |
| 124 | /* See SNP spec SNP_GUEST_REQUEST section for the structure */ |
| 125 | enum msg_type { |
| 126 | SNP_MSG_TYPE_INVALID = 0, |
| 127 | SNP_MSG_CPUID_REQ, |
| 128 | SNP_MSG_CPUID_RSP, |
| 129 | SNP_MSG_KEY_REQ, |
| 130 | SNP_MSG_KEY_RSP, |
| 131 | SNP_MSG_REPORT_REQ, |
| 132 | SNP_MSG_REPORT_RSP, |
| 133 | SNP_MSG_EXPORT_REQ, |
| 134 | SNP_MSG_EXPORT_RSP, |
| 135 | SNP_MSG_IMPORT_REQ, |
| 136 | SNP_MSG_IMPORT_RSP, |
| 137 | SNP_MSG_ABSORB_REQ, |
| 138 | SNP_MSG_ABSORB_RSP, |
| 139 | SNP_MSG_VMRK_REQ, |
| 140 | SNP_MSG_VMRK_RSP, |
| 141 | |
| 142 | SNP_MSG_TYPE_MAX |
| 143 | }; |
| 144 | |
| 145 | enum aead_algo { |
| 146 | SNP_AEAD_INVALID, |
| 147 | SNP_AEAD_AES_256_GCM, |
| 148 | }; |
| 149 | |
| 150 | struct snp_guest_msg_hdr { |
| 151 | u8 authtag[MAX_AUTHTAG_LEN]; |
| 152 | u64 msg_seqno; |
| 153 | u8 rsvd1[8]; |
| 154 | u8 algo; |
| 155 | u8 hdr_version; |
| 156 | u16 hdr_sz; |
| 157 | u8 msg_type; |
| 158 | u8 msg_version; |
| 159 | u16 msg_sz; |
| 160 | u32 rsvd2; |
| 161 | u8 msg_vmpck; |
| 162 | u8 rsvd3[35]; |
| 163 | } __packed; |
| 164 | |
| 165 | struct snp_guest_msg { |
| 166 | struct snp_guest_msg_hdr hdr; |
| 167 | u8 payload[4000]; |
| 168 | } __packed; |
| 169 | |
| 170 | struct sev_guest_platform_data { |
| 171 | u64 secrets_gpa; |
| 172 | }; |
| 173 | |
| 174 | /* |
| 175 | * The secrets page contains 96-bytes of reserved field that can be used by |
| 176 | * the guest OS. The guest OS uses the area to save the message sequence |
| 177 | * number for each VMPCK. |
| 178 | * |
| 179 | * See the GHCB spec section Secret page layout for the format for this area. |
| 180 | */ |
| 181 | struct secrets_os_area { |
| 182 | u32 msg_seqno_0; |
| 183 | u32 msg_seqno_1; |
| 184 | u32 msg_seqno_2; |
| 185 | u32 msg_seqno_3; |
| 186 | u64 ap_jump_table_pa; |
| 187 | u8 rsvd[40]; |
| 188 | u8 guest_usage[32]; |
| 189 | } __packed; |
| 190 | |
| 191 | #define VMPCK_KEY_LEN 32 |
| 192 | |
| 193 | /* See the SNP spec version 0.9 for secrets page format */ |
| 194 | struct snp_secrets_page { |
| 195 | u32 version; |
| 196 | u32 imien : 1, |
| 197 | rsvd1 : 31; |
| 198 | u32 fms; |
| 199 | u32 rsvd2; |
| 200 | u8 gosvw[16]; |
| 201 | u8 vmpck0[VMPCK_KEY_LEN]; |
| 202 | u8 vmpck1[VMPCK_KEY_LEN]; |
| 203 | u8 vmpck2[VMPCK_KEY_LEN]; |
| 204 | u8 vmpck3[VMPCK_KEY_LEN]; |
| 205 | struct secrets_os_area os_area; |
| 206 | |
| 207 | u8 vmsa_tweak_bitmap[64]; |
| 208 | |
| 209 | /* SVSM fields */ |
| 210 | u64 svsm_base; |
| 211 | u64 svsm_size; |
| 212 | u64 svsm_caa; |
| 213 | u32 svsm_max_version; |
| 214 | u8 svsm_guest_vmpl; |
| 215 | u8 rsvd3[3]; |
| 216 | |
| 217 | /* Remainder of page */ |
| 218 | u8 rsvd4[3744]; |
| 219 | } __packed; |
| 220 | |
| 221 | /* |
| 222 | * The SVSM Calling Area (CA) related structures. |
| 223 | */ |
| 224 | struct svsm_ca { |
| 225 | u8 call_pending; |
| 226 | u8 mem_available; |
| 227 | u8 rsvd1[6]; |
| 228 | |
| 229 | u8 svsm_buffer[PAGE_SIZE - 8]; |
| 230 | }; |
| 231 | |
| 232 | #define SVSM_SUCCESS 0 |
| 233 | #define SVSM_ERR_INCOMPLETE 0x80000000 |
| 234 | #define SVSM_ERR_UNSUPPORTED_PROTOCOL 0x80000001 |
| 235 | #define SVSM_ERR_UNSUPPORTED_CALL 0x80000002 |
| 236 | #define SVSM_ERR_INVALID_ADDRESS 0x80000003 |
| 237 | #define SVSM_ERR_INVALID_FORMAT 0x80000004 |
| 238 | #define SVSM_ERR_INVALID_PARAMETER 0x80000005 |
| 239 | #define SVSM_ERR_INVALID_REQUEST 0x80000006 |
| 240 | #define SVSM_ERR_BUSY 0x80000007 |
| 241 | #define SVSM_PVALIDATE_FAIL_SIZEMISMATCH 0x80001006 |
| 242 | |
| 243 | /* |
| 244 | * The SVSM PVALIDATE related structures |
| 245 | */ |
| 246 | struct svsm_pvalidate_entry { |
| 247 | u64 page_size : 2, |
| 248 | action : 1, |
| 249 | ignore_cf : 1, |
| 250 | rsvd : 8, |
| 251 | pfn : 52; |
| 252 | }; |
| 253 | |
| 254 | struct svsm_pvalidate_call { |
| 255 | u16 num_entries; |
| 256 | u16 cur_index; |
| 257 | |
| 258 | u8 rsvd1[4]; |
| 259 | |
| 260 | struct svsm_pvalidate_entry entry[]; |
| 261 | }; |
| 262 | |
| 263 | #define SVSM_PVALIDATE_MAX_COUNT ((sizeof_field(struct svsm_ca, svsm_buffer) - \ |
| 264 | offsetof(struct svsm_pvalidate_call, entry)) / \ |
| 265 | sizeof(struct svsm_pvalidate_entry)) |
| 266 | |
| 267 | /* |
| 268 | * The SVSM Attestation related structures |
| 269 | */ |
| 270 | struct svsm_loc_entry { |
| 271 | u64 pa; |
| 272 | u32 len; |
| 273 | u8 rsvd[4]; |
| 274 | }; |
| 275 | |
| 276 | struct svsm_attest_call { |
| 277 | struct svsm_loc_entry report_buf; |
| 278 | struct svsm_loc_entry nonce; |
| 279 | struct svsm_loc_entry manifest_buf; |
| 280 | struct svsm_loc_entry certificates_buf; |
| 281 | |
| 282 | /* For attesting a single service */ |
| 283 | u8 service_guid[16]; |
| 284 | u32 service_manifest_ver; |
| 285 | u8 rsvd[4]; |
| 286 | }; |
| 287 | |
| 288 | /* |
| 289 | * SVSM protocol structure |
| 290 | */ |
| 291 | struct svsm_call { |
| 292 | struct svsm_ca *caa; |
| 293 | u64 rax; |
| 294 | u64 rcx; |
| 295 | u64 rdx; |
| 296 | u64 r8; |
| 297 | u64 r9; |
| 298 | u64 rax_out; |
| 299 | u64 rcx_out; |
| 300 | u64 rdx_out; |
| 301 | u64 r8_out; |
| 302 | u64 r9_out; |
| 303 | }; |
| 304 | |
| 305 | #define SVSM_CORE_CALL(x) ((0ULL << 32) | (x)) |
| 306 | #define SVSM_CORE_REMAP_CA 0 |
| 307 | #define SVSM_CORE_PVALIDATE 1 |
| 308 | #define SVSM_CORE_CREATE_VCPU 2 |
| 309 | #define SVSM_CORE_DELETE_VCPU 3 |
| 310 | |
| 311 | #define SVSM_ATTEST_CALL(x) ((1ULL << 32) | (x)) |
| 312 | #define SVSM_ATTEST_SERVICES 0 |
| 313 | #define SVSM_ATTEST_SINGLE_SERVICE 1 |
| 314 | |
| 315 | #ifdef CONFIG_AMD_MEM_ENCRYPT |
| 316 | |
| 317 | extern u8 snp_vmpl; |
| 318 | |
| 319 | extern void __sev_es_ist_enter(struct pt_regs *regs); |
| 320 | extern void __sev_es_ist_exit(void); |
| 321 | static __always_inline void sev_es_ist_enter(struct pt_regs *regs) |
| 322 | { |
| 323 | if (cc_vendor == CC_VENDOR_AMD && |
| 324 | cc_platform_has(CC_ATTR_GUEST_STATE_ENCRYPT)) |
| 325 | __sev_es_ist_enter(regs); |
| 326 | } |
| 327 | static __always_inline void sev_es_ist_exit(void) |
| 328 | { |
| 329 | if (cc_vendor == CC_VENDOR_AMD && |
| 330 | cc_platform_has(CC_ATTR_GUEST_STATE_ENCRYPT)) |
| 331 | __sev_es_ist_exit(); |
| 332 | } |
| 333 | extern int sev_es_setup_ap_jump_table(struct real_mode_header *rmh); |
| 334 | extern void __sev_es_nmi_complete(void); |
| 335 | static __always_inline void sev_es_nmi_complete(void) |
| 336 | { |
| 337 | if (cc_vendor == CC_VENDOR_AMD && |
| 338 | cc_platform_has(CC_ATTR_GUEST_STATE_ENCRYPT)) |
| 339 | __sev_es_nmi_complete(); |
| 340 | } |
| 341 | extern int __init sev_es_efi_map_ghcbs(pgd_t *pgd); |
| 342 | extern void sev_enable(struct boot_params *bp); |
| 343 | |
| 344 | /* |
| 345 | * RMPADJUST modifies the RMP permissions of a page of a lesser- |
| 346 | * privileged (numerically higher) VMPL. |
| 347 | * |
| 348 | * If the guest is running at a higher-privilege than the privilege |
| 349 | * level the instruction is targeting, the instruction will succeed, |
| 350 | * otherwise, it will fail. |
| 351 | */ |
| 352 | static inline int rmpadjust(unsigned long vaddr, bool rmp_psize, unsigned long attrs) |
| 353 | { |
| 354 | int rc; |
| 355 | |
| 356 | /* "rmpadjust" mnemonic support in binutils 2.36 and newer */ |
| 357 | asm volatile(".byte 0xF3,0x0F,0x01,0xFE\n\t" |
| 358 | : "=a"(rc) |
| 359 | : "a"(vaddr), "c"(rmp_psize), "d"(attrs) |
| 360 | : "memory", "cc"); |
| 361 | |
| 362 | return rc; |
| 363 | } |
| 364 | static inline int pvalidate(unsigned long vaddr, bool rmp_psize, bool validate) |
| 365 | { |
| 366 | bool no_rmpupdate; |
| 367 | int rc; |
| 368 | |
| 369 | /* "pvalidate" mnemonic support in binutils 2.36 and newer */ |
| 370 | asm volatile(".byte 0xF2, 0x0F, 0x01, 0xFF\n\t" |
| 371 | CC_SET(c) |
| 372 | : CC_OUT(c) (no_rmpupdate), "=a"(rc) |
| 373 | : "a"(vaddr), "c"(rmp_psize), "d"(validate) |
| 374 | : "memory", "cc"); |
| 375 | |
| 376 | if (no_rmpupdate) |
| 377 | return PVALIDATE_FAIL_NOUPDATE; |
| 378 | |
| 379 | return rc; |
| 380 | } |
| 381 | |
| 382 | struct snp_guest_request_ioctl; |
| 383 | |
| 384 | void setup_ghcb(void); |
| 385 | void early_snp_set_memory_private(unsigned long vaddr, unsigned long paddr, |
| 386 | unsigned long npages); |
| 387 | void early_snp_set_memory_shared(unsigned long vaddr, unsigned long paddr, |
| 388 | unsigned long npages); |
| 389 | void snp_set_memory_shared(unsigned long vaddr, unsigned long npages); |
| 390 | void snp_set_memory_private(unsigned long vaddr, unsigned long npages); |
| 391 | void snp_set_wakeup_secondary_cpu(void); |
| 392 | bool snp_init(struct boot_params *bp); |
| 393 | void __noreturn snp_abort(void); |
| 394 | void snp_dmi_setup(void); |
| 395 | int snp_issue_guest_request(u64 exit_code, struct snp_req_data *input, struct snp_guest_request_ioctl *rio); |
| 396 | int snp_issue_svsm_attest_req(u64 call_id, struct svsm_call *call, struct svsm_attest_call *input); |
| 397 | void snp_accept_memory(phys_addr_t start, phys_addr_t end); |
| 398 | u64 snp_get_unsupported_features(u64 status); |
| 399 | u64 sev_get_status(void); |
| 400 | void sev_show_status(void); |
| 401 | void snp_update_svsm_ca(void); |
| 402 | |
| 403 | #else /* !CONFIG_AMD_MEM_ENCRYPT */ |
| 404 | |
| 405 | #define snp_vmpl 0 |
| 406 | static inline void sev_es_ist_enter(struct pt_regs *regs) { } |
| 407 | static inline void sev_es_ist_exit(void) { } |
| 408 | static inline int sev_es_setup_ap_jump_table(struct real_mode_header *rmh) { return 0; } |
| 409 | static inline void sev_es_nmi_complete(void) { } |
| 410 | static inline int sev_es_efi_map_ghcbs(pgd_t *pgd) { return 0; } |
| 411 | static inline void sev_enable(struct boot_params *bp) { } |
| 412 | static inline int pvalidate(unsigned long vaddr, bool rmp_psize, bool validate) { return 0; } |
| 413 | static inline int rmpadjust(unsigned long vaddr, bool rmp_psize, unsigned long attrs) { return 0; } |
| 414 | static inline void setup_ghcb(void) { } |
| 415 | static inline void __init |
| 416 | early_snp_set_memory_private(unsigned long vaddr, unsigned long paddr, unsigned long npages) { } |
| 417 | static inline void __init |
| 418 | early_snp_set_memory_shared(unsigned long vaddr, unsigned long paddr, unsigned long npages) { } |
| 419 | static inline void snp_set_memory_shared(unsigned long vaddr, unsigned long npages) { } |
| 420 | static inline void snp_set_memory_private(unsigned long vaddr, unsigned long npages) { } |
| 421 | static inline void snp_set_wakeup_secondary_cpu(void) { } |
| 422 | static inline bool snp_init(struct boot_params *bp) { return false; } |
| 423 | static inline void snp_abort(void) { } |
| 424 | static inline void snp_dmi_setup(void) { } |
| 425 | static inline int snp_issue_guest_request(u64 exit_code, struct snp_req_data *input, struct snp_guest_request_ioctl *rio) |
| 426 | { |
| 427 | return -ENOTTY; |
| 428 | } |
| 429 | static inline int snp_issue_svsm_attest_req(u64 call_id, struct svsm_call *call, struct svsm_attest_call *input) |
| 430 | { |
| 431 | return -ENOTTY; |
| 432 | } |
| 433 | static inline void snp_accept_memory(phys_addr_t start, phys_addr_t end) { } |
| 434 | static inline u64 snp_get_unsupported_features(u64 status) { return 0; } |
| 435 | static inline u64 sev_get_status(void) { return 0; } |
| 436 | static inline void sev_show_status(void) { } |
| 437 | static inline void snp_update_svsm_ca(void) { } |
| 438 | |
| 439 | #endif /* CONFIG_AMD_MEM_ENCRYPT */ |
| 440 | |
| 441 | #ifdef CONFIG_KVM_AMD_SEV |
| 442 | bool snp_probe_rmptable_info(void); |
| 443 | int snp_lookup_rmpentry(u64 pfn, bool *assigned, int *level); |
| 444 | void snp_dump_hva_rmpentry(unsigned long address); |
| 445 | int psmash(u64 pfn); |
| 446 | int rmp_make_private(u64 pfn, u64 gpa, enum pg_level level, u32 asid, bool immutable); |
| 447 | int rmp_make_shared(u64 pfn, enum pg_level level); |
| 448 | void snp_leak_pages(u64 pfn, unsigned int npages); |
| 449 | void kdump_sev_callback(void); |
| 450 | void snp_fixup_e820_tables(void); |
| 451 | #else |
| 452 | static inline bool snp_probe_rmptable_info(void) { return false; } |
| 453 | static inline int snp_lookup_rmpentry(u64 pfn, bool *assigned, int *level) { return -ENODEV; } |
| 454 | static inline void snp_dump_hva_rmpentry(unsigned long address) {} |
| 455 | static inline int psmash(u64 pfn) { return -ENODEV; } |
| 456 | static inline int rmp_make_private(u64 pfn, u64 gpa, enum pg_level level, u32 asid, |
| 457 | bool immutable) |
| 458 | { |
| 459 | return -ENODEV; |
| 460 | } |
| 461 | static inline int rmp_make_shared(u64 pfn, enum pg_level level) { return -ENODEV; } |
| 462 | static inline void snp_leak_pages(u64 pfn, unsigned int npages) {} |
| 463 | static inline void kdump_sev_callback(void) { } |
| 464 | static inline void snp_fixup_e820_tables(void) {} |
| 465 | #endif |
| 466 | |
| 467 | #endif |