diff options
Diffstat (limited to 'arch/x86/mm/mem_encrypt_amd.c')
| -rw-r--r-- | arch/x86/mm/mem_encrypt_amd.c | 98 |
1 files changed, 65 insertions, 33 deletions
diff --git a/arch/x86/mm/mem_encrypt_amd.c b/arch/x86/mm/mem_encrypt_amd.c index 422602f6039b..7490ff6d83b1 100644 --- a/arch/x86/mm/mem_encrypt_amd.c +++ b/arch/x86/mm/mem_encrypt_amd.c @@ -2,13 +2,11 @@ /* * AMD Memory Encryption Support * - * Copyright (C) 2016 Advanced Micro Devices, Inc. + * Copyright (C) 2016-2024 Advanced Micro Devices, Inc. * * Author: Tom Lendacky <thomas.lendacky@amd.com> */ -#define DISABLE_BRANCH_PROFILING - #include <linux/linkage.h> #include <linux/init.h> #include <linux/mm.h> @@ -283,7 +281,7 @@ static void enc_dec_hypercall(unsigned long vaddr, unsigned long size, bool enc) #endif } -static bool amd_enc_status_change_prepare(unsigned long vaddr, int npages, bool enc) +static int amd_enc_status_change_prepare(unsigned long vaddr, int npages, bool enc) { /* * To maintain the security guarantees of SEV-SNP guests, make sure @@ -292,11 +290,11 @@ static bool amd_enc_status_change_prepare(unsigned long vaddr, int npages, bool if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP) && !enc) snp_set_memory_shared(vaddr, npages); - return true; + return 0; } /* Return true unconditionally: return value doesn't matter for the SEV side */ -static bool amd_enc_status_change_finish(unsigned long vaddr, int npages, bool enc) +static int amd_enc_status_change_finish(unsigned long vaddr, int npages, bool enc) { /* * After memory is mapped encrypted in the page table, validate it @@ -308,62 +306,85 @@ static bool amd_enc_status_change_finish(unsigned long vaddr, int npages, bool e if (!cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT)) enc_dec_hypercall(vaddr, npages << PAGE_SHIFT, enc); - return true; + return 0; } -static void __init __set_clr_pte_enc(pte_t *kpte, int level, bool enc) +int prepare_pte_enc(struct pte_enc_desc *d) { - pgprot_t old_prot, new_prot; - unsigned long pfn, pa, size; - pte_t new_pte; + pgprot_t old_prot; - pfn = pg_level_to_pfn(level, kpte, &old_prot); - if (!pfn) - return; + d->pfn = pg_level_to_pfn(d->pte_level, d->kpte, &old_prot); + if (!d->pfn) + return 1; - new_prot = old_prot; - if (enc) - pgprot_val(new_prot) |= _PAGE_ENC; + d->new_pgprot = old_prot; + if (d->encrypt) + pgprot_val(d->new_pgprot) |= _PAGE_ENC; else - pgprot_val(new_prot) &= ~_PAGE_ENC; + pgprot_val(d->new_pgprot) &= ~_PAGE_ENC; /* If prot is same then do nothing. */ - if (pgprot_val(old_prot) == pgprot_val(new_prot)) - return; + if (pgprot_val(old_prot) == pgprot_val(d->new_pgprot)) + return 1; - pa = pfn << PAGE_SHIFT; - size = page_level_size(level); + d->pa = d->pfn << PAGE_SHIFT; + d->size = page_level_size(d->pte_level); /* - * We are going to perform in-place en-/decryption and change the - * physical page attribute from C=1 to C=0 or vice versa. Flush the - * caches to ensure that data gets accessed with the correct C-bit. + * In-place en-/decryption and physical page attribute change + * from C=1 to C=0 or vice versa will be performed. Flush the + * caches to ensure that data gets accessed with the correct + * C-bit. */ - clflush_cache_range(__va(pa), size); + if (d->va) + clflush_cache_range(d->va, d->size); + else + clflush_cache_range(__va(d->pa), d->size); + + return 0; +} + +void set_pte_enc_mask(pte_t *kpte, unsigned long pfn, pgprot_t new_prot) +{ + pte_t new_pte; + + /* Change the page encryption mask. */ + new_pte = pfn_pte(pfn, new_prot); + set_pte_atomic(kpte, new_pte); +} + +static void __init __set_clr_pte_enc(pte_t *kpte, int level, bool enc) +{ + struct pte_enc_desc d = { + .kpte = kpte, + .pte_level = level, + .encrypt = enc + }; + + if (prepare_pte_enc(&d)) + return; /* Encrypt/decrypt the contents in-place */ if (enc) { - sme_early_encrypt(pa, size); + sme_early_encrypt(d.pa, d.size); } else { - sme_early_decrypt(pa, size); + sme_early_decrypt(d.pa, d.size); /* * ON SNP, the page state in the RMP table must happen * before the page table updates. */ - early_snp_set_memory_shared((unsigned long)__va(pa), pa, 1); + early_snp_set_memory_shared((unsigned long)__va(d.pa), d.pa, 1); } - /* Change the page encryption mask. */ - new_pte = pfn_pte(pfn, new_prot); - set_pte_atomic(kpte, new_pte); + set_pte_enc_mask(kpte, d.pfn, d.new_pgprot); /* * If page is set encrypted in the page table, then update the RMP table to * add this page as private. */ if (enc) - early_snp_set_memory_private((unsigned long)__va(pa), pa, 1); + early_snp_set_memory_private((unsigned long)__va(d.pa), d.pa, 1); } static int __init early_set_memory_enc_dec(unsigned long vaddr, @@ -467,6 +488,8 @@ void __init sme_early_init(void) x86_platform.guest.enc_status_change_finish = amd_enc_status_change_finish; x86_platform.guest.enc_tlb_flush_required = amd_enc_tlb_flush_required; x86_platform.guest.enc_cache_flush_required = amd_enc_cache_flush_required; + x86_platform.guest.enc_kexec_begin = snp_kexec_begin; + x86_platform.guest.enc_kexec_finish = snp_kexec_finish; /* * AMD-SEV-ES intercepts the RDMSR to read the X2APIC ID in the @@ -510,6 +533,15 @@ void __init sme_early_init(void) */ x86_init.resources.dmi_setup = snp_dmi_setup; } + + /* + * Switch the SVSM CA mapping (if active) from identity mapped to + * kernel mapped. + */ + snp_update_svsm_ca(); + + if (sev_status & MSR_AMD64_SNP_SECURE_TSC) + setup_force_cpu_cap(X86_FEATURE_TSC_RELIABLE); } void __init mem_encrypt_free_decrypted_mem(void) |