mm: init_mlocked_on_free_v3

Implements the "init_mlocked_on_free" boot option. When this boot option
is enabled, any mlock'ed pages are zeroed on free. If
the pages are munlock'ed beforehand, no initialization takes place.
This boot option is meant to combat the performance hit of
"init_on_free" as reported in commit 6471384af2a6 ("mm: security:
introduce init_on_alloc=1 and init_on_free=1 boot options"). With
"init_mlocked_on_free=1" only relevant data is freed while everything
else is left untouched by the kernel. Correspondingly, this patch
introduces no performance hit for unmapping non-mlock'ed memory. The
unmapping overhead for purely mlocked memory was measured to be
approximately 13%. Realistically, most systems mlock only a fraction of
the total memory so the real-world system overhead should be close to
zero.

Optimally, userspace programs clear any key material or other
confidential memory before exit and munlock the according memory
regions. If a program crashes, userspace key managers fail to do this
job. Accordingly, no munlock operations are performed so the data is
caught and zeroed by the kernel. Should the program not crash, all
memory will ideally be munlocked so no overhead is caused.

CONFIG_INIT_MLOCKED_ON_FREE_DEFAULT_ON can be set to enable
"init_mlocked_on_free" by default.

Link: https://lkml.kernel.org/r/20240329145605.149917-1-yjnworkstation@gmail.com
Signed-off-by: York Jasper Niebuhr <yjnworkstation@gmail.com>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: York Jasper Niebuhr <yjnworkstation@gmail.com>
Cc: Kees Cook <keescook@chromium.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index 902ecd92a29fbe83df18d32d1a8fe652c8277132..3ff97de349daf180d437aaf38cdf31d489e842a7 100644 (file)
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -2148,6 +2148,12 @@
                        Format: 0 | 1
                        Default set by CONFIG_INIT_ON_FREE_DEFAULT_ON.
 
+       init_mlocked_on_free=   [MM] Fill freed userspace memory with zeroes if
+                               it was mlock'ed and not explicitly munlock'ed
+                               afterwards.
+                               Format: 0 | 1
+                               Default set by CONFIG_INIT_MLOCKED_ON_FREE_DEFAULT_ON
+
        init_pkru=      [X86] Specify the default memory protection keys rights
                        register contents for all processes.  0x55555554 by
                        default (disallow access to all but pkey 0).  Can

diff --git a/include/linux/mm.h b/include/linux/mm.h
index 2d5e492ef57ffb8c5e78e205c0fc0447cf65627d..4f4e460d78531bf3c489f24ceeee09cad19e4087 100644 (file)
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -3762,7 +3762,14 @@ DECLARE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_FREE_DEFAULT_ON, init_on_free);
 static inline bool want_init_on_free(void)
 {
        return static_branch_maybe(CONFIG_INIT_ON_FREE_DEFAULT_ON,
-                                  &init_on_free);
+                               &init_on_free);
+}
+
+DECLARE_STATIC_KEY_MAYBE(CONFIG_INIT_MLOCKED_ON_FREE_DEFAULT_ON, init_mlocked_on_free);
+static inline bool want_init_mlocked_on_free(void)
+{
+       return static_branch_maybe(CONFIG_INIT_MLOCKED_ON_FREE_DEFAULT_ON,
+                               &init_mlocked_on_free);
 }
 
 extern bool _debug_pagealloc_enabled_early;

diff --git a/mm/internal.h b/mm/internal.h
index 6614ba4ca9dec8356dda0e55233971cd5901e372..cf7799e293916196659a1d93e5d3693de60334b6 100644 (file)
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -506,6 +506,7 @@ extern void __putback_isolated_page(struct page *page, unsigned int order,
 extern void memblock_free_pages(struct page *page, unsigned long pfn,
                                        unsigned int order);
 extern void __free_pages_core(struct page *page, unsigned int order);
+extern void kernel_init_pages(struct page *page, int numpages);
 
 /*
  * This will have no effect, other than possibly generating a warning, if the

diff --git a/mm/memory.c b/mm/memory.c
index 0b92336bcebde1a15c8dabf459245e6f08ba1603..80944acb5b4e4a03795e16fbcb465c1655201d40 100644 (file)
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -1506,6 +1506,12 @@ static __always_inline void zap_present_folio_ptes(struct mmu_gather *tlb,
                if (unlikely(page_mapcount(page) < 0))
                        print_bad_pte(vma, addr, ptent, page);
        }
+
+       if (want_init_mlocked_on_free() && folio_test_mlocked(folio) &&
+           !delay_rmap && folio_test_anon(folio)) {
+               kernel_init_pages(page, folio_nr_pages(folio));
+       }
+
        if (unlikely(__tlb_remove_folio_pages(tlb, page, nr, delay_rmap))) {
                *force_flush = true;
                *force_break = true;

diff --git a/mm/mm_init.c b/mm/mm_init.c
index d01912b8a59735cfb816011bd341ba082dc023d3..2c8f3af4430f5ca37743eb9adc3db030fbd50eb7 100644 (file)
--- a/mm/mm_init.c
+++ b/mm/mm_init.c
@@ -2522,6 +2522,9 @@ EXPORT_SYMBOL(init_on_alloc);
 DEFINE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_FREE_DEFAULT_ON, init_on_free);
 EXPORT_SYMBOL(init_on_free);
 
+DEFINE_STATIC_KEY_MAYBE(CONFIG_INIT_MLOCKED_ON_FREE_DEFAULT_ON, init_mlocked_on_free);
+EXPORT_SYMBOL(init_mlocked_on_free);
+
 static bool _init_on_alloc_enabled_early __read_mostly
                                = IS_ENABLED(CONFIG_INIT_ON_ALLOC_DEFAULT_ON);
 static int __init early_init_on_alloc(char *buf)
@@ -2539,6 +2542,14 @@ static int __init early_init_on_free(char *buf)
 }
 early_param("init_on_free", early_init_on_free);
 
+static bool _init_mlocked_on_free_enabled_early __read_mostly
+                               = IS_ENABLED(CONFIG_INIT_MLOCKED_ON_FREE_DEFAULT_ON);
+static int __init early_init_mlocked_on_free(char *buf)
+{
+       return kstrtobool(buf, &_init_mlocked_on_free_enabled_early);
+}
+early_param("init_mlocked_on_free", early_init_mlocked_on_free);
+
 DEFINE_STATIC_KEY_MAYBE(CONFIG_DEBUG_VM, check_pages_enabled);
 
 /*
@@ -2566,12 +2577,21 @@ static void __init mem_debugging_and_hardening_init(void)
        }
 #endif
 
-       if ((_init_on_alloc_enabled_early || _init_on_free_enabled_early) &&
+       if ((_init_on_alloc_enabled_early || _init_on_free_enabled_early ||
+           _init_mlocked_on_free_enabled_early) &&
            page_poisoning_requested) {
                pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, "
-                       "will take precedence over init_on_alloc and init_on_free\n");
+                       "will take precedence over init_on_alloc, init_on_free "
+                       "and init_mlocked_on_free\n");
                _init_on_alloc_enabled_early = false;
                _init_on_free_enabled_early = false;
+               _init_mlocked_on_free_enabled_early = false;
+       }
+
+       if (_init_mlocked_on_free_enabled_early && _init_on_free_enabled_early) {
+               pr_info("mem auto-init: init_on_free is on, "
+                       "will take precedence over init_mlocked_on_free\n");
+               _init_mlocked_on_free_enabled_early = false;
        }
 
        if (_init_on_alloc_enabled_early) {
@@ -2588,9 +2608,17 @@ static void __init mem_debugging_and_hardening_init(void)
                static_branch_disable(&init_on_free);
        }
 
-       if (IS_ENABLED(CONFIG_KMSAN) &&
-           (_init_on_alloc_enabled_early || _init_on_free_enabled_early))
-               pr_info("mem auto-init: please make sure init_on_alloc and init_on_free are disabled when running KMSAN\n");
+       if (_init_mlocked_on_free_enabled_early) {
+               want_check_pages = true;
+               static_branch_enable(&init_mlocked_on_free);
+       } else {
+               static_branch_disable(&init_mlocked_on_free);
+       }
+
+       if (IS_ENABLED(CONFIG_KMSAN) && (_init_on_alloc_enabled_early ||
+           _init_on_free_enabled_early || _init_mlocked_on_free_enabled_early))
+               pr_info("mem auto-init: please make sure init_on_alloc, init_on_free and "
+                       "init_mlocked_on_free are disabled when running KMSAN\n");
 
 #ifdef CONFIG_DEBUG_PAGEALLOC
        if (debug_pagealloc_enabled()) {
@@ -2629,9 +2657,10 @@ static void __init report_meminit(void)
        else
                stack = "off";
 
-       pr_info("mem auto-init: stack:%s, heap alloc:%s, heap free:%s\n",
+       pr_info("mem auto-init: stack:%s, heap alloc:%s, heap free:%s, mlocked free:%s\n",
                stack, want_init_on_alloc(GFP_KERNEL) ? "on" : "off",
-               want_init_on_free() ? "on" : "off");
+               want_init_on_free() ? "on" : "off",
+               want_init_mlocked_on_free() ? "on" : "off");
        if (want_init_on_free())
                pr_info("mem auto-init: clearing system memory may take some time...\n");
 }

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 47ab0297838a66cb756b332127da34129c2983d6..e030ccf9d5bcd51c9734224b57c5c2f9cbdf931c 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -1032,7 +1032,7 @@ static inline bool should_skip_kasan_poison(struct page *page)
        return page_kasan_tag(page) == KASAN_TAG_KERNEL;
 }
 
-static void kernel_init_pages(struct page *page, int numpages)
+void kernel_init_pages(struct page *page, int numpages)
 {
        int i;
 

diff --git a/security/Kconfig.hardening b/security/Kconfig.hardening
index 2cff851ebfd7e13b955693be9f5818ac6f8bbf03..effbf5982be10f209be080a061358ec922a4a754 100644 (file)
--- a/security/Kconfig.hardening
+++ b/security/Kconfig.hardening
@@ -255,6 +255,21 @@ config INIT_ON_FREE_DEFAULT_ON
          touching "cold" memory areas. Most cases see 3-5% impact. Some
          synthetic workloads have measured as high as 8%.
 
+config INIT_MLOCKED_ON_FREE_DEFAULT_ON
+       bool "Enable mlocked memory zeroing on free"
+       depends on !KMSAN
+       help
+         This config has the effect of setting "init_mlocked_on_free=1"
+         on the kernel command line. If it is enabled, all mlocked process
+         memory is zeroed when freed. This restriction to mlocked memory
+         improves performance over "init_on_free" but can still be used to
+         protect confidential data like key material from content exposures
+         to other processes, as well as live forensics and cold boot attacks.
+         Any non-mlocked memory is not cleared before it is reassigned. This
+         configuration can be overwritten by setting "init_mlocked_on_free=0"
+         on the command line. The "init_on_free" boot option takes
+         precedence over "init_mlocked_on_free".
+
 config CC_HAS_ZERO_CALL_USED_REGS
        def_bool $(cc-option,-fzero-call-used-regs=used-gpr)
        # https://github.com/ClangBuiltLinux/linux/issues/1766