Merge tag 'mm-nonmm-stable-2024-05-19-11-56' of git://git.kernel.org/pub/scm/linux...

[linux-block.git] / arch / x86 / lib / copy_mc.c

diff --git a/arch/x86/lib/copy_mc.c b/arch/x86/lib/copy_mc.c
index 6e8b7e600def57913748a93a085c87a87eab6063..97e88e58567bf82c579e05fabef892b48831beae 100644 (file)
--- a/arch/x86/lib/copy_mc.c
+++ b/arch/x86/lib/copy_mc.c
@@ -4,6 +4,7 @@
 #include <linux/jump_label.h>
 #include <linux/uaccess.h>
 #include <linux/export.h>
+#include <linux/instrumented.h>
 #include <linux/string.h>
 #include <linux/types.h>
 
@@ -61,10 +62,20 @@ unsigned long copy_mc_enhanced_fast_string(void *dst, const void *src, unsigned
  */
 unsigned long __must_check copy_mc_to_kernel(void *dst, const void *src, unsigned len)
 {
-       if (copy_mc_fragile_enabled)
-               return copy_mc_fragile(dst, src, len);
-       if (static_cpu_has(X86_FEATURE_ERMS))
-               return copy_mc_enhanced_fast_string(dst, src, len);
+       unsigned long ret;
+
+       if (copy_mc_fragile_enabled) {
+               instrument_memcpy_before(dst, src, len);
+               ret = copy_mc_fragile(dst, src, len);
+               instrument_memcpy_after(dst, src, len, ret);
+               return ret;
+       }
+       if (static_cpu_has(X86_FEATURE_ERMS)) {
+               instrument_memcpy_before(dst, src, len);
+               ret = copy_mc_enhanced_fast_string(dst, src, len);
+               instrument_memcpy_after(dst, src, len, ret);
+               return ret;
+       }
        memcpy(dst, src, len);
        return 0;
 }
@@ -75,6 +86,7 @@ unsigned long __must_check copy_mc_to_user(void __user *dst, const void *src, un
        unsigned long ret;
 
        if (copy_mc_fragile_enabled) {
+               instrument_copy_to_user(dst, src, len);
                __uaccess_begin();
                ret = copy_mc_fragile((__force void *)dst, src, len);
                __uaccess_end();
@@ -82,6 +94,7 @@ unsigned long __must_check copy_mc_to_user(void __user *dst, const void *src, un
        }
 
        if (static_cpu_has(X86_FEATURE_ERMS)) {
+               instrument_copy_to_user(dst, src, len);
                __uaccess_begin();
                ret = copy_mc_enhanced_fast_string((__force void *)dst, src, len);
                __uaccess_end();