mm: Introduce untagged_addr_remote()

untagged_addr() removes tags/metadata from the address and brings it to
the canonical form. The helper is implemented on arm64 and sparc. Both of
them do untagging based on global rules.

However, Linear Address Masking (LAM) on x86 introduces per-process
settings for untagging. As a result, untagged_addr() is now only
suitable for untagging addresses for the current proccess.

The new helper untagged_addr_remote() has to be used when the address
targets remote process. It requires the mmap lock for target mm to be
taken.

Signed-off-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Signed-off-by: Dave Hansen <dave.hansen@linux.intel.com>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Tested-by: Alexander Potapenko <glider@google.com>
Link: https://lore.kernel.org/all/20230312112612.31869-6-kirill.shutemov%40linux.intel.com

diff --git a/arch/sparc/include/asm/uaccess_64.h b/arch/sparc/include/asm/uaccess_64.h
index 94266a5c5b0435aea9d042be483a1b93d2e82174..b825a5dd0210ea47bd9fb3905071eced2200bcc2 100644 (file)
--- a/arch/sparc/include/asm/uaccess_64.h
+++ b/arch/sparc/include/asm/uaccess_64.h
@@ -8,8 +8,10 @@
 
 #include <linux/compiler.h>
 #include <linux/string.h>
+#include <linux/mm_types.h>
 #include <asm/asi.h>
 #include <asm/spitfire.h>
+#include <asm/pgtable.h>
 
 #include <asm/processor.h>
 #include <asm-generic/access_ok.h>

diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c
index 493c31de0edb9b5fb73b6c36e9059204183b458e..3d4dd9420c30701453be9323a0e3e46bfbf58755 100644 (file)
--- a/drivers/vfio/vfio_iommu_type1.c
+++ b/drivers/vfio/vfio_iommu_type1.c
@@ -580,7 +580,7 @@ static int vaddr_get_pfns(struct mm_struct *mm, unsigned long vaddr,
                goto done;
        }
 
-       vaddr = untagged_addr(vaddr);
+       vaddr = untagged_addr_remote(mm, vaddr);
 
 retry:
        vma = vma_lookup(mm, vaddr);

diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c
index 6a96e1713fd588af12fb1a94d7af57d53d0cb322..29fd6b1f40580d0908dc04b30cb25c8a73e1592e 100644 (file)
--- a/fs/proc/task_mmu.c
+++ b/fs/proc/task_mmu.c
@@ -1689,8 +1689,13 @@ static ssize_t pagemap_read(struct file *file, char __user *buf,
 
        /* watch out for wraparound */
        start_vaddr = end_vaddr;
-       if (svpfn <= (ULONG_MAX >> PAGE_SHIFT))
-               start_vaddr = untagged_addr(svpfn << PAGE_SHIFT);
+       if (svpfn <= (ULONG_MAX >> PAGE_SHIFT)) {
+               ret = mmap_read_lock_killable(mm);
+               if (ret)
+                       goto out_free;
+               start_vaddr = untagged_addr_remote(mm, svpfn << PAGE_SHIFT);
+               mmap_read_unlock(mm);
+       }
 
        /* Ensure the address is inside the task */
        if (start_vaddr > mm->task_size)

diff --git a/include/linux/mm.h b/include/linux/mm.h
index 1f79667824eb60cf4ae79353ab014a9cc04eecc2..289ae4caf87869498acc86660eebc73ffb784e1d 100644 (file)
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -96,17 +96,6 @@ extern int mmap_rnd_compat_bits __read_mostly;
 #include <asm/page.h>
 #include <asm/processor.h>
 
-/*
- * Architectures that support memory tagging (assigning tags to memory regions,
- * embedding these tags into addresses that point to these memory regions, and
- * checking that the memory and the pointer tags match on memory accesses)
- * redefine this macro to strip tags from pointers.
- * It's defined as noop for architectures that don't support memory tagging.
- */
-#ifndef untagged_addr
-#define untagged_addr(addr) (addr)
-#endif
-
 #ifndef __pa_symbol
 #define __pa_symbol(x)  __pa(RELOC_HIDE((unsigned long)(x), 0))
 #endif

diff --git a/include/linux/uaccess.h b/include/linux/uaccess.h
index ab9728138ad67f3f7566292060ac8470c43f14a0..3064314f48329cac729b9b4e29fc62072ec2df3b 100644 (file)
--- a/include/linux/uaccess.h
+++ b/include/linux/uaccess.h
@@ -10,6 +10,28 @@
 
 #include <asm/uaccess.h>
 
+/*
+ * Architectures that support memory tagging (assigning tags to memory regions,
+ * embedding these tags into addresses that point to these memory regions, and
+ * checking that the memory and the pointer tags match on memory accesses)
+ * redefine this macro to strip tags from pointers.
+ *
+ * Passing down mm_struct allows to define untagging rules on per-process
+ * basis.
+ *
+ * It's defined as noop for architectures that don't support memory tagging.
+ */
+#ifndef untagged_addr
+#define untagged_addr(addr) (addr)
+#endif
+
+#ifndef untagged_addr_remote
+#define untagged_addr_remote(mm, addr) ({              \
+       mmap_assert_locked(mm);                         \
+       untagged_addr(addr);                            \
+})
+#endif
+
 /*
  * Architectures should provide two primitives (raw_copy_{to,from}_user())
  * and get rid of their private instances of copy_{to,from}_user() and

diff --git a/mm/gup.c b/mm/gup.c
index eab18ba045dbefccd212759088d8d90f5415d02e..5ee8b682a0fe99d6c43719df968259cf847971f6 100644 (file)
--- a/mm/gup.c
+++ b/mm/gup.c
@@ -1085,7 +1085,7 @@ static long __get_user_pages(struct mm_struct *mm,
        if (!nr_pages)
                return 0;
 
-       start = untagged_addr(start);
+       start = untagged_addr_remote(mm, start);
 
        VM_BUG_ON(!!pages != !!(gup_flags & (FOLL_GET | FOLL_PIN)));
 
@@ -1259,7 +1259,7 @@ int fixup_user_fault(struct mm_struct *mm,
        struct vm_area_struct *vma;
        vm_fault_t ret;
 
-       address = untagged_addr(address);
+       address = untagged_addr_remote(mm, address);
 
        if (unlocked)
                fault_flags |= FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;

diff --git a/mm/madvise.c b/mm/madvise.c
index 340125d08c03d9638cc5a47a19f6179039ef2b6c..d4b67f36f70f1c3172ccaed9a5cc4b075b1c650c 100644 (file)
--- a/mm/madvise.c
+++ b/mm/madvise.c
@@ -1402,8 +1402,6 @@ int do_madvise(struct mm_struct *mm, unsigned long start, size_t len_in, int beh
        size_t len;
        struct blk_plug plug;
 
-       start = untagged_addr(start);
-
        if (!madvise_behavior_valid(behavior))
                return -EINVAL;
 
@@ -1435,6 +1433,9 @@ int do_madvise(struct mm_struct *mm, unsigned long start, size_t len_in, int beh
                mmap_read_lock(mm);
        }
 
+       start = untagged_addr_remote(mm, start);
+       end = start + len;
+
        blk_start_plug(&plug);
        error = madvise_walk_vmas(mm, start, end, behavior,
                        madvise_vma_behavior);

diff --git a/mm/migrate.c b/mm/migrate.c
index 98f1c11197a8c5f057ed23f5aea9cdc62536bed4..8cd11bc9208f820a6414260da28b461eed015800 100644 (file)
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -2097,15 +2097,18 @@ static int do_move_pages_to_node(struct mm_struct *mm,
  *         target node
  *     1 - when it has been queued
  */
-static int add_page_for_migration(struct mm_struct *mm, unsigned long addr,
+static int add_page_for_migration(struct mm_struct *mm, const void __user *p,
                int node, struct list_head *pagelist, bool migrate_all)
 {
        struct vm_area_struct *vma;
+       unsigned long addr;
        struct page *page;
        int err;
        bool isolated;
 
        mmap_read_lock(mm);
+       addr = (unsigned long)untagged_addr_remote(mm, p);
+
        err = -EFAULT;
        vma = vma_lookup(mm, addr);
        if (!vma || !vma_migratable(vma))
@@ -2211,7 +2214,6 @@ static int do_pages_move(struct mm_struct *mm, nodemask_t task_nodes,
 
        for (i = start = 0; i < nr_pages; i++) {
                const void __user *p;
-               unsigned long addr;
                int node;
 
                err = -EFAULT;
@@ -2219,7 +2221,6 @@ static int do_pages_move(struct mm_struct *mm, nodemask_t task_nodes,
                        goto out_flush;
                if (get_user(node, nodes + i))
                        goto out_flush;
-               addr = (unsigned long)untagged_addr(p);
 
                err = -ENODEV;
                if (node < 0 || node >= MAX_NUMNODES)
@@ -2247,8 +2248,8 @@ static int do_pages_move(struct mm_struct *mm, nodemask_t task_nodes,
                 * Errors in the page lookup or isolation are not fatal and we simply
                 * report them via status
                 */
-               err = add_page_for_migration(mm, addr, current_node,
-                               &pagelist, flags & MPOL_MF_MOVE_ALL);
+               err = add_page_for_migration(mm, p, current_node, &pagelist,
+                                            flags & MPOL_MF_MOVE_ALL);
 
                if (err > 0) {
                        /* The page is successfully queued for migration */