1 /* SPDX-License-Identifier: GPL-2.0-or-later */
3 * Copyright 2013 Red Hat Inc.
5 * Authors: Jérôme Glisse <jglisse@redhat.com>
8 * Heterogeneous Memory Management (HMM)
10 * See Documentation/vm/hmm.rst for reasons and overview of what HMM is and it
11 * is for. Here we focus on the HMM API description, with some explanation of
12 * the underlying implementation.
14 * Short description: HMM provides a set of helpers to share a virtual address
15 * space between CPU and a device, so that the device can access any valid
16 * address of the process (while still obeying memory protection). HMM also
17 * provides helpers to migrate process memory to device memory, and back. Each
18 * set of functionality (address space mirroring, and migration to and from
19 * device memory) can be used independently of the other.
22 * HMM address space mirroring API:
24 * Use HMM address space mirroring if you want to mirror a range of the CPU
25 * page tables of a process into a device page table. Here, "mirror" means "keep
26 * synchronized". Prerequisites: the device must provide the ability to write-
27 * protect its page tables (at PAGE_SIZE granularity), and must be able to
28 * recover from the resulting potential page faults.
30 * HMM guarantees that at any point in time, a given virtual address points to
31 * either the same memory in both CPU and device page tables (that is: CPU and
32 * device page tables each point to the same pages), or that one page table (CPU
33 * or device) points to no entry, while the other still points to the old page
34 * for the address. The latter case happens when the CPU page table update
35 * happens first, and then the update is mirrored over to the device page table.
36 * This does not cause any issue, because the CPU page table cannot start
37 * pointing to a new page until the device page table is invalidated.
39 * HMM uses mmu_notifiers to monitor the CPU page tables, and forwards any
40 * updates to each device driver that has registered a mirror. It also provides
41 * some API calls to help with taking a snapshot of the CPU page table, and to
42 * synchronize with any updates that might happen concurrently.
45 * HMM migration to and from device memory:
47 * HMM provides a set of helpers to hotplug device memory as ZONE_DEVICE, with
48 * a new MEMORY_DEVICE_PRIVATE type. This provides a struct page for each page
49 * of the device memory, and allows the device driver to manage its memory
50 * using those struct pages. Having struct pages for device memory makes
51 * migration easier. Because that memory is not addressable by the CPU it must
52 * never be pinned to the device; in other words, any CPU page fault can always
53 * cause the device memory to be migrated (copied/moved) back to regular memory.
55 * A new migrate helper (migrate_vma()) has been added (see mm/migrate.c) that
56 * allows use of a device DMA engine to perform the copy operation between
57 * regular system memory and device memory.
62 #include <linux/kconfig.h>
63 #include <asm/pgtable.h>
65 #include <linux/device.h>
66 #include <linux/migrate.h>
67 #include <linux/memremap.h>
68 #include <linux/completion.h>
69 #include <linux/mmu_notifier.h>
72 * hmm_pfn_flag_e - HMM flag enums
75 * HMM_PFN_VALID: pfn is valid. It has, at least, read permission.
76 * HMM_PFN_WRITE: CPU page table has write permission set
78 * The driver provides a flags array for mapping page protections to device
79 * PTE bits. If the driver valid bit for an entry is bit 3,
80 * i.e., (entry & (1 << 3)), then the driver must provide
81 * an array in hmm_range.flags with hmm_range.flags[HMM_PFN_VALID] == 1 << 3.
82 * Same logic apply to all flags. This is the same idea as vm_page_prot in vma
83 * except that this is per device driver rather than per architecture.
92 * hmm_pfn_value_e - HMM pfn special value
95 * HMM_PFN_ERROR: corresponding CPU page table entry points to poisoned memory
96 * HMM_PFN_NONE: corresponding CPU page table entry is pte_none()
97 * HMM_PFN_SPECIAL: corresponding CPU page table entry is special; i.e., the
98 * result of vmf_insert_pfn() or vm_insert_page(). Therefore, it should not
99 * be mirrored by a device, because the entry will never have HMM_PFN_VALID
100 * set and the pfn value is undefined.
102 * Driver provides values for none entry, error entry, and special entry.
103 * Driver can alias (i.e., use same value) error and special, but
104 * it should not alias none with error or special.
106 * HMM pfn value returned by hmm_vma_get_pfns() or hmm_vma_fault() will be:
107 * hmm_range.values[HMM_PFN_ERROR] if CPU page table entry is poisonous,
108 * hmm_range.values[HMM_PFN_NONE] if there is no CPU page table entry,
109 * hmm_range.values[HMM_PFN_SPECIAL] if CPU page table entry is a special one
111 enum hmm_pfn_value_e {
119 * struct hmm_range - track invalidation lock on virtual address range
121 * @notifier: a mmu_interval_notifier that includes the start/end
122 * @notifier_seq: result of mmu_interval_read_begin()
123 * @hmm: the core HMM structure this range is active against
124 * @vma: the vm area struct for the range
125 * @list: all range lock are on a list
126 * @start: range virtual start address (inclusive)
127 * @end: range virtual end address (exclusive)
128 * @pfns: array of pfns (big enough for the range)
129 * @flags: pfn flags to match device driver page table
130 * @values: pfn value for some special case (none, special, error, ...)
131 * @default_flags: default flags for the range (write, read, ... see hmm doc)
132 * @pfn_flags_mask: allows to mask pfn flags so that only default_flags matter
133 * @pfn_shifts: pfn shift value (should be <= PAGE_SHIFT)
134 * @valid: pfns array did not change since it has been fill by an HMM function
137 struct mmu_interval_notifier *notifier;
138 unsigned long notifier_seq;
142 const uint64_t *flags;
143 const uint64_t *values;
144 uint64_t default_flags;
145 uint64_t pfn_flags_mask;
150 * hmm_device_entry_to_page() - return struct page pointed to by a device entry
151 * @range: range use to decode device entry value
152 * @entry: device entry value to get corresponding struct page from
153 * Return: struct page pointer if entry is a valid, NULL otherwise
155 * If the device entry is valid (ie valid flag set) then return the struct page
156 * matching the entry value. Otherwise return NULL.
158 static inline struct page *hmm_device_entry_to_page(const struct hmm_range *range,
161 if (entry == range->values[HMM_PFN_NONE])
163 if (entry == range->values[HMM_PFN_ERROR])
165 if (entry == range->values[HMM_PFN_SPECIAL])
167 if (!(entry & range->flags[HMM_PFN_VALID]))
169 return pfn_to_page(entry >> range->pfn_shift);
173 * hmm_device_entry_to_pfn() - return pfn value store in a device entry
174 * @range: range use to decode device entry value
175 * @entry: device entry to extract pfn from
176 * Return: pfn value if device entry is valid, -1UL otherwise
178 static inline unsigned long
179 hmm_device_entry_to_pfn(const struct hmm_range *range, uint64_t pfn)
181 if (pfn == range->values[HMM_PFN_NONE])
183 if (pfn == range->values[HMM_PFN_ERROR])
185 if (pfn == range->values[HMM_PFN_SPECIAL])
187 if (!(pfn & range->flags[HMM_PFN_VALID]))
189 return (pfn >> range->pfn_shift);
193 * hmm_device_entry_from_page() - create a valid device entry for a page
194 * @range: range use to encode HMM pfn value
195 * @page: page for which to create the device entry
196 * Return: valid device entry for the page
198 static inline uint64_t hmm_device_entry_from_page(const struct hmm_range *range,
201 return (page_to_pfn(page) << range->pfn_shift) |
202 range->flags[HMM_PFN_VALID];
206 * hmm_device_entry_from_pfn() - create a valid device entry value from pfn
207 * @range: range use to encode HMM pfn value
208 * @pfn: pfn value for which to create the device entry
209 * Return: valid device entry for the pfn
211 static inline uint64_t hmm_device_entry_from_pfn(const struct hmm_range *range,
214 return (pfn << range->pfn_shift) |
215 range->flags[HMM_PFN_VALID];
218 /* Don't fault in missing PTEs, just snapshot the current state. */
219 #define HMM_FAULT_SNAPSHOT (1 << 1)
222 * Please see Documentation/vm/hmm.rst for how to use the range API.
224 long hmm_range_fault(struct hmm_range *range, unsigned int flags);
227 * HMM_RANGE_DEFAULT_TIMEOUT - default timeout (ms) when waiting for a range
229 * When waiting for mmu notifiers we need some kind of time out otherwise we
230 * could potentialy wait for ever, 1000ms ie 1s sounds like a long time to
233 #define HMM_RANGE_DEFAULT_TIMEOUT 1000
235 #endif /* LINUX_HMM_H */