Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
a2c16d6c HD |
2 | #ifndef _LINUX_SWAPOPS_H |
3 | #define _LINUX_SWAPOPS_H | |
4 | ||
5 | #include <linux/radix-tree.h> | |
187f1882 | 6 | #include <linux/bug.h> |
2b740303 | 7 | #include <linux/mm_types.h> |
a2c16d6c | 8 | |
9b98fa22 CH |
9 | #ifdef CONFIG_MMU |
10 | ||
2e346877 PX |
11 | #ifdef CONFIG_SWAP |
12 | #include <linux/swapfile.h> | |
13 | #endif /* CONFIG_SWAP */ | |
14 | ||
1da177e4 LT |
15 | /* |
16 | * swapcache pages are stored in the swapper_space radix tree. We want to | |
17 | * get good packing density in that tree, so the index should be dense in | |
18 | * the low-order bits. | |
19 | * | |
a930c210 | 20 | * We arrange the `type' and `offset' fields so that `type' is at the six |
e83a9596 | 21 | * high-order bits of the swp_entry_t and `offset' is right-aligned in the |
9b15b817 | 22 | * remaining bits. Although `type' itself needs only five bits, we allow for |
a930c210 | 23 | * shmem/tmpfs to shift it all up a further one bit: see swp_to_radix_entry(). |
1da177e4 LT |
24 | * |
25 | * swp_entry_t's are *never* stored anywhere in their arch-dependent format. | |
26 | */ | |
3159f943 MW |
27 | #define SWP_TYPE_SHIFT (BITS_PER_XA_VALUE - MAX_SWAPFILES_SHIFT) |
28 | #define SWP_OFFSET_MASK ((1UL << SWP_TYPE_SHIFT) - 1) | |
1da177e4 | 29 | |
0d206b5d PX |
30 | /* |
31 | * Definitions only for PFN swap entries (see is_pfn_swap_entry()). To | |
32 | * store PFN, we only need SWP_PFN_BITS bits. Each of the pfn swap entries | |
33 | * can use the extra bits to store other information besides PFN. | |
34 | */ | |
35 | #ifdef MAX_PHYSMEM_BITS | |
630dc25e | 36 | #define SWP_PFN_BITS (MAX_PHYSMEM_BITS - PAGE_SHIFT) |
0d206b5d | 37 | #else /* MAX_PHYSMEM_BITS */ |
630dc25e DH |
38 | #define SWP_PFN_BITS min_t(int, \ |
39 | sizeof(phys_addr_t) * 8 - PAGE_SHIFT, \ | |
40 | SWP_TYPE_SHIFT) | |
0d206b5d | 41 | #endif /* MAX_PHYSMEM_BITS */ |
630dc25e | 42 | #define SWP_PFN_MASK (BIT(SWP_PFN_BITS) - 1) |
0d206b5d | 43 | |
2e346877 PX |
44 | /** |
45 | * Migration swap entry specific bitfield definitions. Layout: | |
46 | * | |
47 | * |----------+--------------------| | |
48 | * | swp_type | swp_offset | | |
49 | * |----------+--------+-+-+-------| | |
50 | * | | resv |D|A| PFN | | |
51 | * |----------+--------+-+-+-------| | |
52 | * | |
53 | * @SWP_MIG_YOUNG_BIT: Whether the page used to have young bit set (bit A) | |
54 | * @SWP_MIG_DIRTY_BIT: Whether the page used to have dirty bit set (bit D) | |
55 | * | |
56 | * Note: A/D bits will be stored in migration entries iff there're enough | |
57 | * free bits in arch specific swp offset. By default we'll ignore A/D bits | |
58 | * when migrating a page. Please refer to migration_entry_supports_ad() | |
59 | * for more information. If there're more bits besides PFN and A/D bits, | |
60 | * they should be reserved and always be zeros. | |
61 | */ | |
62 | #define SWP_MIG_YOUNG_BIT (SWP_PFN_BITS) | |
63 | #define SWP_MIG_DIRTY_BIT (SWP_PFN_BITS + 1) | |
64 | #define SWP_MIG_TOTAL_BITS (SWP_PFN_BITS + 2) | |
65 | ||
66 | #define SWP_MIG_YOUNG BIT(SWP_MIG_YOUNG_BIT) | |
67 | #define SWP_MIG_DIRTY BIT(SWP_MIG_DIRTY_BIT) | |
68 | ||
0d206b5d PX |
69 | static inline bool is_pfn_swap_entry(swp_entry_t entry); |
70 | ||
099dd687 PX |
71 | /* Clear all flags but only keep swp_entry_t related information */ |
72 | static inline pte_t pte_swp_clear_flags(pte_t pte) | |
73 | { | |
1493a191 DH |
74 | if (pte_swp_exclusive(pte)) |
75 | pte = pte_swp_clear_exclusive(pte); | |
099dd687 PX |
76 | if (pte_swp_soft_dirty(pte)) |
77 | pte = pte_swp_clear_soft_dirty(pte); | |
78 | if (pte_swp_uffd_wp(pte)) | |
79 | pte = pte_swp_clear_uffd_wp(pte); | |
80 | return pte; | |
81 | } | |
82 | ||
1da177e4 LT |
83 | /* |
84 | * Store a type+offset into a swp_entry_t in an arch-independent format | |
85 | */ | |
86 | static inline swp_entry_t swp_entry(unsigned long type, pgoff_t offset) | |
87 | { | |
88 | swp_entry_t ret; | |
89 | ||
3159f943 | 90 | ret.val = (type << SWP_TYPE_SHIFT) | (offset & SWP_OFFSET_MASK); |
1da177e4 LT |
91 | return ret; |
92 | } | |
93 | ||
94 | /* | |
95 | * Extract the `type' field from a swp_entry_t. The swp_entry_t is in | |
96 | * arch-independent format | |
97 | */ | |
98 | static inline unsigned swp_type(swp_entry_t entry) | |
99 | { | |
3159f943 | 100 | return (entry.val >> SWP_TYPE_SHIFT); |
1da177e4 LT |
101 | } |
102 | ||
103 | /* | |
104 | * Extract the `offset' field from a swp_entry_t. The swp_entry_t is in | |
105 | * arch-independent format | |
106 | */ | |
107 | static inline pgoff_t swp_offset(swp_entry_t entry) | |
108 | { | |
3159f943 | 109 | return entry.val & SWP_OFFSET_MASK; |
1da177e4 LT |
110 | } |
111 | ||
0d206b5d PX |
112 | /* |
113 | * This should only be called upon a pfn swap entry to get the PFN stored | |
114 | * in the swap entry. Please refers to is_pfn_swap_entry() for definition | |
115 | * of pfn swap entry. | |
116 | */ | |
117 | static inline unsigned long swp_offset_pfn(swp_entry_t entry) | |
118 | { | |
119 | VM_BUG_ON(!is_pfn_swap_entry(entry)); | |
120 | return swp_offset(entry) & SWP_PFN_MASK; | |
121 | } | |
122 | ||
698dd4ba MM |
123 | /* check whether a pte points to a swap entry */ |
124 | static inline int is_swap_pte(pte_t pte) | |
125 | { | |
21d9ee3e | 126 | return !pte_none(pte) && !pte_present(pte); |
698dd4ba MM |
127 | } |
128 | ||
1da177e4 LT |
129 | /* |
130 | * Convert the arch-dependent pte representation of a swp_entry_t into an | |
131 | * arch-independent swp_entry_t. | |
132 | */ | |
133 | static inline swp_entry_t pte_to_swp_entry(pte_t pte) | |
134 | { | |
135 | swp_entry_t arch_entry; | |
136 | ||
099dd687 | 137 | pte = pte_swp_clear_flags(pte); |
1da177e4 LT |
138 | arch_entry = __pte_to_swp_entry(pte); |
139 | return swp_entry(__swp_type(arch_entry), __swp_offset(arch_entry)); | |
140 | } | |
141 | ||
142 | /* | |
143 | * Convert the arch-independent representation of a swp_entry_t into the | |
144 | * arch-dependent pte representation. | |
145 | */ | |
146 | static inline pte_t swp_entry_to_pte(swp_entry_t entry) | |
147 | { | |
148 | swp_entry_t arch_entry; | |
149 | ||
150 | arch_entry = __swp_entry(swp_type(entry), swp_offset(entry)); | |
1da177e4 LT |
151 | return __swp_entry_to_pte(arch_entry); |
152 | } | |
0697212a | 153 | |
a2c16d6c HD |
154 | static inline swp_entry_t radix_to_swp_entry(void *arg) |
155 | { | |
156 | swp_entry_t entry; | |
157 | ||
3159f943 | 158 | entry.val = xa_to_value(arg); |
a2c16d6c HD |
159 | return entry; |
160 | } | |
161 | ||
162 | static inline void *swp_to_radix_entry(swp_entry_t entry) | |
163 | { | |
3159f943 | 164 | return xa_mk_value(entry.val); |
a2c16d6c HD |
165 | } |
166 | ||
5042db43 | 167 | #if IS_ENABLED(CONFIG_DEVICE_PRIVATE) |
4dd845b5 | 168 | static inline swp_entry_t make_readable_device_private_entry(pgoff_t offset) |
5042db43 | 169 | { |
4dd845b5 | 170 | return swp_entry(SWP_DEVICE_READ, offset); |
5042db43 JG |
171 | } |
172 | ||
4dd845b5 | 173 | static inline swp_entry_t make_writable_device_private_entry(pgoff_t offset) |
5042db43 | 174 | { |
4dd845b5 | 175 | return swp_entry(SWP_DEVICE_WRITE, offset); |
5042db43 JG |
176 | } |
177 | ||
4dd845b5 | 178 | static inline bool is_device_private_entry(swp_entry_t entry) |
5042db43 | 179 | { |
4dd845b5 AP |
180 | int type = swp_type(entry); |
181 | return type == SWP_DEVICE_READ || type == SWP_DEVICE_WRITE; | |
5042db43 JG |
182 | } |
183 | ||
4dd845b5 | 184 | static inline bool is_writable_device_private_entry(swp_entry_t entry) |
5042db43 JG |
185 | { |
186 | return unlikely(swp_type(entry) == SWP_DEVICE_WRITE); | |
187 | } | |
b756a3b5 AP |
188 | |
189 | static inline swp_entry_t make_readable_device_exclusive_entry(pgoff_t offset) | |
190 | { | |
191 | return swp_entry(SWP_DEVICE_EXCLUSIVE_READ, offset); | |
192 | } | |
193 | ||
194 | static inline swp_entry_t make_writable_device_exclusive_entry(pgoff_t offset) | |
195 | { | |
196 | return swp_entry(SWP_DEVICE_EXCLUSIVE_WRITE, offset); | |
197 | } | |
198 | ||
199 | static inline bool is_device_exclusive_entry(swp_entry_t entry) | |
200 | { | |
201 | return swp_type(entry) == SWP_DEVICE_EXCLUSIVE_READ || | |
202 | swp_type(entry) == SWP_DEVICE_EXCLUSIVE_WRITE; | |
203 | } | |
204 | ||
205 | static inline bool is_writable_device_exclusive_entry(swp_entry_t entry) | |
206 | { | |
207 | return unlikely(swp_type(entry) == SWP_DEVICE_EXCLUSIVE_WRITE); | |
208 | } | |
5042db43 | 209 | #else /* CONFIG_DEVICE_PRIVATE */ |
4dd845b5 | 210 | static inline swp_entry_t make_readable_device_private_entry(pgoff_t offset) |
5042db43 JG |
211 | { |
212 | return swp_entry(0, 0); | |
213 | } | |
214 | ||
4dd845b5 | 215 | static inline swp_entry_t make_writable_device_private_entry(pgoff_t offset) |
5042db43 | 216 | { |
4dd845b5 | 217 | return swp_entry(0, 0); |
5042db43 JG |
218 | } |
219 | ||
220 | static inline bool is_device_private_entry(swp_entry_t entry) | |
221 | { | |
222 | return false; | |
223 | } | |
224 | ||
4dd845b5 | 225 | static inline bool is_writable_device_private_entry(swp_entry_t entry) |
5042db43 JG |
226 | { |
227 | return false; | |
228 | } | |
b756a3b5 AP |
229 | |
230 | static inline swp_entry_t make_readable_device_exclusive_entry(pgoff_t offset) | |
231 | { | |
232 | return swp_entry(0, 0); | |
233 | } | |
234 | ||
235 | static inline swp_entry_t make_writable_device_exclusive_entry(pgoff_t offset) | |
236 | { | |
237 | return swp_entry(0, 0); | |
238 | } | |
239 | ||
240 | static inline bool is_device_exclusive_entry(swp_entry_t entry) | |
241 | { | |
242 | return false; | |
243 | } | |
244 | ||
245 | static inline bool is_writable_device_exclusive_entry(swp_entry_t entry) | |
246 | { | |
247 | return false; | |
248 | } | |
5042db43 JG |
249 | #endif /* CONFIG_DEVICE_PRIVATE */ |
250 | ||
0697212a | 251 | #ifdef CONFIG_MIGRATION |
0697212a CL |
252 | static inline int is_migration_entry(swp_entry_t entry) |
253 | { | |
254 | return unlikely(swp_type(entry) == SWP_MIGRATION_READ || | |
6c287605 | 255 | swp_type(entry) == SWP_MIGRATION_READ_EXCLUSIVE || |
0697212a CL |
256 | swp_type(entry) == SWP_MIGRATION_WRITE); |
257 | } | |
258 | ||
4dd845b5 | 259 | static inline int is_writable_migration_entry(swp_entry_t entry) |
0697212a CL |
260 | { |
261 | return unlikely(swp_type(entry) == SWP_MIGRATION_WRITE); | |
262 | } | |
263 | ||
6c287605 DH |
264 | static inline int is_readable_migration_entry(swp_entry_t entry) |
265 | { | |
266 | return unlikely(swp_type(entry) == SWP_MIGRATION_READ); | |
267 | } | |
268 | ||
269 | static inline int is_readable_exclusive_migration_entry(swp_entry_t entry) | |
270 | { | |
271 | return unlikely(swp_type(entry) == SWP_MIGRATION_READ_EXCLUSIVE); | |
272 | } | |
273 | ||
4dd845b5 | 274 | static inline swp_entry_t make_readable_migration_entry(pgoff_t offset) |
0697212a | 275 | { |
4dd845b5 AP |
276 | return swp_entry(SWP_MIGRATION_READ, offset); |
277 | } | |
278 | ||
6c287605 DH |
279 | static inline swp_entry_t make_readable_exclusive_migration_entry(pgoff_t offset) |
280 | { | |
281 | return swp_entry(SWP_MIGRATION_READ_EXCLUSIVE, offset); | |
282 | } | |
283 | ||
4dd845b5 AP |
284 | static inline swp_entry_t make_writable_migration_entry(pgoff_t offset) |
285 | { | |
286 | return swp_entry(SWP_MIGRATION_WRITE, offset); | |
0697212a CL |
287 | } |
288 | ||
2e346877 PX |
289 | /* |
290 | * Returns whether the host has large enough swap offset field to support | |
291 | * carrying over pgtable A/D bits for page migrations. The result is | |
292 | * pretty much arch specific. | |
293 | */ | |
294 | static inline bool migration_entry_supports_ad(void) | |
295 | { | |
2e346877 | 296 | #ifdef CONFIG_SWAP |
5154e607 | 297 | return swap_migration_ad_supported; |
2e346877 PX |
298 | #else /* CONFIG_SWAP */ |
299 | return false; | |
300 | #endif /* CONFIG_SWAP */ | |
301 | } | |
302 | ||
303 | static inline swp_entry_t make_migration_entry_young(swp_entry_t entry) | |
304 | { | |
305 | if (migration_entry_supports_ad()) | |
306 | return swp_entry(swp_type(entry), | |
307 | swp_offset(entry) | SWP_MIG_YOUNG); | |
308 | return entry; | |
309 | } | |
310 | ||
311 | static inline bool is_migration_entry_young(swp_entry_t entry) | |
312 | { | |
313 | if (migration_entry_supports_ad()) | |
314 | return swp_offset(entry) & SWP_MIG_YOUNG; | |
315 | /* Keep the old behavior of aging page after migration */ | |
316 | return false; | |
317 | } | |
318 | ||
319 | static inline swp_entry_t make_migration_entry_dirty(swp_entry_t entry) | |
320 | { | |
321 | if (migration_entry_supports_ad()) | |
322 | return swp_entry(swp_type(entry), | |
323 | swp_offset(entry) | SWP_MIG_DIRTY); | |
324 | return entry; | |
325 | } | |
326 | ||
327 | static inline bool is_migration_entry_dirty(swp_entry_t entry) | |
328 | { | |
329 | if (migration_entry_supports_ad()) | |
330 | return swp_offset(entry) & SWP_MIG_DIRTY; | |
331 | /* Keep the old behavior of clean page after migration */ | |
332 | return false; | |
333 | } | |
334 | ||
e66f17ff NH |
335 | extern void __migration_entry_wait(struct mm_struct *mm, pte_t *ptep, |
336 | spinlock_t *ptl); | |
0697212a CL |
337 | extern void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd, |
338 | unsigned long address); | |
ad1ac596 | 339 | #ifdef CONFIG_HUGETLB_PAGE |
fcd48540 PX |
340 | extern void __migration_entry_wait_huge(struct vm_area_struct *vma, |
341 | pte_t *ptep, spinlock_t *ptl); | |
ad1ac596 | 342 | extern void migration_entry_wait_huge(struct vm_area_struct *vma, pte_t *pte); |
eba4d770 PX |
343 | #endif /* CONFIG_HUGETLB_PAGE */ |
344 | #else /* CONFIG_MIGRATION */ | |
4dd845b5 AP |
345 | static inline swp_entry_t make_readable_migration_entry(pgoff_t offset) |
346 | { | |
347 | return swp_entry(0, 0); | |
348 | } | |
349 | ||
6c287605 DH |
350 | static inline swp_entry_t make_readable_exclusive_migration_entry(pgoff_t offset) |
351 | { | |
352 | return swp_entry(0, 0); | |
353 | } | |
354 | ||
4dd845b5 AP |
355 | static inline swp_entry_t make_writable_migration_entry(pgoff_t offset) |
356 | { | |
357 | return swp_entry(0, 0); | |
358 | } | |
0697212a | 359 | |
5ec553a9 AM |
360 | static inline int is_migration_entry(swp_entry_t swp) |
361 | { | |
362 | return 0; | |
363 | } | |
0d665e7b | 364 | |
e66f17ff NH |
365 | static inline void __migration_entry_wait(struct mm_struct *mm, pte_t *ptep, |
366 | spinlock_t *ptl) { } | |
0697212a CL |
367 | static inline void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd, |
368 | unsigned long address) { } | |
ad1ac596 | 369 | #ifdef CONFIG_HUGETLB_PAGE |
fcd48540 PX |
370 | static inline void __migration_entry_wait_huge(struct vm_area_struct *vma, |
371 | pte_t *ptep, spinlock_t *ptl) { } | |
ad1ac596 | 372 | static inline void migration_entry_wait_huge(struct vm_area_struct *vma, pte_t *pte) { } |
eba4d770 | 373 | #endif /* CONFIG_HUGETLB_PAGE */ |
4dd845b5 | 374 | static inline int is_writable_migration_entry(swp_entry_t entry) |
0697212a CL |
375 | { |
376 | return 0; | |
377 | } | |
6c287605 DH |
378 | static inline int is_readable_migration_entry(swp_entry_t entry) |
379 | { | |
380 | return 0; | |
381 | } | |
0697212a | 382 | |
2e346877 PX |
383 | static inline swp_entry_t make_migration_entry_young(swp_entry_t entry) |
384 | { | |
385 | return entry; | |
386 | } | |
387 | ||
388 | static inline bool is_migration_entry_young(swp_entry_t entry) | |
389 | { | |
390 | return false; | |
391 | } | |
392 | ||
393 | static inline swp_entry_t make_migration_entry_dirty(swp_entry_t entry) | |
394 | { | |
395 | return entry; | |
396 | } | |
397 | ||
398 | static inline bool is_migration_entry_dirty(swp_entry_t entry) | |
399 | { | |
400 | return false; | |
401 | } | |
eba4d770 | 402 | #endif /* CONFIG_MIGRATION */ |
0697212a | 403 | |
679d1033 PX |
404 | typedef unsigned long pte_marker; |
405 | ||
15520a3f PX |
406 | #define PTE_MARKER_UFFD_WP BIT(0) |
407 | #define PTE_MARKER_SWAPIN_ERROR BIT(1) | |
408 | #define PTE_MARKER_MASK (BIT(2) - 1) | |
679d1033 | 409 | |
679d1033 PX |
410 | static inline swp_entry_t make_pte_marker_entry(pte_marker marker) |
411 | { | |
412 | return swp_entry(SWP_PTE_MARKER, marker); | |
413 | } | |
414 | ||
415 | static inline bool is_pte_marker_entry(swp_entry_t entry) | |
416 | { | |
417 | return swp_type(entry) == SWP_PTE_MARKER; | |
418 | } | |
419 | ||
420 | static inline pte_marker pte_marker_get(swp_entry_t entry) | |
421 | { | |
422 | return swp_offset(entry) & PTE_MARKER_MASK; | |
423 | } | |
424 | ||
425 | static inline bool is_pte_marker(pte_t pte) | |
426 | { | |
427 | return is_swap_pte(pte) && is_pte_marker_entry(pte_to_swp_entry(pte)); | |
428 | } | |
429 | ||
679d1033 PX |
430 | static inline pte_t make_pte_marker(pte_marker marker) |
431 | { | |
432 | return swp_entry_to_pte(make_pte_marker_entry(marker)); | |
433 | } | |
434 | ||
15520a3f PX |
435 | static inline swp_entry_t make_swapin_error_entry(void) |
436 | { | |
437 | return make_pte_marker_entry(PTE_MARKER_SWAPIN_ERROR); | |
438 | } | |
439 | ||
440 | static inline int is_swapin_error_entry(swp_entry_t entry) | |
441 | { | |
442 | return is_pte_marker_entry(entry) && | |
443 | (pte_marker_get(entry) & PTE_MARKER_SWAPIN_ERROR); | |
444 | } | |
445 | ||
679d1033 PX |
446 | /* |
447 | * This is a special version to check pte_none() just to cover the case when | |
448 | * the pte is a pte marker. It existed because in many cases the pte marker | |
449 | * should be seen as a none pte; it's just that we have stored some information | |
450 | * onto the none pte so it becomes not-none any more. | |
451 | * | |
452 | * It should be used when the pte is file-backed, ram-based and backing | |
453 | * userspace pages, like shmem. It is not needed upon pgtables that do not | |
454 | * support pte markers at all. For example, it's not needed on anonymous | |
455 | * memory, kernel-only memory (including when the system is during-boot), | |
456 | * non-ram based generic file-system. It's fine to be used even there, but the | |
457 | * extra pte marker check will be pure overhead. | |
679d1033 PX |
458 | */ |
459 | static inline int pte_none_mostly(pte_t pte) | |
460 | { | |
461 | return pte_none(pte) || is_pte_marker(pte); | |
462 | } | |
463 | ||
af5cdaf8 AP |
464 | static inline struct page *pfn_swap_entry_to_page(swp_entry_t entry) |
465 | { | |
0d206b5d | 466 | struct page *p = pfn_to_page(swp_offset_pfn(entry)); |
af5cdaf8 AP |
467 | |
468 | /* | |
469 | * Any use of migration entries may only occur while the | |
470 | * corresponding page is locked | |
471 | */ | |
472 | BUG_ON(is_migration_entry(entry) && !PageLocked(p)); | |
473 | ||
474 | return p; | |
475 | } | |
476 | ||
477 | /* | |
478 | * A pfn swap entry is a special type of swap entry that always has a pfn stored | |
479 | * in the swap offset. They are used to represent unaddressable device memory | |
480 | * and to restrict access to a page undergoing migration. | |
481 | */ | |
482 | static inline bool is_pfn_swap_entry(swp_entry_t entry) | |
483 | { | |
0d206b5d PX |
484 | /* Make sure the swp offset can always store the needed fields */ |
485 | BUILD_BUG_ON(SWP_TYPE_SHIFT < SWP_PFN_BITS); | |
486 | ||
b756a3b5 AP |
487 | return is_migration_entry(entry) || is_device_private_entry(entry) || |
488 | is_device_exclusive_entry(entry); | |
af5cdaf8 AP |
489 | } |
490 | ||
616b8371 ZY |
491 | struct page_vma_mapped_walk; |
492 | ||
493 | #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION | |
7f5abe60 | 494 | extern int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, |
616b8371 ZY |
495 | struct page *page); |
496 | ||
497 | extern void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, | |
498 | struct page *new); | |
499 | ||
500 | extern void pmd_migration_entry_wait(struct mm_struct *mm, pmd_t *pmd); | |
501 | ||
502 | static inline swp_entry_t pmd_to_swp_entry(pmd_t pmd) | |
503 | { | |
504 | swp_entry_t arch_entry; | |
505 | ||
ab6e3d09 NH |
506 | if (pmd_swp_soft_dirty(pmd)) |
507 | pmd = pmd_swp_clear_soft_dirty(pmd); | |
8f34f1ea PX |
508 | if (pmd_swp_uffd_wp(pmd)) |
509 | pmd = pmd_swp_clear_uffd_wp(pmd); | |
616b8371 ZY |
510 | arch_entry = __pmd_to_swp_entry(pmd); |
511 | return swp_entry(__swp_type(arch_entry), __swp_offset(arch_entry)); | |
512 | } | |
513 | ||
514 | static inline pmd_t swp_entry_to_pmd(swp_entry_t entry) | |
515 | { | |
516 | swp_entry_t arch_entry; | |
517 | ||
518 | arch_entry = __swp_entry(swp_type(entry), swp_offset(entry)); | |
519 | return __swp_entry_to_pmd(arch_entry); | |
520 | } | |
521 | ||
522 | static inline int is_pmd_migration_entry(pmd_t pmd) | |
523 | { | |
b304c6f0 | 524 | return is_swap_pmd(pmd) && is_migration_entry(pmd_to_swp_entry(pmd)); |
616b8371 | 525 | } |
eba4d770 | 526 | #else /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ |
7f5abe60 | 527 | static inline int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, |
616b8371 ZY |
528 | struct page *page) |
529 | { | |
530 | BUILD_BUG(); | |
531 | } | |
532 | ||
533 | static inline void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, | |
534 | struct page *new) | |
535 | { | |
536 | BUILD_BUG(); | |
537 | } | |
538 | ||
539 | static inline void pmd_migration_entry_wait(struct mm_struct *m, pmd_t *p) { } | |
540 | ||
541 | static inline swp_entry_t pmd_to_swp_entry(pmd_t pmd) | |
542 | { | |
543 | return swp_entry(0, 0); | |
544 | } | |
545 | ||
546 | static inline pmd_t swp_entry_to_pmd(swp_entry_t entry) | |
547 | { | |
548 | return __pmd(0); | |
549 | } | |
550 | ||
551 | static inline int is_pmd_migration_entry(pmd_t pmd) | |
552 | { | |
553 | return 0; | |
554 | } | |
eba4d770 | 555 | #endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ |
616b8371 | 556 | |
a7420aa5 | 557 | #ifdef CONFIG_MEMORY_FAILURE |
8e30456b | 558 | |
a7420aa5 AK |
559 | /* |
560 | * Support for hardware poisoned pages | |
561 | */ | |
562 | static inline swp_entry_t make_hwpoison_entry(struct page *page) | |
563 | { | |
564 | BUG_ON(!PageLocked(page)); | |
565 | return swp_entry(SWP_HWPOISON, page_to_pfn(page)); | |
566 | } | |
567 | ||
568 | static inline int is_hwpoison_entry(swp_entry_t entry) | |
569 | { | |
570 | return swp_type(entry) == SWP_HWPOISON; | |
571 | } | |
8e30456b | 572 | |
d027122d | 573 | #else |
a7420aa5 AK |
574 | |
575 | static inline swp_entry_t make_hwpoison_entry(struct page *page) | |
576 | { | |
577 | return swp_entry(0, 0); | |
578 | } | |
579 | ||
580 | static inline int is_hwpoison_entry(swp_entry_t swp) | |
581 | { | |
582 | return 0; | |
583 | } | |
d027122d | 584 | #endif |
a7420aa5 | 585 | |
a7420aa5 AK |
586 | static inline int non_swap_entry(swp_entry_t entry) |
587 | { | |
588 | return swp_type(entry) >= MAX_SWAPFILES; | |
589 | } | |
a2c16d6c | 590 | |
9b98fa22 | 591 | #endif /* CONFIG_MMU */ |
a2c16d6c | 592 | #endif /* _LINUX_SWAPOPS_H */ |