Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
a2c16d6c HD |
2 | #ifndef _LINUX_SWAPOPS_H |
3 | #define _LINUX_SWAPOPS_H | |
4 | ||
5 | #include <linux/radix-tree.h> | |
187f1882 | 6 | #include <linux/bug.h> |
2b740303 | 7 | #include <linux/mm_types.h> |
a2c16d6c | 8 | |
9b98fa22 CH |
9 | #ifdef CONFIG_MMU |
10 | ||
2e346877 PX |
11 | #ifdef CONFIG_SWAP |
12 | #include <linux/swapfile.h> | |
13 | #endif /* CONFIG_SWAP */ | |
14 | ||
1da177e4 LT |
15 | /* |
16 | * swapcache pages are stored in the swapper_space radix tree. We want to | |
17 | * get good packing density in that tree, so the index should be dense in | |
18 | * the low-order bits. | |
19 | * | |
a930c210 | 20 | * We arrange the `type' and `offset' fields so that `type' is at the six |
e83a9596 | 21 | * high-order bits of the swp_entry_t and `offset' is right-aligned in the |
9b15b817 | 22 | * remaining bits. Although `type' itself needs only five bits, we allow for |
a930c210 | 23 | * shmem/tmpfs to shift it all up a further one bit: see swp_to_radix_entry(). |
1da177e4 LT |
24 | * |
25 | * swp_entry_t's are *never* stored anywhere in their arch-dependent format. | |
26 | */ | |
3159f943 MW |
27 | #define SWP_TYPE_SHIFT (BITS_PER_XA_VALUE - MAX_SWAPFILES_SHIFT) |
28 | #define SWP_OFFSET_MASK ((1UL << SWP_TYPE_SHIFT) - 1) | |
1da177e4 | 29 | |
0d206b5d PX |
30 | /* |
31 | * Definitions only for PFN swap entries (see is_pfn_swap_entry()). To | |
32 | * store PFN, we only need SWP_PFN_BITS bits. Each of the pfn swap entries | |
33 | * can use the extra bits to store other information besides PFN. | |
34 | */ | |
35 | #ifdef MAX_PHYSMEM_BITS | |
630dc25e | 36 | #define SWP_PFN_BITS (MAX_PHYSMEM_BITS - PAGE_SHIFT) |
0d206b5d | 37 | #else /* MAX_PHYSMEM_BITS */ |
630dc25e DH |
38 | #define SWP_PFN_BITS min_t(int, \ |
39 | sizeof(phys_addr_t) * 8 - PAGE_SHIFT, \ | |
40 | SWP_TYPE_SHIFT) | |
0d206b5d | 41 | #endif /* MAX_PHYSMEM_BITS */ |
630dc25e | 42 | #define SWP_PFN_MASK (BIT(SWP_PFN_BITS) - 1) |
0d206b5d | 43 | |
2e346877 PX |
44 | /** |
45 | * Migration swap entry specific bitfield definitions. Layout: | |
46 | * | |
47 | * |----------+--------------------| | |
48 | * | swp_type | swp_offset | | |
49 | * |----------+--------+-+-+-------| | |
50 | * | | resv |D|A| PFN | | |
51 | * |----------+--------+-+-+-------| | |
52 | * | |
53 | * @SWP_MIG_YOUNG_BIT: Whether the page used to have young bit set (bit A) | |
54 | * @SWP_MIG_DIRTY_BIT: Whether the page used to have dirty bit set (bit D) | |
55 | * | |
56 | * Note: A/D bits will be stored in migration entries iff there're enough | |
57 | * free bits in arch specific swp offset. By default we'll ignore A/D bits | |
58 | * when migrating a page. Please refer to migration_entry_supports_ad() | |
59 | * for more information. If there're more bits besides PFN and A/D bits, | |
60 | * they should be reserved and always be zeros. | |
61 | */ | |
62 | #define SWP_MIG_YOUNG_BIT (SWP_PFN_BITS) | |
63 | #define SWP_MIG_DIRTY_BIT (SWP_PFN_BITS + 1) | |
64 | #define SWP_MIG_TOTAL_BITS (SWP_PFN_BITS + 2) | |
65 | ||
66 | #define SWP_MIG_YOUNG BIT(SWP_MIG_YOUNG_BIT) | |
67 | #define SWP_MIG_DIRTY BIT(SWP_MIG_DIRTY_BIT) | |
68 | ||
0d206b5d PX |
69 | static inline bool is_pfn_swap_entry(swp_entry_t entry); |
70 | ||
099dd687 PX |
71 | /* Clear all flags but only keep swp_entry_t related information */ |
72 | static inline pte_t pte_swp_clear_flags(pte_t pte) | |
73 | { | |
1493a191 DH |
74 | if (pte_swp_exclusive(pte)) |
75 | pte = pte_swp_clear_exclusive(pte); | |
099dd687 PX |
76 | if (pte_swp_soft_dirty(pte)) |
77 | pte = pte_swp_clear_soft_dirty(pte); | |
78 | if (pte_swp_uffd_wp(pte)) | |
79 | pte = pte_swp_clear_uffd_wp(pte); | |
80 | return pte; | |
81 | } | |
82 | ||
1da177e4 LT |
83 | /* |
84 | * Store a type+offset into a swp_entry_t in an arch-independent format | |
85 | */ | |
86 | static inline swp_entry_t swp_entry(unsigned long type, pgoff_t offset) | |
87 | { | |
88 | swp_entry_t ret; | |
89 | ||
3159f943 | 90 | ret.val = (type << SWP_TYPE_SHIFT) | (offset & SWP_OFFSET_MASK); |
1da177e4 LT |
91 | return ret; |
92 | } | |
93 | ||
94 | /* | |
95 | * Extract the `type' field from a swp_entry_t. The swp_entry_t is in | |
96 | * arch-independent format | |
97 | */ | |
98 | static inline unsigned swp_type(swp_entry_t entry) | |
99 | { | |
3159f943 | 100 | return (entry.val >> SWP_TYPE_SHIFT); |
1da177e4 LT |
101 | } |
102 | ||
103 | /* | |
104 | * Extract the `offset' field from a swp_entry_t. The swp_entry_t is in | |
105 | * arch-independent format | |
106 | */ | |
107 | static inline pgoff_t swp_offset(swp_entry_t entry) | |
108 | { | |
3159f943 | 109 | return entry.val & SWP_OFFSET_MASK; |
1da177e4 LT |
110 | } |
111 | ||
0d206b5d PX |
112 | /* |
113 | * This should only be called upon a pfn swap entry to get the PFN stored | |
114 | * in the swap entry. Please refers to is_pfn_swap_entry() for definition | |
115 | * of pfn swap entry. | |
116 | */ | |
117 | static inline unsigned long swp_offset_pfn(swp_entry_t entry) | |
118 | { | |
119 | VM_BUG_ON(!is_pfn_swap_entry(entry)); | |
120 | return swp_offset(entry) & SWP_PFN_MASK; | |
121 | } | |
122 | ||
698dd4ba MM |
123 | /* check whether a pte points to a swap entry */ |
124 | static inline int is_swap_pte(pte_t pte) | |
125 | { | |
21d9ee3e | 126 | return !pte_none(pte) && !pte_present(pte); |
698dd4ba MM |
127 | } |
128 | ||
1da177e4 LT |
129 | /* |
130 | * Convert the arch-dependent pte representation of a swp_entry_t into an | |
131 | * arch-independent swp_entry_t. | |
132 | */ | |
133 | static inline swp_entry_t pte_to_swp_entry(pte_t pte) | |
134 | { | |
135 | swp_entry_t arch_entry; | |
136 | ||
099dd687 | 137 | pte = pte_swp_clear_flags(pte); |
1da177e4 LT |
138 | arch_entry = __pte_to_swp_entry(pte); |
139 | return swp_entry(__swp_type(arch_entry), __swp_offset(arch_entry)); | |
140 | } | |
141 | ||
142 | /* | |
143 | * Convert the arch-independent representation of a swp_entry_t into the | |
144 | * arch-dependent pte representation. | |
145 | */ | |
146 | static inline pte_t swp_entry_to_pte(swp_entry_t entry) | |
147 | { | |
148 | swp_entry_t arch_entry; | |
149 | ||
150 | arch_entry = __swp_entry(swp_type(entry), swp_offset(entry)); | |
1da177e4 LT |
151 | return __swp_entry_to_pte(arch_entry); |
152 | } | |
0697212a | 153 | |
a2c16d6c HD |
154 | static inline swp_entry_t radix_to_swp_entry(void *arg) |
155 | { | |
156 | swp_entry_t entry; | |
157 | ||
3159f943 | 158 | entry.val = xa_to_value(arg); |
a2c16d6c HD |
159 | return entry; |
160 | } | |
161 | ||
162 | static inline void *swp_to_radix_entry(swp_entry_t entry) | |
163 | { | |
3159f943 | 164 | return xa_mk_value(entry.val); |
a2c16d6c HD |
165 | } |
166 | ||
5042db43 | 167 | #if IS_ENABLED(CONFIG_DEVICE_PRIVATE) |
4dd845b5 | 168 | static inline swp_entry_t make_readable_device_private_entry(pgoff_t offset) |
5042db43 | 169 | { |
4dd845b5 | 170 | return swp_entry(SWP_DEVICE_READ, offset); |
5042db43 JG |
171 | } |
172 | ||
4dd845b5 | 173 | static inline swp_entry_t make_writable_device_private_entry(pgoff_t offset) |
5042db43 | 174 | { |
4dd845b5 | 175 | return swp_entry(SWP_DEVICE_WRITE, offset); |
5042db43 JG |
176 | } |
177 | ||
4dd845b5 | 178 | static inline bool is_device_private_entry(swp_entry_t entry) |
5042db43 | 179 | { |
4dd845b5 AP |
180 | int type = swp_type(entry); |
181 | return type == SWP_DEVICE_READ || type == SWP_DEVICE_WRITE; | |
5042db43 JG |
182 | } |
183 | ||
4dd845b5 | 184 | static inline bool is_writable_device_private_entry(swp_entry_t entry) |
5042db43 JG |
185 | { |
186 | return unlikely(swp_type(entry) == SWP_DEVICE_WRITE); | |
187 | } | |
b756a3b5 AP |
188 | |
189 | static inline swp_entry_t make_readable_device_exclusive_entry(pgoff_t offset) | |
190 | { | |
191 | return swp_entry(SWP_DEVICE_EXCLUSIVE_READ, offset); | |
192 | } | |
193 | ||
194 | static inline swp_entry_t make_writable_device_exclusive_entry(pgoff_t offset) | |
195 | { | |
196 | return swp_entry(SWP_DEVICE_EXCLUSIVE_WRITE, offset); | |
197 | } | |
198 | ||
199 | static inline bool is_device_exclusive_entry(swp_entry_t entry) | |
200 | { | |
201 | return swp_type(entry) == SWP_DEVICE_EXCLUSIVE_READ || | |
202 | swp_type(entry) == SWP_DEVICE_EXCLUSIVE_WRITE; | |
203 | } | |
204 | ||
205 | static inline bool is_writable_device_exclusive_entry(swp_entry_t entry) | |
206 | { | |
207 | return unlikely(swp_type(entry) == SWP_DEVICE_EXCLUSIVE_WRITE); | |
208 | } | |
5042db43 | 209 | #else /* CONFIG_DEVICE_PRIVATE */ |
4dd845b5 | 210 | static inline swp_entry_t make_readable_device_private_entry(pgoff_t offset) |
5042db43 JG |
211 | { |
212 | return swp_entry(0, 0); | |
213 | } | |
214 | ||
4dd845b5 | 215 | static inline swp_entry_t make_writable_device_private_entry(pgoff_t offset) |
5042db43 | 216 | { |
4dd845b5 | 217 | return swp_entry(0, 0); |
5042db43 JG |
218 | } |
219 | ||
220 | static inline bool is_device_private_entry(swp_entry_t entry) | |
221 | { | |
222 | return false; | |
223 | } | |
224 | ||
4dd845b5 | 225 | static inline bool is_writable_device_private_entry(swp_entry_t entry) |
5042db43 JG |
226 | { |
227 | return false; | |
228 | } | |
b756a3b5 AP |
229 | |
230 | static inline swp_entry_t make_readable_device_exclusive_entry(pgoff_t offset) | |
231 | { | |
232 | return swp_entry(0, 0); | |
233 | } | |
234 | ||
235 | static inline swp_entry_t make_writable_device_exclusive_entry(pgoff_t offset) | |
236 | { | |
237 | return swp_entry(0, 0); | |
238 | } | |
239 | ||
240 | static inline bool is_device_exclusive_entry(swp_entry_t entry) | |
241 | { | |
242 | return false; | |
243 | } | |
244 | ||
245 | static inline bool is_writable_device_exclusive_entry(swp_entry_t entry) | |
246 | { | |
247 | return false; | |
248 | } | |
5042db43 JG |
249 | #endif /* CONFIG_DEVICE_PRIVATE */ |
250 | ||
0697212a | 251 | #ifdef CONFIG_MIGRATION |
0697212a CL |
252 | static inline int is_migration_entry(swp_entry_t entry) |
253 | { | |
254 | return unlikely(swp_type(entry) == SWP_MIGRATION_READ || | |
6c287605 | 255 | swp_type(entry) == SWP_MIGRATION_READ_EXCLUSIVE || |
0697212a CL |
256 | swp_type(entry) == SWP_MIGRATION_WRITE); |
257 | } | |
258 | ||
4dd845b5 | 259 | static inline int is_writable_migration_entry(swp_entry_t entry) |
0697212a CL |
260 | { |
261 | return unlikely(swp_type(entry) == SWP_MIGRATION_WRITE); | |
262 | } | |
263 | ||
6c287605 DH |
264 | static inline int is_readable_migration_entry(swp_entry_t entry) |
265 | { | |
266 | return unlikely(swp_type(entry) == SWP_MIGRATION_READ); | |
267 | } | |
268 | ||
269 | static inline int is_readable_exclusive_migration_entry(swp_entry_t entry) | |
270 | { | |
271 | return unlikely(swp_type(entry) == SWP_MIGRATION_READ_EXCLUSIVE); | |
272 | } | |
273 | ||
4dd845b5 | 274 | static inline swp_entry_t make_readable_migration_entry(pgoff_t offset) |
0697212a | 275 | { |
4dd845b5 AP |
276 | return swp_entry(SWP_MIGRATION_READ, offset); |
277 | } | |
278 | ||
6c287605 DH |
279 | static inline swp_entry_t make_readable_exclusive_migration_entry(pgoff_t offset) |
280 | { | |
281 | return swp_entry(SWP_MIGRATION_READ_EXCLUSIVE, offset); | |
282 | } | |
283 | ||
4dd845b5 AP |
284 | static inline swp_entry_t make_writable_migration_entry(pgoff_t offset) |
285 | { | |
286 | return swp_entry(SWP_MIGRATION_WRITE, offset); | |
0697212a CL |
287 | } |
288 | ||
2e346877 PX |
289 | /* |
290 | * Returns whether the host has large enough swap offset field to support | |
291 | * carrying over pgtable A/D bits for page migrations. The result is | |
292 | * pretty much arch specific. | |
293 | */ | |
294 | static inline bool migration_entry_supports_ad(void) | |
295 | { | |
2e346877 | 296 | #ifdef CONFIG_SWAP |
5154e607 | 297 | return swap_migration_ad_supported; |
2e346877 PX |
298 | #else /* CONFIG_SWAP */ |
299 | return false; | |
300 | #endif /* CONFIG_SWAP */ | |
301 | } | |
302 | ||
303 | static inline swp_entry_t make_migration_entry_young(swp_entry_t entry) | |
304 | { | |
305 | if (migration_entry_supports_ad()) | |
306 | return swp_entry(swp_type(entry), | |
307 | swp_offset(entry) | SWP_MIG_YOUNG); | |
308 | return entry; | |
309 | } | |
310 | ||
311 | static inline bool is_migration_entry_young(swp_entry_t entry) | |
312 | { | |
313 | if (migration_entry_supports_ad()) | |
314 | return swp_offset(entry) & SWP_MIG_YOUNG; | |
315 | /* Keep the old behavior of aging page after migration */ | |
316 | return false; | |
317 | } | |
318 | ||
319 | static inline swp_entry_t make_migration_entry_dirty(swp_entry_t entry) | |
320 | { | |
321 | if (migration_entry_supports_ad()) | |
322 | return swp_entry(swp_type(entry), | |
323 | swp_offset(entry) | SWP_MIG_DIRTY); | |
324 | return entry; | |
325 | } | |
326 | ||
327 | static inline bool is_migration_entry_dirty(swp_entry_t entry) | |
328 | { | |
329 | if (migration_entry_supports_ad()) | |
330 | return swp_offset(entry) & SWP_MIG_DIRTY; | |
331 | /* Keep the old behavior of clean page after migration */ | |
332 | return false; | |
333 | } | |
334 | ||
0697212a CL |
335 | extern void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd, |
336 | unsigned long address); | |
ad1ac596 | 337 | extern void migration_entry_wait_huge(struct vm_area_struct *vma, pte_t *pte); |
eba4d770 | 338 | #else /* CONFIG_MIGRATION */ |
4dd845b5 AP |
339 | static inline swp_entry_t make_readable_migration_entry(pgoff_t offset) |
340 | { | |
341 | return swp_entry(0, 0); | |
342 | } | |
343 | ||
6c287605 DH |
344 | static inline swp_entry_t make_readable_exclusive_migration_entry(pgoff_t offset) |
345 | { | |
346 | return swp_entry(0, 0); | |
347 | } | |
348 | ||
4dd845b5 AP |
349 | static inline swp_entry_t make_writable_migration_entry(pgoff_t offset) |
350 | { | |
351 | return swp_entry(0, 0); | |
352 | } | |
0697212a | 353 | |
5ec553a9 AM |
354 | static inline int is_migration_entry(swp_entry_t swp) |
355 | { | |
356 | return 0; | |
357 | } | |
0d665e7b | 358 | |
0697212a | 359 | static inline void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd, |
0cb8fd4d HD |
360 | unsigned long address) { } |
361 | static inline void migration_entry_wait_huge(struct vm_area_struct *vma, | |
362 | pte_t *pte) { } | |
4dd845b5 | 363 | static inline int is_writable_migration_entry(swp_entry_t entry) |
0697212a CL |
364 | { |
365 | return 0; | |
366 | } | |
6c287605 DH |
367 | static inline int is_readable_migration_entry(swp_entry_t entry) |
368 | { | |
369 | return 0; | |
370 | } | |
0697212a | 371 | |
2e346877 PX |
372 | static inline swp_entry_t make_migration_entry_young(swp_entry_t entry) |
373 | { | |
374 | return entry; | |
375 | } | |
376 | ||
377 | static inline bool is_migration_entry_young(swp_entry_t entry) | |
378 | { | |
379 | return false; | |
380 | } | |
381 | ||
382 | static inline swp_entry_t make_migration_entry_dirty(swp_entry_t entry) | |
383 | { | |
384 | return entry; | |
385 | } | |
386 | ||
387 | static inline bool is_migration_entry_dirty(swp_entry_t entry) | |
388 | { | |
389 | return false; | |
390 | } | |
eba4d770 | 391 | #endif /* CONFIG_MIGRATION */ |
0697212a | 392 | |
679d1033 PX |
393 | typedef unsigned long pte_marker; |
394 | ||
15520a3f | 395 | #define PTE_MARKER_UFFD_WP BIT(0) |
af19487f AR |
396 | /* |
397 | * "Poisoned" here is meant in the very general sense of "future accesses are | |
398 | * invalid", instead of referring very specifically to hardware memory errors. | |
399 | * This marker is meant to represent any of various different causes of this. | |
400 | */ | |
401 | #define PTE_MARKER_POISONED BIT(1) | |
15520a3f | 402 | #define PTE_MARKER_MASK (BIT(2) - 1) |
679d1033 | 403 | |
679d1033 PX |
404 | static inline swp_entry_t make_pte_marker_entry(pte_marker marker) |
405 | { | |
406 | return swp_entry(SWP_PTE_MARKER, marker); | |
407 | } | |
408 | ||
409 | static inline bool is_pte_marker_entry(swp_entry_t entry) | |
410 | { | |
411 | return swp_type(entry) == SWP_PTE_MARKER; | |
412 | } | |
413 | ||
414 | static inline pte_marker pte_marker_get(swp_entry_t entry) | |
415 | { | |
416 | return swp_offset(entry) & PTE_MARKER_MASK; | |
417 | } | |
418 | ||
419 | static inline bool is_pte_marker(pte_t pte) | |
420 | { | |
421 | return is_swap_pte(pte) && is_pte_marker_entry(pte_to_swp_entry(pte)); | |
422 | } | |
423 | ||
679d1033 PX |
424 | static inline pte_t make_pte_marker(pte_marker marker) |
425 | { | |
426 | return swp_entry_to_pte(make_pte_marker_entry(marker)); | |
427 | } | |
428 | ||
af19487f | 429 | static inline swp_entry_t make_poisoned_swp_entry(void) |
15520a3f | 430 | { |
af19487f | 431 | return make_pte_marker_entry(PTE_MARKER_POISONED); |
15520a3f PX |
432 | } |
433 | ||
af19487f | 434 | static inline int is_poisoned_swp_entry(swp_entry_t entry) |
15520a3f PX |
435 | { |
436 | return is_pte_marker_entry(entry) && | |
af19487f | 437 | (pte_marker_get(entry) & PTE_MARKER_POISONED); |
15520a3f PX |
438 | } |
439 | ||
679d1033 PX |
440 | /* |
441 | * This is a special version to check pte_none() just to cover the case when | |
442 | * the pte is a pte marker. It existed because in many cases the pte marker | |
443 | * should be seen as a none pte; it's just that we have stored some information | |
444 | * onto the none pte so it becomes not-none any more. | |
445 | * | |
446 | * It should be used when the pte is file-backed, ram-based and backing | |
447 | * userspace pages, like shmem. It is not needed upon pgtables that do not | |
448 | * support pte markers at all. For example, it's not needed on anonymous | |
449 | * memory, kernel-only memory (including when the system is during-boot), | |
450 | * non-ram based generic file-system. It's fine to be used even there, but the | |
451 | * extra pte marker check will be pure overhead. | |
679d1033 PX |
452 | */ |
453 | static inline int pte_none_mostly(pte_t pte) | |
454 | { | |
455 | return pte_none(pte) || is_pte_marker(pte); | |
456 | } | |
457 | ||
af5cdaf8 AP |
458 | static inline struct page *pfn_swap_entry_to_page(swp_entry_t entry) |
459 | { | |
0d206b5d | 460 | struct page *p = pfn_to_page(swp_offset_pfn(entry)); |
af5cdaf8 AP |
461 | |
462 | /* | |
463 | * Any use of migration entries may only occur while the | |
464 | * corresponding page is locked | |
465 | */ | |
466 | BUG_ON(is_migration_entry(entry) && !PageLocked(p)); | |
467 | ||
468 | return p; | |
469 | } | |
470 | ||
471 | /* | |
472 | * A pfn swap entry is a special type of swap entry that always has a pfn stored | |
473 | * in the swap offset. They are used to represent unaddressable device memory | |
474 | * and to restrict access to a page undergoing migration. | |
475 | */ | |
476 | static inline bool is_pfn_swap_entry(swp_entry_t entry) | |
477 | { | |
0d206b5d PX |
478 | /* Make sure the swp offset can always store the needed fields */ |
479 | BUILD_BUG_ON(SWP_TYPE_SHIFT < SWP_PFN_BITS); | |
480 | ||
b756a3b5 AP |
481 | return is_migration_entry(entry) || is_device_private_entry(entry) || |
482 | is_device_exclusive_entry(entry); | |
af5cdaf8 AP |
483 | } |
484 | ||
616b8371 ZY |
485 | struct page_vma_mapped_walk; |
486 | ||
487 | #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION | |
7f5abe60 | 488 | extern int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, |
616b8371 ZY |
489 | struct page *page); |
490 | ||
491 | extern void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, | |
492 | struct page *new); | |
493 | ||
494 | extern void pmd_migration_entry_wait(struct mm_struct *mm, pmd_t *pmd); | |
495 | ||
496 | static inline swp_entry_t pmd_to_swp_entry(pmd_t pmd) | |
497 | { | |
498 | swp_entry_t arch_entry; | |
499 | ||
ab6e3d09 NH |
500 | if (pmd_swp_soft_dirty(pmd)) |
501 | pmd = pmd_swp_clear_soft_dirty(pmd); | |
8f34f1ea PX |
502 | if (pmd_swp_uffd_wp(pmd)) |
503 | pmd = pmd_swp_clear_uffd_wp(pmd); | |
616b8371 ZY |
504 | arch_entry = __pmd_to_swp_entry(pmd); |
505 | return swp_entry(__swp_type(arch_entry), __swp_offset(arch_entry)); | |
506 | } | |
507 | ||
508 | static inline pmd_t swp_entry_to_pmd(swp_entry_t entry) | |
509 | { | |
510 | swp_entry_t arch_entry; | |
511 | ||
512 | arch_entry = __swp_entry(swp_type(entry), swp_offset(entry)); | |
513 | return __swp_entry_to_pmd(arch_entry); | |
514 | } | |
515 | ||
516 | static inline int is_pmd_migration_entry(pmd_t pmd) | |
517 | { | |
b304c6f0 | 518 | return is_swap_pmd(pmd) && is_migration_entry(pmd_to_swp_entry(pmd)); |
616b8371 | 519 | } |
eba4d770 | 520 | #else /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ |
7f5abe60 | 521 | static inline int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, |
616b8371 ZY |
522 | struct page *page) |
523 | { | |
524 | BUILD_BUG(); | |
525 | } | |
526 | ||
527 | static inline void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, | |
528 | struct page *new) | |
529 | { | |
530 | BUILD_BUG(); | |
531 | } | |
532 | ||
533 | static inline void pmd_migration_entry_wait(struct mm_struct *m, pmd_t *p) { } | |
534 | ||
535 | static inline swp_entry_t pmd_to_swp_entry(pmd_t pmd) | |
536 | { | |
537 | return swp_entry(0, 0); | |
538 | } | |
539 | ||
540 | static inline pmd_t swp_entry_to_pmd(swp_entry_t entry) | |
541 | { | |
542 | return __pmd(0); | |
543 | } | |
544 | ||
545 | static inline int is_pmd_migration_entry(pmd_t pmd) | |
546 | { | |
547 | return 0; | |
548 | } | |
eba4d770 | 549 | #endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ |
616b8371 | 550 | |
a7420aa5 | 551 | #ifdef CONFIG_MEMORY_FAILURE |
8e30456b | 552 | |
a7420aa5 AK |
553 | /* |
554 | * Support for hardware poisoned pages | |
555 | */ | |
556 | static inline swp_entry_t make_hwpoison_entry(struct page *page) | |
557 | { | |
558 | BUG_ON(!PageLocked(page)); | |
559 | return swp_entry(SWP_HWPOISON, page_to_pfn(page)); | |
560 | } | |
561 | ||
562 | static inline int is_hwpoison_entry(swp_entry_t entry) | |
563 | { | |
564 | return swp_type(entry) == SWP_HWPOISON; | |
565 | } | |
8e30456b | 566 | |
d027122d | 567 | #else |
a7420aa5 AK |
568 | |
569 | static inline swp_entry_t make_hwpoison_entry(struct page *page) | |
570 | { | |
571 | return swp_entry(0, 0); | |
572 | } | |
573 | ||
574 | static inline int is_hwpoison_entry(swp_entry_t swp) | |
575 | { | |
576 | return 0; | |
577 | } | |
d027122d | 578 | #endif |
a7420aa5 | 579 | |
a7420aa5 AK |
580 | static inline int non_swap_entry(swp_entry_t entry) |
581 | { | |
582 | return swp_type(entry) >= MAX_SWAPFILES; | |
583 | } | |
a2c16d6c | 584 | |
9b98fa22 | 585 | #endif /* CONFIG_MMU */ |
a2c16d6c | 586 | #endif /* _LINUX_SWAPOPS_H */ |