Commit | Line | Data |
---|---|---|
a28397be SP |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* | |
3 | * DAMON Primitives for The Physical Address Space | |
4 | * | |
5 | * Author: SeongJae Park <sj@kernel.org> | |
6 | */ | |
7 | ||
8 | #define pr_fmt(fmt) "damon-pa: " fmt | |
9 | ||
10 | #include <linux/mmu_notifier.h> | |
11 | #include <linux/page_idle.h> | |
12 | #include <linux/pagemap.h> | |
13 | #include <linux/rmap.h> | |
57223ac2 | 14 | #include <linux/swap.h> |
a28397be | 15 | |
57223ac2 | 16 | #include "../internal.h" |
f7d911c3 | 17 | #include "ops-common.h" |
a28397be | 18 | |
2f031c6f | 19 | static bool __damon_pa_mkold(struct folio *folio, struct vm_area_struct *vma, |
a28397be SP |
20 | unsigned long addr, void *arg) |
21 | { | |
2f031c6f | 22 | DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0); |
a28397be SP |
23 | |
24 | while (page_vma_mapped_walk(&pvmw)) { | |
25 | addr = pvmw.address; | |
26 | if (pvmw.pte) | |
27 | damon_ptep_mkold(pvmw.pte, vma->vm_mm, addr); | |
28 | else | |
29 | damon_pmdp_mkold(pvmw.pmd, vma->vm_mm, addr); | |
30 | } | |
31 | return true; | |
32 | } | |
33 | ||
34 | static void damon_pa_mkold(unsigned long paddr) | |
35 | { | |
6d42dba3 | 36 | struct folio *folio; |
a28397be SP |
37 | struct page *page = damon_get_page(PHYS_PFN(paddr)); |
38 | struct rmap_walk_control rwc = { | |
39 | .rmap_one = __damon_pa_mkold, | |
2f031c6f | 40 | .anon_lock = folio_lock_anon_vma_read, |
a28397be SP |
41 | }; |
42 | bool need_lock; | |
43 | ||
44 | if (!page) | |
45 | return; | |
6d42dba3 | 46 | folio = page_folio(page); |
a28397be | 47 | |
6d42dba3 MWO |
48 | if (!folio_mapped(folio) || !folio_raw_mapping(folio)) { |
49 | folio_set_idle(folio); | |
a28397be SP |
50 | goto out; |
51 | } | |
52 | ||
6d42dba3 MWO |
53 | need_lock = !folio_test_anon(folio) || folio_test_ksm(folio); |
54 | if (need_lock && !folio_trylock(folio)) | |
a28397be SP |
55 | goto out; |
56 | ||
2f031c6f | 57 | rmap_walk(folio, &rwc); |
a28397be SP |
58 | |
59 | if (need_lock) | |
6d42dba3 | 60 | folio_unlock(folio); |
a28397be SP |
61 | |
62 | out: | |
6d42dba3 | 63 | folio_put(folio); |
a28397be SP |
64 | } |
65 | ||
8ef4d5ca | 66 | static void __damon_pa_prepare_access_check(struct damon_region *r) |
a28397be SP |
67 | { |
68 | r->sampling_addr = damon_rand(r->ar.start, r->ar.end); | |
69 | ||
70 | damon_pa_mkold(r->sampling_addr); | |
71 | } | |
72 | ||
cdeed009 | 73 | static void damon_pa_prepare_access_checks(struct damon_ctx *ctx) |
a28397be SP |
74 | { |
75 | struct damon_target *t; | |
76 | struct damon_region *r; | |
77 | ||
78 | damon_for_each_target(t, ctx) { | |
79 | damon_for_each_region(r, t) | |
8ef4d5ca | 80 | __damon_pa_prepare_access_check(r); |
a28397be SP |
81 | } |
82 | } | |
83 | ||
84 | struct damon_pa_access_chk_result { | |
85 | unsigned long page_sz; | |
86 | bool accessed; | |
87 | }; | |
88 | ||
2f031c6f | 89 | static bool __damon_pa_young(struct folio *folio, struct vm_area_struct *vma, |
a28397be SP |
90 | unsigned long addr, void *arg) |
91 | { | |
92 | struct damon_pa_access_chk_result *result = arg; | |
c8423186 | 93 | DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0); |
a28397be SP |
94 | |
95 | result->accessed = false; | |
96 | result->page_sz = PAGE_SIZE; | |
97 | while (page_vma_mapped_walk(&pvmw)) { | |
98 | addr = pvmw.address; | |
99 | if (pvmw.pte) { | |
100 | result->accessed = pte_young(*pvmw.pte) || | |
c8423186 | 101 | !folio_test_idle(folio) || |
a28397be SP |
102 | mmu_notifier_test_young(vma->vm_mm, addr); |
103 | } else { | |
104 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE | |
105 | result->accessed = pmd_young(*pvmw.pmd) || | |
c8423186 | 106 | !folio_test_idle(folio) || |
a28397be | 107 | mmu_notifier_test_young(vma->vm_mm, addr); |
02e34fff | 108 | result->page_sz = HPAGE_PMD_SIZE; |
a28397be SP |
109 | #else |
110 | WARN_ON_ONCE(1); | |
111 | #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ | |
112 | } | |
113 | if (result->accessed) { | |
114 | page_vma_mapped_walk_done(&pvmw); | |
115 | break; | |
116 | } | |
117 | } | |
118 | ||
119 | /* If accessed, stop walking */ | |
120 | return !result->accessed; | |
121 | } | |
122 | ||
123 | static bool damon_pa_young(unsigned long paddr, unsigned long *page_sz) | |
124 | { | |
c8423186 | 125 | struct folio *folio; |
a28397be SP |
126 | struct page *page = damon_get_page(PHYS_PFN(paddr)); |
127 | struct damon_pa_access_chk_result result = { | |
128 | .page_sz = PAGE_SIZE, | |
129 | .accessed = false, | |
130 | }; | |
131 | struct rmap_walk_control rwc = { | |
132 | .arg = &result, | |
133 | .rmap_one = __damon_pa_young, | |
2f031c6f | 134 | .anon_lock = folio_lock_anon_vma_read, |
a28397be SP |
135 | }; |
136 | bool need_lock; | |
137 | ||
138 | if (!page) | |
139 | return false; | |
c8423186 | 140 | folio = page_folio(page); |
a28397be | 141 | |
c8423186 MWO |
142 | if (!folio_mapped(folio) || !folio_raw_mapping(folio)) { |
143 | if (folio_test_idle(folio)) | |
a28397be SP |
144 | result.accessed = false; |
145 | else | |
146 | result.accessed = true; | |
c8423186 | 147 | folio_put(folio); |
a28397be SP |
148 | goto out; |
149 | } | |
150 | ||
c8423186 MWO |
151 | need_lock = !folio_test_anon(folio) || folio_test_ksm(folio); |
152 | if (need_lock && !folio_trylock(folio)) { | |
153 | folio_put(folio); | |
2a3c4bce | 154 | return false; |
a28397be SP |
155 | } |
156 | ||
2f031c6f | 157 | rmap_walk(folio, &rwc); |
a28397be SP |
158 | |
159 | if (need_lock) | |
c8423186 MWO |
160 | folio_unlock(folio); |
161 | folio_put(folio); | |
a28397be SP |
162 | |
163 | out: | |
164 | *page_sz = result.page_sz; | |
165 | return result.accessed; | |
166 | } | |
167 | ||
09876ae7 | 168 | static void __damon_pa_check_access(struct damon_region *r) |
a28397be SP |
169 | { |
170 | static unsigned long last_addr; | |
171 | static unsigned long last_page_sz = PAGE_SIZE; | |
172 | static bool last_accessed; | |
173 | ||
174 | /* If the region is in the last checked page, reuse the result */ | |
175 | if (ALIGN_DOWN(last_addr, last_page_sz) == | |
176 | ALIGN_DOWN(r->sampling_addr, last_page_sz)) { | |
177 | if (last_accessed) | |
178 | r->nr_accesses++; | |
179 | return; | |
180 | } | |
181 | ||
182 | last_accessed = damon_pa_young(r->sampling_addr, &last_page_sz); | |
183 | if (last_accessed) | |
184 | r->nr_accesses++; | |
185 | ||
186 | last_addr = r->sampling_addr; | |
187 | } | |
188 | ||
cdeed009 | 189 | static unsigned int damon_pa_check_accesses(struct damon_ctx *ctx) |
a28397be SP |
190 | { |
191 | struct damon_target *t; | |
192 | struct damon_region *r; | |
193 | unsigned int max_nr_accesses = 0; | |
194 | ||
195 | damon_for_each_target(t, ctx) { | |
196 | damon_for_each_region(r, t) { | |
09876ae7 | 197 | __damon_pa_check_access(r); |
a28397be SP |
198 | max_nr_accesses = max(r->nr_accesses, max_nr_accesses); |
199 | } | |
200 | } | |
201 | ||
202 | return max_nr_accesses; | |
203 | } | |
204 | ||
0e93e8bf | 205 | static unsigned long damon_pa_pageout(struct damon_region *r) |
57223ac2 | 206 | { |
0e92c2ee | 207 | unsigned long addr, applied; |
57223ac2 SP |
208 | LIST_HEAD(page_list); |
209 | ||
57223ac2 SP |
210 | for (addr = r->ar.start; addr < r->ar.end; addr += PAGE_SIZE) { |
211 | struct page *page = damon_get_page(PHYS_PFN(addr)); | |
212 | ||
213 | if (!page) | |
214 | continue; | |
215 | ||
216 | ClearPageReferenced(page); | |
217 | test_and_clear_page_young(page); | |
218 | if (isolate_lru_page(page)) { | |
219 | put_page(page); | |
220 | continue; | |
221 | } | |
222 | if (PageUnevictable(page)) { | |
223 | putback_lru_page(page); | |
224 | } else { | |
225 | list_add(&page->lru, &page_list); | |
226 | put_page(page); | |
227 | } | |
228 | } | |
0e92c2ee | 229 | applied = reclaim_pages(&page_list); |
57223ac2 | 230 | cond_resched(); |
0e92c2ee | 231 | return applied * PAGE_SIZE; |
57223ac2 SP |
232 | } |
233 | ||
8193321a SP |
234 | static inline unsigned long damon_pa_mark_accessed_or_deactivate( |
235 | struct damon_region *r, bool mark_accessed) | |
8cdcc532 SP |
236 | { |
237 | unsigned long addr, applied = 0; | |
238 | ||
239 | for (addr = r->ar.start; addr < r->ar.end; addr += PAGE_SIZE) { | |
240 | struct page *page = damon_get_page(PHYS_PFN(addr)); | |
241 | ||
242 | if (!page) | |
243 | continue; | |
8193321a SP |
244 | if (mark_accessed) |
245 | mark_page_accessed(page); | |
246 | else | |
247 | deactivate_page(page); | |
8cdcc532 SP |
248 | put_page(page); |
249 | applied++; | |
250 | } | |
251 | return applied * PAGE_SIZE; | |
252 | } | |
253 | ||
8193321a | 254 | static unsigned long damon_pa_mark_accessed(struct damon_region *r) |
99cdc2cd | 255 | { |
8193321a SP |
256 | return damon_pa_mark_accessed_or_deactivate(r, true); |
257 | } | |
99cdc2cd | 258 | |
8193321a SP |
259 | static unsigned long damon_pa_deactivate_pages(struct damon_region *r) |
260 | { | |
261 | return damon_pa_mark_accessed_or_deactivate(r, false); | |
99cdc2cd SP |
262 | } |
263 | ||
0e93e8bf SP |
264 | static unsigned long damon_pa_apply_scheme(struct damon_ctx *ctx, |
265 | struct damon_target *t, struct damon_region *r, | |
266 | struct damos *scheme) | |
267 | { | |
268 | switch (scheme->action) { | |
269 | case DAMOS_PAGEOUT: | |
270 | return damon_pa_pageout(r); | |
8cdcc532 SP |
271 | case DAMOS_LRU_PRIO: |
272 | return damon_pa_mark_accessed(r); | |
99cdc2cd SP |
273 | case DAMOS_LRU_DEPRIO: |
274 | return damon_pa_deactivate_pages(r); | |
f82e70e2 SP |
275 | case DAMOS_STAT: |
276 | break; | |
0e93e8bf | 277 | default: |
f82e70e2 | 278 | /* DAMOS actions that not yet supported by 'paddr'. */ |
0e93e8bf SP |
279 | break; |
280 | } | |
281 | return 0; | |
282 | } | |
283 | ||
cdeed009 XH |
284 | static int damon_pa_scheme_score(struct damon_ctx *context, |
285 | struct damon_target *t, struct damon_region *r, | |
286 | struct damos *scheme) | |
198f0f4c SP |
287 | { |
288 | switch (scheme->action) { | |
289 | case DAMOS_PAGEOUT: | |
e3e486e6 | 290 | return damon_cold_score(context, r, scheme); |
8cdcc532 SP |
291 | case DAMOS_LRU_PRIO: |
292 | return damon_hot_score(context, r, scheme); | |
99cdc2cd | 293 | case DAMOS_LRU_DEPRIO: |
e3e486e6 | 294 | return damon_cold_score(context, r, scheme); |
198f0f4c SP |
295 | default: |
296 | break; | |
297 | } | |
298 | ||
299 | return DAMOS_MAX_SCORE; | |
300 | } | |
301 | ||
7752925f SP |
302 | static int __init damon_pa_initcall(void) |
303 | { | |
304 | struct damon_operations ops = { | |
305 | .id = DAMON_OPS_PADDR, | |
306 | .init = NULL, | |
307 | .update = NULL, | |
308 | .prepare_access_checks = damon_pa_prepare_access_checks, | |
309 | .check_accesses = damon_pa_check_accesses, | |
310 | .reset_aggregated = NULL, | |
85104056 | 311 | .target_valid = NULL, |
7752925f SP |
312 | .cleanup = NULL, |
313 | .apply_scheme = damon_pa_apply_scheme, | |
314 | .get_scheme_score = damon_pa_scheme_score, | |
315 | }; | |
316 | ||
317 | return damon_register_ops(&ops); | |
318 | }; | |
319 | ||
320 | subsys_initcall(damon_pa_initcall); |