Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
748446bb MG |
2 | #ifndef _LINUX_COMPACTION_H |
3 | #define _LINUX_COMPACTION_H | |
4 | ||
a5508cd8 VB |
5 | /* |
6 | * Determines how hard direct compaction should try to succeed. | |
7 | * Lower value means higher priority, analogically to reclaim priority. | |
8 | */ | |
9 | enum compact_priority { | |
a8e025e5 VB |
10 | COMPACT_PRIO_SYNC_FULL, |
11 | MIN_COMPACT_PRIORITY = COMPACT_PRIO_SYNC_FULL, | |
a5508cd8 | 12 | COMPACT_PRIO_SYNC_LIGHT, |
c2033b00 | 13 | MIN_COMPACT_COSTLY_PRIORITY = COMPACT_PRIO_SYNC_LIGHT, |
a5508cd8 VB |
14 | DEF_COMPACT_PRIORITY = COMPACT_PRIO_SYNC_LIGHT, |
15 | COMPACT_PRIO_ASYNC, | |
16 | INIT_COMPACT_PRIORITY = COMPACT_PRIO_ASYNC | |
17 | }; | |
18 | ||
56de7263 | 19 | /* Return values for compact_zone() and try_to_compact_pages() */ |
fa6c7b46 | 20 | /* When adding new states, please adjust include/trace/events/compaction.h */ |
ea7ab982 | 21 | enum compact_result { |
4f9a358c MH |
22 | /* For more detailed tracepoint output - internal to compaction */ |
23 | COMPACT_NOT_SUITABLE_ZONE, | |
ea7ab982 MH |
24 | /* |
25 | * compaction didn't start as it was not possible or direct reclaim | |
26 | * was more suitable | |
27 | */ | |
28 | COMPACT_SKIPPED, | |
1d4746d3 MH |
29 | /* compaction didn't start as it was deferred due to past failures */ |
30 | COMPACT_DEFERRED, | |
4f9a358c | 31 | |
4f9a358c MH |
32 | /* For more detailed tracepoint output - internal to compaction */ |
33 | COMPACT_NO_SUITABLE_PAGE, | |
ea7ab982 MH |
34 | /* compaction should continue to another pageblock */ |
35 | COMPACT_CONTINUE, | |
4f9a358c | 36 | |
ea7ab982 | 37 | /* |
06c88398 | 38 | * The full zone was compacted scanned but wasn't successful to compact |
4f9a358c | 39 | * suitable pages. |
ea7ab982 | 40 | */ |
4f9a358c | 41 | COMPACT_COMPLETE, |
c8f7de0b | 42 | /* |
06c88398 | 43 | * direct compaction has scanned part of the zone but wasn't successful |
c8f7de0b MH |
44 | * to compact suitable pages. |
45 | */ | |
46 | COMPACT_PARTIAL_SKIPPED, | |
4f9a358c MH |
47 | |
48 | /* compaction terminated prematurely due to lock contentions */ | |
49 | COMPACT_CONTENDED, | |
50 | ||
c8f7de0b | 51 | /* |
cf378319 VB |
52 | * direct compaction terminated after concluding that the allocation |
53 | * should now succeed | |
c8f7de0b | 54 | */ |
cf378319 | 55 | COMPACT_SUCCESS, |
ea7ab982 | 56 | }; |
748446bb | 57 | |
1a6d53a1 VB |
58 | struct alloc_context; /* in mm/internal.h */ |
59 | ||
9861a62c VB |
60 | /* |
61 | * Number of free order-0 pages that should be available above given watermark | |
62 | * to make sure compaction has reasonable chance of not running out of free | |
63 | * pages that it needs to isolate as migration target during its work. | |
64 | */ | |
65 | static inline unsigned long compact_gap(unsigned int order) | |
66 | { | |
67 | /* | |
68 | * Although all the isolations for migration are temporary, compaction | |
69 | * free scanner may have up to 1 << order pages on its list and then | |
70 | * try to split an (order - 1) free page. At that point, a gap of | |
71 | * 1 << order might not be enough, so it's safer to require twice that | |
72 | * amount. Note that the number of pages on the list is also | |
73 | * effectively limited by COMPACT_CLUSTER_MAX, as that's the maximum | |
74 | * that the migrate scanner can have isolated on migrate list, and free | |
75 | * scanner is only invoked when the number of isolated free pages is | |
76 | * lower than that. But it's not worth to complicate the formula here | |
77 | * as a bigger gap for higher orders than strictly necessary can also | |
78 | * improve chances of compaction success. | |
79 | */ | |
80 | return 2UL << order; | |
81 | } | |
82 | ||
76ab0f53 | 83 | #ifdef CONFIG_COMPACTION |
d34c0a75 | 84 | extern unsigned int sysctl_compaction_proactiveness; |
76ab0f53 | 85 | extern int sysctl_compaction_handler(struct ctl_table *table, int write, |
32927393 | 86 | void *buffer, size_t *length, loff_t *ppos); |
65d759c8 CTR |
87 | extern int compaction_proactiveness_sysctl_handler(struct ctl_table *table, |
88 | int write, void *buffer, size_t *length, loff_t *ppos); | |
5e771905 | 89 | extern int sysctl_extfrag_threshold; |
5bbe3547 | 90 | extern int sysctl_compact_unevictable_allowed; |
56de7263 | 91 | |
d34c0a75 | 92 | extern unsigned int extfrag_for_order(struct zone *zone, unsigned int order); |
56de7263 | 93 | extern int fragmentation_index(struct zone *zone, unsigned int order); |
ea7ab982 | 94 | extern enum compact_result try_to_compact_pages(gfp_t gfp_mask, |
c3486f53 | 95 | unsigned int order, unsigned int alloc_flags, |
5e1f0f09 MG |
96 | const struct alloc_context *ac, enum compact_priority prio, |
97 | struct page **page); | |
62997027 | 98 | extern void reset_isolation_suitable(pg_data_t *pgdat); |
ea7ab982 | 99 | extern enum compact_result compaction_suitable(struct zone *zone, int order, |
97a225e6 | 100 | unsigned int alloc_flags, int highest_zoneidx); |
4f92e258 | 101 | |
24e2716f JK |
102 | extern void compaction_defer_reset(struct zone *zone, int order, |
103 | bool alloc_success); | |
62997027 | 104 | |
cab1802b MH |
105 | /* Compaction has made some progress and retrying makes sense */ |
106 | static inline bool compaction_made_progress(enum compact_result result) | |
107 | { | |
108 | /* | |
109 | * Even though this might sound confusing this in fact tells us | |
110 | * that the compaction successfully isolated and migrated some | |
111 | * pageblocks. | |
112 | */ | |
cf378319 | 113 | if (result == COMPACT_SUCCESS) |
cab1802b MH |
114 | return true; |
115 | ||
116 | return false; | |
117 | } | |
118 | ||
119 | /* Compaction has failed and it doesn't make much sense to keep retrying. */ | |
120 | static inline bool compaction_failed(enum compact_result result) | |
121 | { | |
122 | /* All zones were scanned completely and still not result. */ | |
123 | if (result == COMPACT_COMPLETE) | |
124 | return true; | |
125 | ||
126 | return false; | |
127 | } | |
128 | ||
49433085 VB |
129 | /* Compaction needs reclaim to be performed first, so it can continue. */ |
130 | static inline bool compaction_needs_reclaim(enum compact_result result) | |
cab1802b MH |
131 | { |
132 | /* | |
133 | * Compaction backed off due to watermark checks for order-0 | |
134 | * so the regular reclaim has to try harder and reclaim something. | |
135 | */ | |
136 | if (result == COMPACT_SKIPPED) | |
137 | return true; | |
138 | ||
49433085 VB |
139 | return false; |
140 | } | |
141 | ||
142 | /* | |
143 | * Compaction has backed off for some reason after doing some work or none | |
144 | * at all. It might be throttling or lock contention. Retrying might be still | |
145 | * worthwhile, but with a higher priority if allowed. | |
146 | */ | |
147 | static inline bool compaction_withdrawn(enum compact_result result) | |
148 | { | |
cab1802b MH |
149 | /* |
150 | * If compaction is deferred for high-order allocations, it is | |
151 | * because sync compaction recently failed. If this is the case | |
152 | * and the caller requested a THP allocation, we do not want | |
153 | * to heavily disrupt the system, so we fail the allocation | |
154 | * instead of entering direct reclaim. | |
155 | */ | |
156 | if (result == COMPACT_DEFERRED) | |
157 | return true; | |
158 | ||
159 | /* | |
160 | * If compaction in async mode encounters contention or blocks higher | |
161 | * priority task we back off early rather than cause stalls. | |
162 | */ | |
163 | if (result == COMPACT_CONTENDED) | |
164 | return true; | |
165 | ||
166 | /* | |
167 | * Page scanners have met but we haven't scanned full zones so this | |
168 | * is a back off in fact. | |
169 | */ | |
170 | if (result == COMPACT_PARTIAL_SKIPPED) | |
171 | return true; | |
172 | ||
173 | return false; | |
174 | } | |
175 | ||
86a294a8 MH |
176 | |
177 | bool compaction_zonelist_suitable(struct alloc_context *ac, int order, | |
178 | int alloc_flags); | |
179 | ||
698b1b30 VB |
180 | extern int kcompactd_run(int nid); |
181 | extern void kcompactd_stop(int nid); | |
97a225e6 | 182 | extern void wakeup_kcompactd(pg_data_t *pgdat, int order, int highest_zoneidx); |
698b1b30 | 183 | |
56de7263 | 184 | #else |
62997027 MG |
185 | static inline void reset_isolation_suitable(pg_data_t *pgdat) |
186 | { | |
187 | } | |
188 | ||
ea7ab982 | 189 | static inline enum compact_result compaction_suitable(struct zone *zone, int order, |
97a225e6 | 190 | int alloc_flags, int highest_zoneidx) |
3e7d3449 MG |
191 | { |
192 | return COMPACT_SKIPPED; | |
193 | } | |
194 | ||
cab1802b MH |
195 | static inline bool compaction_made_progress(enum compact_result result) |
196 | { | |
197 | return false; | |
198 | } | |
199 | ||
200 | static inline bool compaction_failed(enum compact_result result) | |
201 | { | |
202 | return false; | |
203 | } | |
204 | ||
49433085 VB |
205 | static inline bool compaction_needs_reclaim(enum compact_result result) |
206 | { | |
207 | return false; | |
208 | } | |
209 | ||
cab1802b MH |
210 | static inline bool compaction_withdrawn(enum compact_result result) |
211 | { | |
212 | return true; | |
213 | } | |
214 | ||
698b1b30 VB |
215 | static inline int kcompactd_run(int nid) |
216 | { | |
217 | return 0; | |
218 | } | |
219 | static inline void kcompactd_stop(int nid) | |
220 | { | |
221 | } | |
222 | ||
97a225e6 JK |
223 | static inline void wakeup_kcompactd(pg_data_t *pgdat, |
224 | int order, int highest_zoneidx) | |
698b1b30 VB |
225 | { |
226 | } | |
227 | ||
76ab0f53 MG |
228 | #endif /* CONFIG_COMPACTION */ |
229 | ||
bda807d4 | 230 | struct node; |
ace451eb | 231 | #if defined(CONFIG_COMPACTION) && defined(CONFIG_SYSFS) && defined(CONFIG_NUMA) |
ed4a6d7f MG |
232 | extern int compaction_register_node(struct node *node); |
233 | extern void compaction_unregister_node(struct node *node); | |
234 | ||
235 | #else | |
236 | ||
237 | static inline int compaction_register_node(struct node *node) | |
238 | { | |
239 | return 0; | |
240 | } | |
241 | ||
242 | static inline void compaction_unregister_node(struct node *node) | |
243 | { | |
244 | } | |
245 | #endif /* CONFIG_COMPACTION && CONFIG_SYSFS && CONFIG_NUMA */ | |
246 | ||
748446bb | 247 | #endif /* _LINUX_COMPACTION_H */ |