Commit | Line | Data |
---|---|---|
13f876ba TG |
1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | #ifndef _LINUX_HIGHMEM_INTERNAL_H | |
3 | #define _LINUX_HIGHMEM_INTERNAL_H | |
4 | ||
5 | /* | |
6 | * Outside of CONFIG_HIGHMEM to support X86 32bit iomap_atomic() cruft. | |
7 | */ | |
8 | #ifdef CONFIG_KMAP_LOCAL | |
9 | void *__kmap_local_pfn_prot(unsigned long pfn, pgprot_t prot); | |
10 | void *__kmap_local_page_prot(struct page *page, pgprot_t prot); | |
39ade048 | 11 | void kunmap_local_indexed(const void *vaddr); |
5fbda3ec TG |
12 | void kmap_local_fork(struct task_struct *tsk); |
13 | void __kmap_local_sched_out(void); | |
14 | void __kmap_local_sched_in(void); | |
15 | static inline void kmap_assert_nomap(void) | |
16 | { | |
17 | DEBUG_LOCKS_WARN_ON(current->kmap_ctrl.idx); | |
18 | } | |
19 | #else | |
20 | static inline void kmap_local_fork(struct task_struct *tsk) { } | |
21 | static inline void kmap_assert_nomap(void) { } | |
13f876ba TG |
22 | #endif |
23 | ||
24 | #ifdef CONFIG_HIGHMEM | |
25 | #include <asm/highmem.h> | |
26 | ||
27 | #ifndef ARCH_HAS_KMAP_FLUSH_TLB | |
28 | static inline void kmap_flush_tlb(unsigned long addr) { } | |
29 | #endif | |
30 | ||
31 | #ifndef kmap_prot | |
32 | #define kmap_prot PAGE_KERNEL | |
33 | #endif | |
34 | ||
35 | void *kmap_high(struct page *page); | |
36 | void kunmap_high(struct page *page); | |
37 | void __kmap_flush_unused(void); | |
38 | struct page *__kmap_to_page(void *addr); | |
39 | ||
40 | static inline void *kmap(struct page *page) | |
41 | { | |
42 | void *addr; | |
43 | ||
44 | might_sleep(); | |
45 | if (!PageHighMem(page)) | |
46 | addr = page_address(page); | |
47 | else | |
48 | addr = kmap_high(page); | |
49 | kmap_flush_tlb((unsigned long)addr); | |
50 | return addr; | |
51 | } | |
52 | ||
53 | static inline void kunmap(struct page *page) | |
54 | { | |
55 | might_sleep(); | |
56 | if (!PageHighMem(page)) | |
57 | return; | |
58 | kunmap_high(page); | |
59 | } | |
60 | ||
61 | static inline struct page *kmap_to_page(void *addr) | |
62 | { | |
63 | return __kmap_to_page(addr); | |
64 | } | |
65 | ||
66 | static inline void kmap_flush_unused(void) | |
67 | { | |
68 | __kmap_flush_unused(); | |
69 | } | |
70 | ||
f3ba3c71 TG |
71 | static inline void *kmap_local_page(struct page *page) |
72 | { | |
73 | return __kmap_local_page_prot(page, kmap_prot); | |
74 | } | |
75 | ||
53c36de0 MWO |
76 | static inline void *kmap_local_folio(struct folio *folio, size_t offset) |
77 | { | |
78 | struct page *page = folio_page(folio, offset / PAGE_SIZE); | |
79 | return __kmap_local_page_prot(page, kmap_prot) + offset % PAGE_SIZE; | |
80 | } | |
81 | ||
f3ba3c71 TG |
82 | static inline void *kmap_local_page_prot(struct page *page, pgprot_t prot) |
83 | { | |
84 | return __kmap_local_page_prot(page, prot); | |
85 | } | |
86 | ||
87 | static inline void *kmap_local_pfn(unsigned long pfn) | |
88 | { | |
89 | return __kmap_local_pfn_prot(pfn, kmap_prot); | |
90 | } | |
91 | ||
39ade048 | 92 | static inline void __kunmap_local(const void *vaddr) |
f3ba3c71 TG |
93 | { |
94 | kunmap_local_indexed(vaddr); | |
95 | } | |
96 | ||
13f876ba TG |
97 | static inline void *kmap_atomic_prot(struct page *page, pgprot_t prot) |
98 | { | |
51386120 SAS |
99 | if (IS_ENABLED(CONFIG_PREEMPT_RT)) |
100 | migrate_disable(); | |
101 | else | |
102 | preempt_disable(); | |
103 | ||
13f876ba TG |
104 | pagefault_disable(); |
105 | return __kmap_local_page_prot(page, prot); | |
106 | } | |
107 | ||
108 | static inline void *kmap_atomic(struct page *page) | |
109 | { | |
110 | return kmap_atomic_prot(page, kmap_prot); | |
111 | } | |
112 | ||
113 | static inline void *kmap_atomic_pfn(unsigned long pfn) | |
114 | { | |
51386120 SAS |
115 | if (IS_ENABLED(CONFIG_PREEMPT_RT)) |
116 | migrate_disable(); | |
117 | else | |
118 | preempt_disable(); | |
119 | ||
13f876ba TG |
120 | pagefault_disable(); |
121 | return __kmap_local_pfn_prot(pfn, kmap_prot); | |
122 | } | |
123 | ||
39ade048 | 124 | static inline void __kunmap_atomic(const void *addr) |
13f876ba TG |
125 | { |
126 | kunmap_local_indexed(addr); | |
127 | pagefault_enable(); | |
51386120 SAS |
128 | if (IS_ENABLED(CONFIG_PREEMPT_RT)) |
129 | migrate_enable(); | |
130 | else | |
131 | preempt_enable(); | |
13f876ba TG |
132 | } |
133 | ||
134 | unsigned int __nr_free_highpages(void); | |
135 | extern atomic_long_t _totalhigh_pages; | |
136 | ||
137 | static inline unsigned int nr_free_highpages(void) | |
138 | { | |
139 | return __nr_free_highpages(); | |
140 | } | |
141 | ||
142 | static inline unsigned long totalhigh_pages(void) | |
143 | { | |
144 | return (unsigned long)atomic_long_read(&_totalhigh_pages); | |
145 | } | |
146 | ||
13f876ba TG |
147 | static inline void totalhigh_pages_add(long count) |
148 | { | |
149 | atomic_long_add(count, &_totalhigh_pages); | |
150 | } | |
151 | ||
4e140f59 MWO |
152 | static inline bool is_kmap_addr(const void *x) |
153 | { | |
154 | unsigned long addr = (unsigned long)x; | |
f3837ab7 IW |
155 | |
156 | return (addr >= PKMAP_ADDR(0) && addr < PKMAP_ADDR(LAST_PKMAP)) || | |
157 | (addr >= __fix_to_virt(FIX_KMAP_END) && | |
158 | addr < __fix_to_virt(FIX_KMAP_BEGIN)); | |
4e140f59 | 159 | } |
13f876ba TG |
160 | #else /* CONFIG_HIGHMEM */ |
161 | ||
162 | static inline struct page *kmap_to_page(void *addr) | |
163 | { | |
164 | return virt_to_page(addr); | |
165 | } | |
166 | ||
167 | static inline void *kmap(struct page *page) | |
168 | { | |
169 | might_sleep(); | |
170 | return page_address(page); | |
171 | } | |
172 | ||
173 | static inline void kunmap_high(struct page *page) { } | |
174 | static inline void kmap_flush_unused(void) { } | |
175 | ||
176 | static inline void kunmap(struct page *page) | |
177 | { | |
178 | #ifdef ARCH_HAS_FLUSH_ON_KUNMAP | |
179 | kunmap_flush_on_unmap(page_address(page)); | |
180 | #endif | |
181 | } | |
182 | ||
f3ba3c71 TG |
183 | static inline void *kmap_local_page(struct page *page) |
184 | { | |
185 | return page_address(page); | |
186 | } | |
187 | ||
53c36de0 MWO |
188 | static inline void *kmap_local_folio(struct folio *folio, size_t offset) |
189 | { | |
190 | return page_address(&folio->page) + offset; | |
191 | } | |
192 | ||
f3ba3c71 TG |
193 | static inline void *kmap_local_page_prot(struct page *page, pgprot_t prot) |
194 | { | |
195 | return kmap_local_page(page); | |
196 | } | |
197 | ||
198 | static inline void *kmap_local_pfn(unsigned long pfn) | |
199 | { | |
200 | return kmap_local_page(pfn_to_page(pfn)); | |
201 | } | |
202 | ||
39ade048 | 203 | static inline void __kunmap_local(const void *addr) |
f3ba3c71 TG |
204 | { |
205 | #ifdef ARCH_HAS_FLUSH_ON_KUNMAP | |
88d7b120 | 206 | kunmap_flush_on_unmap(PTR_ALIGN_DOWN(addr, PAGE_SIZE)); |
f3ba3c71 TG |
207 | #endif |
208 | } | |
209 | ||
13f876ba TG |
210 | static inline void *kmap_atomic(struct page *page) |
211 | { | |
51386120 SAS |
212 | if (IS_ENABLED(CONFIG_PREEMPT_RT)) |
213 | migrate_disable(); | |
214 | else | |
215 | preempt_disable(); | |
13f876ba TG |
216 | pagefault_disable(); |
217 | return page_address(page); | |
218 | } | |
219 | ||
220 | static inline void *kmap_atomic_prot(struct page *page, pgprot_t prot) | |
221 | { | |
222 | return kmap_atomic(page); | |
223 | } | |
224 | ||
225 | static inline void *kmap_atomic_pfn(unsigned long pfn) | |
226 | { | |
227 | return kmap_atomic(pfn_to_page(pfn)); | |
228 | } | |
229 | ||
39ade048 | 230 | static inline void __kunmap_atomic(const void *addr) |
13f876ba TG |
231 | { |
232 | #ifdef ARCH_HAS_FLUSH_ON_KUNMAP | |
88d7b120 | 233 | kunmap_flush_on_unmap(PTR_ALIGN_DOWN(addr, PAGE_SIZE)); |
13f876ba TG |
234 | #endif |
235 | pagefault_enable(); | |
51386120 SAS |
236 | if (IS_ENABLED(CONFIG_PREEMPT_RT)) |
237 | migrate_enable(); | |
238 | else | |
239 | preempt_enable(); | |
13f876ba TG |
240 | } |
241 | ||
242 | static inline unsigned int nr_free_highpages(void) { return 0; } | |
243 | static inline unsigned long totalhigh_pages(void) { return 0UL; } | |
244 | ||
4e140f59 MWO |
245 | static inline bool is_kmap_addr(const void *x) |
246 | { | |
247 | return false; | |
248 | } | |
249 | ||
13f876ba TG |
250 | #endif /* CONFIG_HIGHMEM */ |
251 | ||
e7392b4e FDF |
252 | /** |
253 | * kunmap_atomic - Unmap the virtual address mapped by kmap_atomic() - deprecated! | |
254 | * @__addr: Virtual address to be unmapped | |
255 | * | |
256 | * Unmaps an address previously mapped by kmap_atomic() and re-enables | |
257 | * pagefaults. Depending on PREEMP_RT configuration, re-enables also | |
258 | * migration and preemption. Users should not count on these side effects. | |
259 | * | |
260 | * Mappings should be unmapped in the reverse order that they were mapped. | |
261 | * See kmap_local_page() for details on nesting. | |
262 | * | |
263 | * @__addr can be any address within the mapped page, so there is no need | |
264 | * to subtract any offset that has been added. In contrast to kunmap(), | |
265 | * this function takes the address returned from kmap_atomic(), not the | |
266 | * page passed to it. The compiler will warn you if you pass the page. | |
13f876ba TG |
267 | */ |
268 | #define kunmap_atomic(__addr) \ | |
269 | do { \ | |
270 | BUILD_BUG_ON(__same_type((__addr), struct page *)); \ | |
271 | __kunmap_atomic(__addr); \ | |
272 | } while (0) | |
273 | ||
d7ca25c5 IW |
274 | /** |
275 | * kunmap_local - Unmap a page mapped via kmap_local_page(). | |
276 | * @__addr: An address within the page mapped | |
277 | * | |
278 | * @__addr can be any address within the mapped page. Commonly it is the | |
279 | * address return from kmap_local_page(), but it can also include offsets. | |
280 | * | |
281 | * Unmapping should be done in the reverse order of the mapping. See | |
282 | * kmap_local_page() for details. | |
283 | */ | |
f3ba3c71 TG |
284 | #define kunmap_local(__addr) \ |
285 | do { \ | |
286 | BUILD_BUG_ON(__same_type((__addr), struct page *)); \ | |
287 | __kunmap_local(__addr); \ | |
288 | } while (0) | |
289 | ||
13f876ba | 290 | #endif |