x86/uaccess: Avoid barrier_nospec() in 64-bit copy_from_user()
[linux-block.git] / mm / page_table_check.c
1 // SPDX-License-Identifier: GPL-2.0
2
3 /*
4  * Copyright (c) 2021, Google LLC.
5  * Pasha Tatashin <pasha.tatashin@soleen.com>
6  */
7 #include <linux/kstrtox.h>
8 #include <linux/mm.h>
9 #include <linux/page_table_check.h>
10 #include <linux/swap.h>
11 #include <linux/swapops.h>
12
13 #undef pr_fmt
14 #define pr_fmt(fmt)     "page_table_check: " fmt
15
16 struct page_table_check {
17         atomic_t anon_map_count;
18         atomic_t file_map_count;
19 };
20
21 static bool __page_table_check_enabled __initdata =
22                                 IS_ENABLED(CONFIG_PAGE_TABLE_CHECK_ENFORCED);
23
24 DEFINE_STATIC_KEY_TRUE(page_table_check_disabled);
25 EXPORT_SYMBOL(page_table_check_disabled);
26
27 static int __init early_page_table_check_param(char *buf)
28 {
29         return kstrtobool(buf, &__page_table_check_enabled);
30 }
31
32 early_param("page_table_check", early_page_table_check_param);
33
34 static bool __init need_page_table_check(void)
35 {
36         return __page_table_check_enabled;
37 }
38
39 static void __init init_page_table_check(void)
40 {
41         if (!__page_table_check_enabled)
42                 return;
43         static_branch_disable(&page_table_check_disabled);
44 }
45
46 struct page_ext_operations page_table_check_ops = {
47         .size = sizeof(struct page_table_check),
48         .need = need_page_table_check,
49         .init = init_page_table_check,
50         .need_shared_flags = false,
51 };
52
53 static struct page_table_check *get_page_table_check(struct page_ext *page_ext)
54 {
55         BUG_ON(!page_ext);
56         return page_ext_data(page_ext, &page_table_check_ops);
57 }
58
59 /*
60  * An entry is removed from the page table, decrement the counters for that page
61  * verify that it is of correct type and counters do not become negative.
62  */
63 static void page_table_check_clear(unsigned long pfn, unsigned long pgcnt)
64 {
65         struct page_ext *page_ext;
66         struct page *page;
67         unsigned long i;
68         bool anon;
69
70         if (!pfn_valid(pfn))
71                 return;
72
73         page = pfn_to_page(pfn);
74         page_ext = page_ext_get(page);
75
76         if (!page_ext)
77                 return;
78
79         BUG_ON(PageSlab(page));
80         anon = PageAnon(page);
81
82         for (i = 0; i < pgcnt; i++) {
83                 struct page_table_check *ptc = get_page_table_check(page_ext);
84
85                 if (anon) {
86                         BUG_ON(atomic_read(&ptc->file_map_count));
87                         BUG_ON(atomic_dec_return(&ptc->anon_map_count) < 0);
88                 } else {
89                         BUG_ON(atomic_read(&ptc->anon_map_count));
90                         BUG_ON(atomic_dec_return(&ptc->file_map_count) < 0);
91                 }
92                 page_ext = page_ext_next(page_ext);
93         }
94         page_ext_put(page_ext);
95 }
96
97 /*
98  * A new entry is added to the page table, increment the counters for that page
99  * verify that it is of correct type and is not being mapped with a different
100  * type to a different process.
101  */
102 static void page_table_check_set(unsigned long pfn, unsigned long pgcnt,
103                                  bool rw)
104 {
105         struct page_ext *page_ext;
106         struct page *page;
107         unsigned long i;
108         bool anon;
109
110         if (!pfn_valid(pfn))
111                 return;
112
113         page = pfn_to_page(pfn);
114         page_ext = page_ext_get(page);
115
116         if (!page_ext)
117                 return;
118
119         BUG_ON(PageSlab(page));
120         anon = PageAnon(page);
121
122         for (i = 0; i < pgcnt; i++) {
123                 struct page_table_check *ptc = get_page_table_check(page_ext);
124
125                 if (anon) {
126                         BUG_ON(atomic_read(&ptc->file_map_count));
127                         BUG_ON(atomic_inc_return(&ptc->anon_map_count) > 1 && rw);
128                 } else {
129                         BUG_ON(atomic_read(&ptc->anon_map_count));
130                         BUG_ON(atomic_inc_return(&ptc->file_map_count) < 0);
131                 }
132                 page_ext = page_ext_next(page_ext);
133         }
134         page_ext_put(page_ext);
135 }
136
137 /*
138  * page is on free list, or is being allocated, verify that counters are zeroes
139  * crash if they are not.
140  */
141 void __page_table_check_zero(struct page *page, unsigned int order)
142 {
143         struct page_ext *page_ext;
144         unsigned long i;
145
146         BUG_ON(PageSlab(page));
147
148         page_ext = page_ext_get(page);
149
150         if (!page_ext)
151                 return;
152
153         for (i = 0; i < (1ul << order); i++) {
154                 struct page_table_check *ptc = get_page_table_check(page_ext);
155
156                 BUG_ON(atomic_read(&ptc->anon_map_count));
157                 BUG_ON(atomic_read(&ptc->file_map_count));
158                 page_ext = page_ext_next(page_ext);
159         }
160         page_ext_put(page_ext);
161 }
162
163 void __page_table_check_pte_clear(struct mm_struct *mm, pte_t pte)
164 {
165         if (&init_mm == mm)
166                 return;
167
168         if (pte_user_accessible_page(pte)) {
169                 page_table_check_clear(pte_pfn(pte), PAGE_SIZE >> PAGE_SHIFT);
170         }
171 }
172 EXPORT_SYMBOL(__page_table_check_pte_clear);
173
174 void __page_table_check_pmd_clear(struct mm_struct *mm, pmd_t pmd)
175 {
176         if (&init_mm == mm)
177                 return;
178
179         if (pmd_user_accessible_page(pmd)) {
180                 page_table_check_clear(pmd_pfn(pmd), PMD_SIZE >> PAGE_SHIFT);
181         }
182 }
183 EXPORT_SYMBOL(__page_table_check_pmd_clear);
184
185 void __page_table_check_pud_clear(struct mm_struct *mm, pud_t pud)
186 {
187         if (&init_mm == mm)
188                 return;
189
190         if (pud_user_accessible_page(pud)) {
191                 page_table_check_clear(pud_pfn(pud), PUD_SIZE >> PAGE_SHIFT);
192         }
193 }
194 EXPORT_SYMBOL(__page_table_check_pud_clear);
195
196 /* Whether the swap entry cached writable information */
197 static inline bool swap_cached_writable(swp_entry_t entry)
198 {
199         return is_writable_device_exclusive_entry(entry) ||
200             is_writable_device_private_entry(entry) ||
201             is_writable_migration_entry(entry);
202 }
203
204 static inline void page_table_check_pte_flags(pte_t pte)
205 {
206         if (pte_present(pte) && pte_uffd_wp(pte))
207                 WARN_ON_ONCE(pte_write(pte));
208         else if (is_swap_pte(pte) && pte_swp_uffd_wp(pte))
209                 WARN_ON_ONCE(swap_cached_writable(pte_to_swp_entry(pte)));
210 }
211
212 void __page_table_check_ptes_set(struct mm_struct *mm, pte_t *ptep, pte_t pte,
213                 unsigned int nr)
214 {
215         unsigned int i;
216
217         if (&init_mm == mm)
218                 return;
219
220         page_table_check_pte_flags(pte);
221
222         for (i = 0; i < nr; i++)
223                 __page_table_check_pte_clear(mm, ptep_get(ptep + i));
224         if (pte_user_accessible_page(pte))
225                 page_table_check_set(pte_pfn(pte), nr, pte_write(pte));
226 }
227 EXPORT_SYMBOL(__page_table_check_ptes_set);
228
229 static inline void page_table_check_pmd_flags(pmd_t pmd)
230 {
231         if (pmd_present(pmd) && pmd_uffd_wp(pmd))
232                 WARN_ON_ONCE(pmd_write(pmd));
233         else if (is_swap_pmd(pmd) && pmd_swp_uffd_wp(pmd))
234                 WARN_ON_ONCE(swap_cached_writable(pmd_to_swp_entry(pmd)));
235 }
236
237 void __page_table_check_pmd_set(struct mm_struct *mm, pmd_t *pmdp, pmd_t pmd)
238 {
239         if (&init_mm == mm)
240                 return;
241
242         page_table_check_pmd_flags(pmd);
243
244         __page_table_check_pmd_clear(mm, *pmdp);
245         if (pmd_user_accessible_page(pmd)) {
246                 page_table_check_set(pmd_pfn(pmd), PMD_SIZE >> PAGE_SHIFT,
247                                      pmd_write(pmd));
248         }
249 }
250 EXPORT_SYMBOL(__page_table_check_pmd_set);
251
252 void __page_table_check_pud_set(struct mm_struct *mm, pud_t *pudp, pud_t pud)
253 {
254         if (&init_mm == mm)
255                 return;
256
257         __page_table_check_pud_clear(mm, *pudp);
258         if (pud_user_accessible_page(pud)) {
259                 page_table_check_set(pud_pfn(pud), PUD_SIZE >> PAGE_SHIFT,
260                                      pud_write(pud));
261         }
262 }
263 EXPORT_SYMBOL(__page_table_check_pud_set);
264
265 void __page_table_check_pte_clear_range(struct mm_struct *mm,
266                                         unsigned long addr,
267                                         pmd_t pmd)
268 {
269         if (&init_mm == mm)
270                 return;
271
272         if (!pmd_bad(pmd) && !pmd_leaf(pmd)) {
273                 pte_t *ptep = pte_offset_map(&pmd, addr);
274                 unsigned long i;
275
276                 if (WARN_ON(!ptep))
277                         return;
278                 for (i = 0; i < PTRS_PER_PTE; i++) {
279                         __page_table_check_pte_clear(mm, ptep_get(ptep));
280                         addr += PAGE_SIZE;
281                         ptep++;
282                 }
283                 pte_unmap(ptep - PTRS_PER_PTE);
284         }
285 }