Commit | Line | Data |
---|---|---|
9e2369c0 RPM |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | #include <linux/errno.h> | |
3 | #include <linux/gfp.h> | |
4 | #include <linux/kernel.h> | |
5 | #include <linux/mm.h> | |
6 | #include <linux/memremap.h> | |
7 | #include <linux/slab.h> | |
8 | ||
9 | #include <asm/page.h> | |
10 | ||
d1a928ea | 11 | #include <xen/balloon.h> |
9e2369c0 RPM |
12 | #include <xen/page.h> |
13 | #include <xen/xen.h> | |
14 | ||
15 | static DEFINE_MUTEX(list_lock); | |
ee32f323 | 16 | static struct page *page_list; |
9e2369c0 RPM |
17 | static unsigned int list_count; |
18 | ||
d1a928ea OT |
19 | static struct resource *target_resource; |
20 | ||
21 | /* | |
22 | * If arch is not happy with system "iomem_resource" being used for | |
23 | * the region allocation it can provide it's own view by creating specific | |
24 | * Xen resource with unused regions of guest physical address space provided | |
25 | * by the hypervisor. | |
26 | */ | |
27 | int __weak __init arch_xen_unpopulated_init(struct resource **res) | |
28 | { | |
29 | *res = &iomem_resource; | |
30 | ||
31 | return 0; | |
32 | } | |
33 | ||
9e2369c0 RPM |
34 | static int fill_list(unsigned int nr_pages) |
35 | { | |
36 | struct dev_pagemap *pgmap; | |
d1a928ea | 37 | struct resource *res, *tmp_res = NULL; |
9e2369c0 RPM |
38 | void *vaddr; |
39 | unsigned int i, alloc_pages = round_up(nr_pages, PAGES_PER_SECTION); | |
d1a928ea OT |
40 | struct range mhp_range; |
41 | int ret; | |
a4574f63 DW |
42 | |
43 | res = kzalloc(sizeof(*res), GFP_KERNEL); | |
44 | if (!res) | |
45 | return -ENOMEM; | |
9e2369c0 | 46 | |
a4574f63 DW |
47 | res->name = "Xen scratch"; |
48 | res->flags = IORESOURCE_MEM | IORESOURCE_BUSY; | |
9e2369c0 | 49 | |
d1a928ea OT |
50 | mhp_range = mhp_get_pluggable_range(true); |
51 | ||
52 | ret = allocate_resource(target_resource, res, | |
53 | alloc_pages * PAGE_SIZE, mhp_range.start, mhp_range.end, | |
9e2369c0 RPM |
54 | PAGES_PER_SECTION * PAGE_SIZE, NULL, NULL); |
55 | if (ret < 0) { | |
56 | pr_err("Cannot allocate new IOMEM resource\n"); | |
a4574f63 | 57 | goto err_resource; |
9e2369c0 RPM |
58 | } |
59 | ||
d1a928ea OT |
60 | /* |
61 | * Reserve the region previously allocated from Xen resource to avoid | |
62 | * re-using it by someone else. | |
63 | */ | |
64 | if (target_resource != &iomem_resource) { | |
65 | tmp_res = kzalloc(sizeof(*tmp_res), GFP_KERNEL); | |
66 | if (!tmp_res) { | |
67 | ret = -ENOMEM; | |
68 | goto err_insert; | |
69 | } | |
70 | ||
71 | tmp_res->name = res->name; | |
72 | tmp_res->start = res->start; | |
73 | tmp_res->end = res->end; | |
74 | tmp_res->flags = res->flags; | |
75 | ||
76 | ret = request_resource(&iomem_resource, tmp_res); | |
77 | if (ret < 0) { | |
78 | pr_err("Cannot request resource %pR (%d)\n", tmp_res, ret); | |
79 | kfree(tmp_res); | |
80 | goto err_insert; | |
81 | } | |
82 | } | |
83 | ||
3a250629 | 84 | pgmap = kzalloc(sizeof(*pgmap), GFP_KERNEL); |
dbc03e81 ZL |
85 | if (!pgmap) { |
86 | ret = -ENOMEM; | |
3a250629 | 87 | goto err_pgmap; |
dbc03e81 | 88 | } |
3a250629 DW |
89 | |
90 | pgmap->type = MEMORY_DEVICE_GENERIC; | |
a4574f63 DW |
91 | pgmap->range = (struct range) { |
92 | .start = res->start, | |
93 | .end = res->end, | |
94 | }; | |
b7b3c01b | 95 | pgmap->nr_range = 1; |
a4574f63 DW |
96 | pgmap->owner = res; |
97 | ||
9e2369c0 RPM |
98 | #ifdef CONFIG_XEN_HAVE_PVMMU |
99 | /* | |
100 | * memremap will build page tables for the new memory so | |
101 | * the p2m must contain invalid entries so the correct | |
102 | * non-present PTEs will be written. | |
103 | * | |
104 | * If a failure occurs, the original (identity) p2m entries | |
105 | * are not restored since this region is now known not to | |
106 | * conflict with any devices. | |
107 | */ | |
108 | if (!xen_feature(XENFEAT_auto_translated_physmap)) { | |
a4574f63 | 109 | xen_pfn_t pfn = PFN_DOWN(res->start); |
9e2369c0 RPM |
110 | |
111 | for (i = 0; i < alloc_pages; i++) { | |
112 | if (!set_phys_to_machine(pfn + i, INVALID_P2M_ENTRY)) { | |
113 | pr_warn("set_phys_to_machine() failed, no memory added\n"); | |
a4574f63 DW |
114 | ret = -ENOMEM; |
115 | goto err_memremap; | |
9e2369c0 RPM |
116 | } |
117 | } | |
118 | } | |
119 | #endif | |
120 | ||
121 | vaddr = memremap_pages(pgmap, NUMA_NO_NODE); | |
122 | if (IS_ERR(vaddr)) { | |
123 | pr_err("Cannot remap memory range\n"); | |
a4574f63 DW |
124 | ret = PTR_ERR(vaddr); |
125 | goto err_memremap; | |
9e2369c0 RPM |
126 | } |
127 | ||
128 | for (i = 0; i < alloc_pages; i++) { | |
129 | struct page *pg = virt_to_page(vaddr + PAGE_SIZE * i); | |
130 | ||
ee32f323 JG |
131 | pg->zone_device_data = page_list; |
132 | page_list = pg; | |
9e2369c0 RPM |
133 | list_count++; |
134 | } | |
135 | ||
136 | return 0; | |
a4574f63 DW |
137 | |
138 | err_memremap: | |
a4574f63 DW |
139 | kfree(pgmap); |
140 | err_pgmap: | |
d1a928ea OT |
141 | if (tmp_res) { |
142 | release_resource(tmp_res); | |
143 | kfree(tmp_res); | |
144 | } | |
145 | err_insert: | |
3a250629 DW |
146 | release_resource(res); |
147 | err_resource: | |
a4574f63 DW |
148 | kfree(res); |
149 | return ret; | |
9e2369c0 RPM |
150 | } |
151 | ||
152 | /** | |
153 | * xen_alloc_unpopulated_pages - alloc unpopulated pages | |
154 | * @nr_pages: Number of pages | |
155 | * @pages: pages returned | |
156 | * @return 0 on success, error otherwise | |
157 | */ | |
158 | int xen_alloc_unpopulated_pages(unsigned int nr_pages, struct page **pages) | |
159 | { | |
160 | unsigned int i; | |
161 | int ret = 0; | |
162 | ||
d1a928ea OT |
163 | /* |
164 | * Fallback to default behavior if we do not have any suitable resource | |
165 | * to allocate required region from and as the result we won't be able to | |
166 | * construct pages. | |
167 | */ | |
168 | if (!target_resource) | |
169 | return xen_alloc_ballooned_pages(nr_pages, pages); | |
170 | ||
9e2369c0 RPM |
171 | mutex_lock(&list_lock); |
172 | if (list_count < nr_pages) { | |
173 | ret = fill_list(nr_pages - list_count); | |
174 | if (ret) | |
175 | goto out; | |
176 | } | |
177 | ||
178 | for (i = 0; i < nr_pages; i++) { | |
ee32f323 | 179 | struct page *pg = page_list; |
9e2369c0 RPM |
180 | |
181 | BUG_ON(!pg); | |
ee32f323 | 182 | page_list = pg->zone_device_data; |
9e2369c0 RPM |
183 | list_count--; |
184 | pages[i] = pg; | |
185 | ||
186 | #ifdef CONFIG_XEN_HAVE_PVMMU | |
187 | if (!xen_feature(XENFEAT_auto_translated_physmap)) { | |
188 | ret = xen_alloc_p2m_entry(page_to_pfn(pg)); | |
189 | if (ret < 0) { | |
190 | unsigned int j; | |
191 | ||
192 | for (j = 0; j <= i; j++) { | |
ee32f323 JG |
193 | pages[j]->zone_device_data = page_list; |
194 | page_list = pages[j]; | |
9e2369c0 RPM |
195 | list_count++; |
196 | } | |
197 | goto out; | |
198 | } | |
199 | } | |
200 | #endif | |
201 | } | |
202 | ||
203 | out: | |
204 | mutex_unlock(&list_lock); | |
205 | return ret; | |
206 | } | |
207 | EXPORT_SYMBOL(xen_alloc_unpopulated_pages); | |
208 | ||
209 | /** | |
210 | * xen_free_unpopulated_pages - return unpopulated pages | |
211 | * @nr_pages: Number of pages | |
212 | * @pages: pages to return | |
213 | */ | |
214 | void xen_free_unpopulated_pages(unsigned int nr_pages, struct page **pages) | |
215 | { | |
216 | unsigned int i; | |
217 | ||
d1a928ea OT |
218 | if (!target_resource) { |
219 | xen_free_ballooned_pages(nr_pages, pages); | |
220 | return; | |
221 | } | |
222 | ||
9e2369c0 RPM |
223 | mutex_lock(&list_lock); |
224 | for (i = 0; i < nr_pages; i++) { | |
ee32f323 JG |
225 | pages[i]->zone_device_data = page_list; |
226 | page_list = pages[i]; | |
9e2369c0 RPM |
227 | list_count++; |
228 | } | |
229 | mutex_unlock(&list_lock); | |
230 | } | |
231 | EXPORT_SYMBOL(xen_free_unpopulated_pages); | |
232 | ||
d1a928ea OT |
233 | static int __init unpopulated_init(void) |
234 | { | |
235 | int ret; | |
236 | ||
237 | if (!xen_domain()) | |
238 | return -ENODEV; | |
239 | ||
240 | ret = arch_xen_unpopulated_init(&target_resource); | |
241 | if (ret) { | |
242 | pr_err("xen:unpopulated: Cannot initialize target resource\n"); | |
243 | target_resource = NULL; | |
244 | } | |
245 | ||
246 | return ret; | |
247 | } | |
248 | early_initcall(unpopulated_init); |