Commit | Line | Data |
---|---|---|
b3383974 OA |
1 | // SPDX-License-Identifier: GPL-2.0 OR MIT |
2 | ||
3 | /* | |
4 | * Xen frontend/backend page directory based shared buffer | |
5 | * helper module. | |
6 | * | |
7 | * Copyright (C) 2018 EPAM Systems Inc. | |
8 | * | |
9 | * Author: Oleksandr Andrushchenko <oleksandr_andrushchenko@epam.com> | |
10 | */ | |
11 | ||
12 | #include <linux/module.h> | |
13 | #include <linux/errno.h> | |
14 | #include <linux/mm.h> | |
15 | ||
16 | #include <asm/xen/hypervisor.h> | |
17 | #include <xen/balloon.h> | |
18 | #include <xen/xen.h> | |
19 | #include <xen/xenbus.h> | |
20 | #include <xen/interface/io/ring.h> | |
21 | ||
22 | #include <xen/xen-front-pgdir-shbuf.h> | |
23 | ||
24 | #ifndef GRANT_INVALID_REF | |
25 | /* | |
26 | * FIXME: usage of grant reference 0 as invalid grant reference: | |
27 | * grant reference 0 is valid, but never exposed to a PV driver, | |
28 | * because of the fact it is already in use/reserved by the PV console. | |
29 | */ | |
30 | #define GRANT_INVALID_REF 0 | |
31 | #endif | |
32 | ||
33 | /** | |
34 | * This structure represents the structure of a shared page | |
35 | * that contains grant references to the pages of the shared | |
36 | * buffer. This structure is common to many Xen para-virtualized | |
37 | * protocols at include/xen/interface/io/ | |
38 | */ | |
39 | struct xen_page_directory { | |
40 | grant_ref_t gref_dir_next_page; | |
41 | grant_ref_t gref[1]; /* Variable length */ | |
42 | }; | |
43 | ||
44 | /** | |
45 | * Shared buffer ops which are differently implemented | |
46 | * depending on the allocation mode, e.g. if the buffer | |
47 | * is allocated by the corresponding backend or frontend. | |
48 | * Some of the operations. | |
49 | */ | |
50 | struct xen_front_pgdir_shbuf_ops { | |
51 | /* | |
52 | * Calculate number of grefs required to handle this buffer, | |
53 | * e.g. if grefs are required for page directory only or the buffer | |
54 | * pages as well. | |
55 | */ | |
56 | void (*calc_num_grefs)(struct xen_front_pgdir_shbuf *buf); | |
57 | ||
58 | /* Fill page directory according to para-virtual display protocol. */ | |
59 | void (*fill_page_dir)(struct xen_front_pgdir_shbuf *buf); | |
60 | ||
61 | /* Claim grant references for the pages of the buffer. */ | |
62 | int (*grant_refs_for_buffer)(struct xen_front_pgdir_shbuf *buf, | |
63 | grant_ref_t *priv_gref_head, int gref_idx); | |
64 | ||
65 | /* Map grant references of the buffer. */ | |
66 | int (*map)(struct xen_front_pgdir_shbuf *buf); | |
67 | ||
68 | /* Unmap grant references of the buffer. */ | |
69 | int (*unmap)(struct xen_front_pgdir_shbuf *buf); | |
70 | }; | |
71 | ||
72 | /** | |
73 | * Get granted reference to the very first page of the | |
74 | * page directory. Usually this is passed to the backend, | |
75 | * so it can find/fill the grant references to the buffer's | |
76 | * pages. | |
77 | * | |
78 | * \param buf shared buffer which page directory is of interest. | |
79 | * \return granted reference to the very first page of the | |
80 | * page directory. | |
81 | */ | |
82 | grant_ref_t | |
83 | xen_front_pgdir_shbuf_get_dir_start(struct xen_front_pgdir_shbuf *buf) | |
84 | { | |
85 | if (!buf->grefs) | |
86 | return GRANT_INVALID_REF; | |
87 | ||
88 | return buf->grefs[0]; | |
89 | } | |
90 | EXPORT_SYMBOL_GPL(xen_front_pgdir_shbuf_get_dir_start); | |
91 | ||
92 | /** | |
93 | * Map granted references of the shared buffer. | |
94 | * | |
95 | * Depending on the shared buffer mode of allocation | |
96 | * (be_alloc flag) this can either do nothing (for buffers | |
97 | * shared by the frontend itself) or map the provided granted | |
98 | * references onto the backing storage (buf->pages). | |
99 | * | |
100 | * \param buf shared buffer which grants to be maped. | |
101 | * \return zero on success or a negative number on failure. | |
102 | */ | |
103 | int xen_front_pgdir_shbuf_map(struct xen_front_pgdir_shbuf *buf) | |
104 | { | |
105 | if (buf->ops && buf->ops->map) | |
106 | return buf->ops->map(buf); | |
107 | ||
108 | /* No need to map own grant references. */ | |
109 | return 0; | |
110 | } | |
111 | EXPORT_SYMBOL_GPL(xen_front_pgdir_shbuf_map); | |
112 | ||
113 | /** | |
114 | * Unmap granted references of the shared buffer. | |
115 | * | |
116 | * Depending on the shared buffer mode of allocation | |
117 | * (be_alloc flag) this can either do nothing (for buffers | |
118 | * shared by the frontend itself) or unmap the provided granted | |
119 | * references. | |
120 | * | |
121 | * \param buf shared buffer which grants to be unmaped. | |
122 | * \return zero on success or a negative number on failure. | |
123 | */ | |
124 | int xen_front_pgdir_shbuf_unmap(struct xen_front_pgdir_shbuf *buf) | |
125 | { | |
126 | if (buf->ops && buf->ops->unmap) | |
127 | return buf->ops->unmap(buf); | |
128 | ||
129 | /* No need to unmap own grant references. */ | |
130 | return 0; | |
131 | } | |
132 | EXPORT_SYMBOL_GPL(xen_front_pgdir_shbuf_unmap); | |
133 | ||
134 | /** | |
135 | * Free all the resources of the shared buffer. | |
136 | * | |
137 | * \param buf shared buffer which resources to be freed. | |
138 | */ | |
139 | void xen_front_pgdir_shbuf_free(struct xen_front_pgdir_shbuf *buf) | |
140 | { | |
141 | if (buf->grefs) { | |
142 | int i; | |
143 | ||
144 | for (i = 0; i < buf->num_grefs; i++) | |
145 | if (buf->grefs[i] != GRANT_INVALID_REF) | |
146 | gnttab_end_foreign_access(buf->grefs[i], | |
147 | 0, 0UL); | |
148 | } | |
149 | kfree(buf->grefs); | |
150 | kfree(buf->directory); | |
151 | } | |
152 | EXPORT_SYMBOL_GPL(xen_front_pgdir_shbuf_free); | |
153 | ||
154 | /* | |
155 | * Number of grefs a page can hold with respect to the | |
156 | * struct xen_page_directory header. | |
157 | */ | |
158 | #define XEN_NUM_GREFS_PER_PAGE ((PAGE_SIZE - \ | |
159 | offsetof(struct xen_page_directory, \ | |
160 | gref)) / sizeof(grant_ref_t)) | |
161 | ||
162 | /** | |
163 | * Get the number of pages the page directory consumes itself. | |
164 | * | |
165 | * \param buf shared buffer. | |
166 | */ | |
167 | static int get_num_pages_dir(struct xen_front_pgdir_shbuf *buf) | |
168 | { | |
169 | return DIV_ROUND_UP(buf->num_pages, XEN_NUM_GREFS_PER_PAGE); | |
170 | } | |
171 | ||
172 | /** | |
173 | * Calculate the number of grant references needed to share the buffer | |
174 | * and its pages when backend allocates the buffer. | |
175 | * | |
176 | * \param buf shared buffer. | |
177 | */ | |
178 | static void backend_calc_num_grefs(struct xen_front_pgdir_shbuf *buf) | |
179 | { | |
180 | /* Only for pages the page directory consumes itself. */ | |
181 | buf->num_grefs = get_num_pages_dir(buf); | |
182 | } | |
183 | ||
184 | /** | |
185 | * Calculate the number of grant references needed to share the buffer | |
186 | * and its pages when frontend allocates the buffer. | |
187 | * | |
188 | * \param buf shared buffer. | |
189 | */ | |
190 | static void guest_calc_num_grefs(struct xen_front_pgdir_shbuf *buf) | |
191 | { | |
192 | /* | |
193 | * Number of pages the page directory consumes itself | |
194 | * plus grefs for the buffer pages. | |
195 | */ | |
196 | buf->num_grefs = get_num_pages_dir(buf) + buf->num_pages; | |
197 | } | |
198 | ||
199 | #define xen_page_to_vaddr(page) \ | |
200 | ((uintptr_t)pfn_to_kaddr(page_to_xen_pfn(page))) | |
201 | ||
202 | /** | |
203 | * Unmap the buffer previously mapped with grant references | |
204 | * provided by the backend. | |
205 | * | |
206 | * \param buf shared buffer. | |
207 | * \return zero on success or a negative number on failure. | |
208 | */ | |
209 | static int backend_unmap(struct xen_front_pgdir_shbuf *buf) | |
210 | { | |
211 | struct gnttab_unmap_grant_ref *unmap_ops; | |
212 | int i, ret; | |
213 | ||
214 | if (!buf->pages || !buf->backend_map_handles || !buf->grefs) | |
215 | return 0; | |
216 | ||
217 | unmap_ops = kcalloc(buf->num_pages, sizeof(*unmap_ops), | |
218 | GFP_KERNEL); | |
219 | if (!unmap_ops) | |
220 | return -ENOMEM; | |
221 | ||
222 | for (i = 0; i < buf->num_pages; i++) { | |
223 | phys_addr_t addr; | |
224 | ||
225 | addr = xen_page_to_vaddr(buf->pages[i]); | |
226 | gnttab_set_unmap_op(&unmap_ops[i], addr, GNTMAP_host_map, | |
227 | buf->backend_map_handles[i]); | |
228 | } | |
229 | ||
230 | ret = gnttab_unmap_refs(unmap_ops, NULL, buf->pages, | |
231 | buf->num_pages); | |
232 | ||
233 | for (i = 0; i < buf->num_pages; i++) { | |
234 | if (unlikely(unmap_ops[i].status != GNTST_okay)) | |
235 | dev_err(&buf->xb_dev->dev, | |
236 | "Failed to unmap page %d: %d\n", | |
237 | i, unmap_ops[i].status); | |
238 | } | |
239 | ||
240 | if (ret) | |
241 | dev_err(&buf->xb_dev->dev, | |
242 | "Failed to unmap grant references, ret %d", ret); | |
243 | ||
244 | kfree(unmap_ops); | |
245 | kfree(buf->backend_map_handles); | |
246 | buf->backend_map_handles = NULL; | |
247 | return ret; | |
248 | } | |
249 | ||
250 | /** | |
251 | * Map the buffer with grant references provided by the backend. | |
252 | * | |
253 | * \param buf shared buffer. | |
254 | * \return zero on success or a negative number on failure. | |
255 | */ | |
256 | static int backend_map(struct xen_front_pgdir_shbuf *buf) | |
257 | { | |
258 | struct gnttab_map_grant_ref *map_ops = NULL; | |
259 | unsigned char *ptr; | |
260 | int ret, cur_gref, cur_dir_page, cur_page, grefs_left; | |
261 | ||
262 | map_ops = kcalloc(buf->num_pages, sizeof(*map_ops), GFP_KERNEL); | |
263 | if (!map_ops) | |
264 | return -ENOMEM; | |
265 | ||
266 | buf->backend_map_handles = kcalloc(buf->num_pages, | |
267 | sizeof(*buf->backend_map_handles), | |
268 | GFP_KERNEL); | |
269 | if (!buf->backend_map_handles) { | |
270 | kfree(map_ops); | |
271 | return -ENOMEM; | |
272 | } | |
273 | ||
274 | /* | |
275 | * Read page directory to get grefs from the backend: for external | |
276 | * buffer we only allocate buf->grefs for the page directory, | |
277 | * so buf->num_grefs has number of pages in the page directory itself. | |
278 | */ | |
279 | ptr = buf->directory; | |
280 | grefs_left = buf->num_pages; | |
281 | cur_page = 0; | |
282 | for (cur_dir_page = 0; cur_dir_page < buf->num_grefs; cur_dir_page++) { | |
283 | struct xen_page_directory *page_dir = | |
284 | (struct xen_page_directory *)ptr; | |
285 | int to_copy = XEN_NUM_GREFS_PER_PAGE; | |
286 | ||
287 | if (to_copy > grefs_left) | |
288 | to_copy = grefs_left; | |
289 | ||
290 | for (cur_gref = 0; cur_gref < to_copy; cur_gref++) { | |
291 | phys_addr_t addr; | |
292 | ||
293 | addr = xen_page_to_vaddr(buf->pages[cur_page]); | |
294 | gnttab_set_map_op(&map_ops[cur_page], addr, | |
295 | GNTMAP_host_map, | |
296 | page_dir->gref[cur_gref], | |
297 | buf->xb_dev->otherend_id); | |
298 | cur_page++; | |
299 | } | |
300 | ||
301 | grefs_left -= to_copy; | |
302 | ptr += PAGE_SIZE; | |
303 | } | |
304 | ret = gnttab_map_refs(map_ops, NULL, buf->pages, buf->num_pages); | |
305 | ||
306 | /* Save handles even if error, so we can unmap. */ | |
307 | for (cur_page = 0; cur_page < buf->num_pages; cur_page++) { | |
308 | buf->backend_map_handles[cur_page] = map_ops[cur_page].handle; | |
309 | if (unlikely(map_ops[cur_page].status != GNTST_okay)) | |
310 | dev_err(&buf->xb_dev->dev, | |
311 | "Failed to map page %d: %d\n", | |
312 | cur_page, map_ops[cur_page].status); | |
313 | } | |
314 | ||
315 | if (ret) { | |
316 | dev_err(&buf->xb_dev->dev, | |
317 | "Failed to map grant references, ret %d", ret); | |
318 | backend_unmap(buf); | |
319 | } | |
320 | ||
321 | kfree(map_ops); | |
322 | return ret; | |
323 | } | |
324 | ||
325 | /** | |
326 | * Fill page directory with grant references to the pages of the | |
327 | * page directory itself. | |
328 | * | |
329 | * The grant references to the buffer pages are provided by the | |
330 | * backend in this case. | |
331 | * | |
332 | * \param buf shared buffer. | |
333 | */ | |
334 | static void backend_fill_page_dir(struct xen_front_pgdir_shbuf *buf) | |
335 | { | |
336 | struct xen_page_directory *page_dir; | |
337 | unsigned char *ptr; | |
338 | int i, num_pages_dir; | |
339 | ||
340 | ptr = buf->directory; | |
341 | num_pages_dir = get_num_pages_dir(buf); | |
342 | ||
343 | /* Fill only grefs for the page directory itself. */ | |
344 | for (i = 0; i < num_pages_dir - 1; i++) { | |
345 | page_dir = (struct xen_page_directory *)ptr; | |
346 | ||
347 | page_dir->gref_dir_next_page = buf->grefs[i + 1]; | |
348 | ptr += PAGE_SIZE; | |
349 | } | |
350 | /* Last page must say there is no more pages. */ | |
351 | page_dir = (struct xen_page_directory *)ptr; | |
352 | page_dir->gref_dir_next_page = GRANT_INVALID_REF; | |
353 | } | |
354 | ||
355 | /** | |
356 | * Fill page directory with grant references to the pages of the | |
357 | * page directory and the buffer we share with the backend. | |
358 | * | |
359 | * \param buf shared buffer. | |
360 | */ | |
361 | static void guest_fill_page_dir(struct xen_front_pgdir_shbuf *buf) | |
362 | { | |
363 | unsigned char *ptr; | |
364 | int cur_gref, grefs_left, to_copy, i, num_pages_dir; | |
365 | ||
366 | ptr = buf->directory; | |
367 | num_pages_dir = get_num_pages_dir(buf); | |
368 | ||
369 | /* | |
370 | * While copying, skip grefs at start, they are for pages | |
371 | * granted for the page directory itself. | |
372 | */ | |
373 | cur_gref = num_pages_dir; | |
374 | grefs_left = buf->num_pages; | |
375 | for (i = 0; i < num_pages_dir; i++) { | |
376 | struct xen_page_directory *page_dir = | |
377 | (struct xen_page_directory *)ptr; | |
378 | ||
379 | if (grefs_left <= XEN_NUM_GREFS_PER_PAGE) { | |
380 | to_copy = grefs_left; | |
381 | page_dir->gref_dir_next_page = GRANT_INVALID_REF; | |
382 | } else { | |
383 | to_copy = XEN_NUM_GREFS_PER_PAGE; | |
384 | page_dir->gref_dir_next_page = buf->grefs[i + 1]; | |
385 | } | |
386 | memcpy(&page_dir->gref, &buf->grefs[cur_gref], | |
387 | to_copy * sizeof(grant_ref_t)); | |
388 | ptr += PAGE_SIZE; | |
389 | grefs_left -= to_copy; | |
390 | cur_gref += to_copy; | |
391 | } | |
392 | } | |
393 | ||
394 | /** | |
395 | * Grant references to the frontend's buffer pages. | |
396 | * | |
397 | * These will be shared with the backend, so it can | |
398 | * access the buffer's data. | |
399 | * | |
400 | * \param buf shared buffer. | |
401 | * \return zero on success or a negative number on failure. | |
402 | */ | |
403 | static int guest_grant_refs_for_buffer(struct xen_front_pgdir_shbuf *buf, | |
404 | grant_ref_t *priv_gref_head, | |
405 | int gref_idx) | |
406 | { | |
407 | int i, cur_ref, otherend_id; | |
408 | ||
409 | otherend_id = buf->xb_dev->otherend_id; | |
410 | for (i = 0; i < buf->num_pages; i++) { | |
411 | cur_ref = gnttab_claim_grant_reference(priv_gref_head); | |
412 | if (cur_ref < 0) | |
413 | return cur_ref; | |
414 | ||
415 | gnttab_grant_foreign_access_ref(cur_ref, otherend_id, | |
416 | xen_page_to_gfn(buf->pages[i]), | |
417 | 0); | |
418 | buf->grefs[gref_idx++] = cur_ref; | |
419 | } | |
420 | return 0; | |
421 | } | |
422 | ||
423 | /** | |
424 | * Grant all the references needed to share the buffer. | |
425 | * | |
426 | * Grant references to the page directory pages and, if | |
427 | * needed, also to the pages of the shared buffer data. | |
428 | * | |
429 | * \param buf shared buffer. | |
430 | * \return zero on success or a negative number on failure. | |
431 | */ | |
432 | static int grant_references(struct xen_front_pgdir_shbuf *buf) | |
433 | { | |
434 | grant_ref_t priv_gref_head; | |
435 | int ret, i, j, cur_ref; | |
436 | int otherend_id, num_pages_dir; | |
437 | ||
438 | ret = gnttab_alloc_grant_references(buf->num_grefs, &priv_gref_head); | |
439 | if (ret < 0) { | |
440 | dev_err(&buf->xb_dev->dev, | |
441 | "Cannot allocate grant references\n"); | |
442 | return ret; | |
443 | } | |
444 | ||
445 | otherend_id = buf->xb_dev->otherend_id; | |
446 | j = 0; | |
447 | num_pages_dir = get_num_pages_dir(buf); | |
448 | for (i = 0; i < num_pages_dir; i++) { | |
449 | unsigned long frame; | |
450 | ||
451 | cur_ref = gnttab_claim_grant_reference(&priv_gref_head); | |
452 | if (cur_ref < 0) | |
453 | return cur_ref; | |
454 | ||
455 | frame = xen_page_to_gfn(virt_to_page(buf->directory + | |
456 | PAGE_SIZE * i)); | |
457 | gnttab_grant_foreign_access_ref(cur_ref, otherend_id, frame, 0); | |
458 | buf->grefs[j++] = cur_ref; | |
459 | } | |
460 | ||
461 | if (buf->ops->grant_refs_for_buffer) { | |
462 | ret = buf->ops->grant_refs_for_buffer(buf, &priv_gref_head, j); | |
463 | if (ret) | |
464 | return ret; | |
465 | } | |
466 | ||
467 | gnttab_free_grant_references(priv_gref_head); | |
468 | return 0; | |
469 | } | |
470 | ||
471 | /** | |
472 | * Allocate all required structures to mange shared buffer. | |
473 | * | |
474 | * \param buf shared buffer. | |
475 | * \return zero on success or a negative number on failure. | |
476 | */ | |
477 | static int alloc_storage(struct xen_front_pgdir_shbuf *buf) | |
478 | { | |
479 | buf->grefs = kcalloc(buf->num_grefs, sizeof(*buf->grefs), GFP_KERNEL); | |
480 | if (!buf->grefs) | |
481 | return -ENOMEM; | |
482 | ||
483 | buf->directory = kcalloc(get_num_pages_dir(buf), PAGE_SIZE, GFP_KERNEL); | |
484 | if (!buf->directory) | |
485 | return -ENOMEM; | |
486 | ||
487 | return 0; | |
488 | } | |
489 | ||
490 | /* | |
491 | * For backend allocated buffers we don't need grant_refs_for_buffer | |
492 | * as those grant references are allocated at backend side. | |
493 | */ | |
494 | static const struct xen_front_pgdir_shbuf_ops backend_ops = { | |
495 | .calc_num_grefs = backend_calc_num_grefs, | |
496 | .fill_page_dir = backend_fill_page_dir, | |
497 | .map = backend_map, | |
498 | .unmap = backend_unmap | |
499 | }; | |
500 | ||
501 | /* | |
502 | * For locally granted references we do not need to map/unmap | |
503 | * the references. | |
504 | */ | |
505 | static const struct xen_front_pgdir_shbuf_ops local_ops = { | |
506 | .calc_num_grefs = guest_calc_num_grefs, | |
507 | .fill_page_dir = guest_fill_page_dir, | |
508 | .grant_refs_for_buffer = guest_grant_refs_for_buffer, | |
509 | }; | |
510 | ||
511 | /** | |
512 | * Allocate a new instance of a shared buffer. | |
513 | * | |
514 | * \param cfg configuration to be used while allocating a new shared buffer. | |
515 | * \return zero on success or a negative number on failure. | |
516 | */ | |
517 | int xen_front_pgdir_shbuf_alloc(struct xen_front_pgdir_shbuf_cfg *cfg) | |
518 | { | |
519 | struct xen_front_pgdir_shbuf *buf = cfg->pgdir; | |
520 | int ret; | |
521 | ||
522 | if (cfg->be_alloc) | |
523 | buf->ops = &backend_ops; | |
524 | else | |
525 | buf->ops = &local_ops; | |
526 | buf->xb_dev = cfg->xb_dev; | |
527 | buf->num_pages = cfg->num_pages; | |
528 | buf->pages = cfg->pages; | |
529 | ||
530 | buf->ops->calc_num_grefs(buf); | |
531 | ||
532 | ret = alloc_storage(buf); | |
533 | if (ret) | |
534 | goto fail; | |
535 | ||
536 | ret = grant_references(buf); | |
537 | if (ret) | |
538 | goto fail; | |
539 | ||
540 | buf->ops->fill_page_dir(buf); | |
541 | ||
542 | return 0; | |
543 | ||
544 | fail: | |
545 | xen_front_pgdir_shbuf_free(buf); | |
546 | return ret; | |
547 | } | |
548 | EXPORT_SYMBOL_GPL(xen_front_pgdir_shbuf_alloc); | |
549 | ||
550 | MODULE_DESCRIPTION("Xen frontend/backend page directory based " | |
551 | "shared buffer handling"); | |
552 | MODULE_AUTHOR("Oleksandr Andrushchenko"); | |
553 | MODULE_LICENSE("GPL"); |