Commit | Line | Data |
---|---|---|
52488734 GX |
1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* | |
3 | * Copyright (C) Gao Xiang <xiang@kernel.org> | |
4 | * | |
5 | * For low-latency decompression algorithms (e.g. lz4), reserve consecutive | |
6 | * per-CPU virtual memory (in pages) in advance to store such inplace I/O | |
7 | * data if inplace decompression is failed (due to unmet inplace margin for | |
8 | * example). | |
9 | */ | |
10 | #include "internal.h" | |
11 | ||
12 | struct erofs_pcpubuf { | |
13 | raw_spinlock_t lock; | |
14 | void *ptr; | |
15 | struct page **pages; | |
16 | unsigned int nrpages; | |
17 | }; | |
18 | ||
19 | static DEFINE_PER_CPU(struct erofs_pcpubuf, erofs_pcb); | |
20 | ||
21 | void *erofs_get_pcpubuf(unsigned int requiredpages) | |
22 | __acquires(pcb->lock) | |
23 | { | |
24 | struct erofs_pcpubuf *pcb = &get_cpu_var(erofs_pcb); | |
25 | ||
26 | raw_spin_lock(&pcb->lock); | |
27 | /* check if the per-CPU buffer is too small */ | |
28 | if (requiredpages > pcb->nrpages) { | |
29 | raw_spin_unlock(&pcb->lock); | |
30 | put_cpu_var(erofs_pcb); | |
31 | /* (for sparse checker) pretend pcb->lock is still taken */ | |
32 | __acquire(pcb->lock); | |
33 | return NULL; | |
34 | } | |
35 | return pcb->ptr; | |
36 | } | |
37 | ||
38 | void erofs_put_pcpubuf(void *ptr) __releases(pcb->lock) | |
39 | { | |
40 | struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, smp_processor_id()); | |
41 | ||
42 | DBG_BUGON(pcb->ptr != ptr); | |
43 | raw_spin_unlock(&pcb->lock); | |
44 | put_cpu_var(erofs_pcb); | |
45 | } | |
46 | ||
47 | /* the next step: support per-CPU page buffers hotplug */ | |
48 | int erofs_pcpubuf_growsize(unsigned int nrpages) | |
49 | { | |
50 | static DEFINE_MUTEX(pcb_resize_mutex); | |
51 | static unsigned int pcb_nrpages; | |
eaa9172a | 52 | struct page *pagepool = NULL; |
52488734 GX |
53 | int delta, cpu, ret, i; |
54 | ||
55 | mutex_lock(&pcb_resize_mutex); | |
56 | delta = nrpages - pcb_nrpages; | |
57 | ret = 0; | |
58 | /* avoid shrinking pcpubuf, since no idea how many fses rely on */ | |
59 | if (delta <= 0) | |
60 | goto out; | |
61 | ||
62 | for_each_possible_cpu(cpu) { | |
63 | struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, cpu); | |
64 | struct page **pages, **oldpages; | |
65 | void *ptr, *old_ptr; | |
66 | ||
67 | pages = kmalloc_array(nrpages, sizeof(*pages), GFP_KERNEL); | |
68 | if (!pages) { | |
69 | ret = -ENOMEM; | |
70 | break; | |
71 | } | |
72 | ||
73 | for (i = 0; i < nrpages; ++i) { | |
74 | pages[i] = erofs_allocpage(&pagepool, GFP_KERNEL); | |
75 | if (!pages[i]) { | |
76 | ret = -ENOMEM; | |
77 | oldpages = pages; | |
78 | goto free_pagearray; | |
79 | } | |
80 | } | |
81 | ptr = vmap(pages, nrpages, VM_MAP, PAGE_KERNEL); | |
82 | if (!ptr) { | |
83 | ret = -ENOMEM; | |
84 | oldpages = pages; | |
85 | goto free_pagearray; | |
86 | } | |
87 | raw_spin_lock(&pcb->lock); | |
88 | old_ptr = pcb->ptr; | |
89 | pcb->ptr = ptr; | |
90 | oldpages = pcb->pages; | |
91 | pcb->pages = pages; | |
92 | i = pcb->nrpages; | |
93 | pcb->nrpages = nrpages; | |
94 | raw_spin_unlock(&pcb->lock); | |
95 | ||
96 | if (!oldpages) { | |
97 | DBG_BUGON(old_ptr); | |
98 | continue; | |
99 | } | |
100 | ||
101 | if (old_ptr) | |
102 | vunmap(old_ptr); | |
103 | free_pagearray: | |
104 | while (i) | |
eaa9172a | 105 | erofs_pagepool_add(&pagepool, oldpages[--i]); |
52488734 GX |
106 | kfree(oldpages); |
107 | if (ret) | |
108 | break; | |
109 | } | |
110 | pcb_nrpages = nrpages; | |
eaa9172a | 111 | erofs_release_pages(&pagepool); |
52488734 GX |
112 | out: |
113 | mutex_unlock(&pcb_resize_mutex); | |
114 | return ret; | |
115 | } | |
116 | ||
117 | void erofs_pcpubuf_init(void) | |
118 | { | |
119 | int cpu; | |
120 | ||
121 | for_each_possible_cpu(cpu) { | |
122 | struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, cpu); | |
123 | ||
124 | raw_spin_lock_init(&pcb->lock); | |
125 | } | |
126 | } | |
127 | ||
128 | void erofs_pcpubuf_exit(void) | |
129 | { | |
130 | int cpu, i; | |
131 | ||
132 | for_each_possible_cpu(cpu) { | |
133 | struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, cpu); | |
134 | ||
135 | if (pcb->ptr) { | |
136 | vunmap(pcb->ptr); | |
137 | pcb->ptr = NULL; | |
138 | } | |
139 | if (!pcb->pages) | |
140 | continue; | |
141 | ||
142 | for (i = 0; i < pcb->nrpages; ++i) | |
143 | if (pcb->pages[i]) | |
144 | put_page(pcb->pages[i]); | |
145 | kfree(pcb->pages); | |
146 | pcb->pages = NULL; | |
147 | } | |
148 | } |