Commit | Line | Data |
---|---|---|
2874c5fd | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
e7e1ef43 SS |
2 | /* |
3 | * Copyright (C) 2014 Sergey Senozhatsky. | |
e7e1ef43 SS |
4 | */ |
5 | ||
6 | #include <linux/kernel.h> | |
7 | #include <linux/string.h> | |
fcfa8d95 | 8 | #include <linux/err.h> |
e7e1ef43 SS |
9 | #include <linux/slab.h> |
10 | #include <linux/wait.h> | |
11 | #include <linux/sched.h> | |
da9556a2 | 12 | #include <linux/cpu.h> |
ebaf9ab5 | 13 | #include <linux/crypto.h> |
e7e1ef43 SS |
14 | |
15 | #include "zcomp.h" | |
e7e1ef43 | 16 | |
ebaf9ab5 | 17 | static const char * const backends[] = { |
3d711a38 | 18 | #if IS_ENABLED(CONFIG_CRYPTO_LZO) |
ebaf9ab5 | 19 | "lzo", |
45ec975e | 20 | "lzo-rle", |
3d711a38 | 21 | #endif |
ce1ed9f9 | 22 | #if IS_ENABLED(CONFIG_CRYPTO_LZ4) |
ebaf9ab5 | 23 | "lz4", |
eb9f56d8 | 24 | #endif |
eb9f56d8 SS |
25 | #if IS_ENABLED(CONFIG_CRYPTO_LZ4HC) |
26 | "lz4hc", | |
27 | #endif | |
28 | #if IS_ENABLED(CONFIG_CRYPTO_842) | |
29 | "842", | |
5ef3a8b1 SS |
30 | #endif |
31 | #if IS_ENABLED(CONFIG_CRYPTO_ZSTD) | |
32 | "zstd", | |
6e76668e | 33 | #endif |
e46b8a03 SS |
34 | }; |
35 | ||
ebaf9ab5 | 36 | static void zcomp_strm_free(struct zcomp_strm *zstrm) |
e7e1ef43 | 37 | { |
ebaf9ab5 SS |
38 | if (!IS_ERR_OR_NULL(zstrm->tfm)) |
39 | crypto_free_comp(zstrm->tfm); | |
e7e1ef43 | 40 | free_pages((unsigned long)zstrm->buffer, 1); |
ed19f192 SAS |
41 | zstrm->tfm = NULL; |
42 | zstrm->buffer = NULL; | |
e7e1ef43 SS |
43 | } |
44 | ||
45 | /* | |
ed19f192 SAS |
46 | * Initialize zcomp_strm structure with ->tfm initialized by backend, and |
47 | * ->buffer. Return a negative value on error. | |
e7e1ef43 | 48 | */ |
ed19f192 | 49 | static int zcomp_strm_init(struct zcomp_strm *zstrm, struct zcomp *comp) |
e7e1ef43 | 50 | { |
ebaf9ab5 | 51 | zstrm->tfm = crypto_alloc_comp(comp->name, 0, 0); |
e7e1ef43 SS |
52 | /* |
53 | * allocate 2 pages. 1 for compressed data, plus 1 extra for the | |
54 | * case when compressed size is larger than the original one | |
55 | */ | |
16d37725 | 56 | zstrm->buffer = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 1); |
ebaf9ab5 SS |
57 | if (IS_ERR_OR_NULL(zstrm->tfm) || !zstrm->buffer) { |
58 | zcomp_strm_free(zstrm); | |
ed19f192 | 59 | return -ENOMEM; |
e7e1ef43 | 60 | } |
ed19f192 | 61 | return 0; |
e7e1ef43 SS |
62 | } |
63 | ||
415403be SS |
64 | bool zcomp_available_algorithm(const char *comp) |
65 | { | |
ed8a5553 | 66 | int i; |
415403be | 67 | |
276aa42e | 68 | i = sysfs_match_string(backends, comp); |
ed8a5553 AS |
69 | if (i >= 0) |
70 | return true; | |
415403be SS |
71 | |
72 | /* | |
73 | * Crypto does not ignore a trailing new line symbol, | |
74 | * so make sure you don't supply a string containing | |
75 | * one. | |
76 | * This also means that we permit zcomp initialisation | |
77 | * with any compressing algorithm known to crypto api. | |
78 | */ | |
79 | return crypto_has_comp(comp, 0, 0) == 1; | |
80 | } | |
81 | ||
e46b8a03 SS |
82 | /* show available compressors */ |
83 | ssize_t zcomp_available_show(const char *comp, char *buf) | |
84 | { | |
415403be | 85 | bool known_algorithm = false; |
e46b8a03 | 86 | ssize_t sz = 0; |
276aa42e | 87 | int i; |
e46b8a03 | 88 | |
276aa42e | 89 | for (i = 0; i < ARRAY_SIZE(backends); i++) { |
415403be SS |
90 | if (!strcmp(comp, backends[i])) { |
91 | known_algorithm = true; | |
56b4e8cb | 92 | sz += scnprintf(buf + sz, PAGE_SIZE - sz - 2, |
ebaf9ab5 | 93 | "[%s] ", backends[i]); |
415403be | 94 | } else { |
56b4e8cb | 95 | sz += scnprintf(buf + sz, PAGE_SIZE - sz - 2, |
ebaf9ab5 | 96 | "%s ", backends[i]); |
415403be | 97 | } |
e46b8a03 | 98 | } |
415403be SS |
99 | |
100 | /* | |
101 | * Out-of-tree module known to crypto api or a missing | |
102 | * entry in `backends'. | |
103 | */ | |
104 | if (!known_algorithm && crypto_has_comp(comp, 0, 0) == 1) | |
105 | sz += scnprintf(buf + sz, PAGE_SIZE - sz - 2, | |
106 | "[%s] ", comp); | |
107 | ||
56b4e8cb | 108 | sz += scnprintf(buf + sz, PAGE_SIZE - sz, "\n"); |
e46b8a03 SS |
109 | return sz; |
110 | } | |
111 | ||
2aea8493 | 112 | struct zcomp_strm *zcomp_stream_get(struct zcomp *comp) |
e7e1ef43 | 113 | { |
19f545b6 MG |
114 | local_lock(&comp->stream->lock); |
115 | return this_cpu_ptr(comp->stream); | |
e7e1ef43 SS |
116 | } |
117 | ||
2aea8493 | 118 | void zcomp_stream_put(struct zcomp *comp) |
e7e1ef43 | 119 | { |
19f545b6 | 120 | local_unlock(&comp->stream->lock); |
e7e1ef43 SS |
121 | } |
122 | ||
ebaf9ab5 SS |
123 | int zcomp_compress(struct zcomp_strm *zstrm, |
124 | const void *src, unsigned int *dst_len) | |
e7e1ef43 | 125 | { |
ebaf9ab5 SS |
126 | /* |
127 | * Our dst memory (zstrm->buffer) is always `2 * PAGE_SIZE' sized | |
128 | * because sometimes we can endup having a bigger compressed data | |
129 | * due to various reasons: for example compression algorithms tend | |
130 | * to add some padding to the compressed buffer. Speaking of padding, | |
131 | * comp algorithm `842' pads the compressed length to multiple of 8 | |
132 | * and returns -ENOSP when the dst memory is not big enough, which | |
133 | * is not something that ZRAM wants to see. We can handle the | |
134 | * `compressed_size > PAGE_SIZE' case easily in ZRAM, but when we | |
135 | * receive -ERRNO from the compressing backend we can't help it | |
136 | * anymore. To make `842' happy we need to tell the exact size of | |
137 | * the dst buffer, zram_drv will take care of the fact that | |
138 | * compressed buffer is too big. | |
139 | */ | |
140 | *dst_len = PAGE_SIZE * 2; | |
141 | ||
142 | return crypto_comp_compress(zstrm->tfm, | |
143 | src, PAGE_SIZE, | |
144 | zstrm->buffer, dst_len); | |
e7e1ef43 SS |
145 | } |
146 | ||
ebaf9ab5 SS |
147 | int zcomp_decompress(struct zcomp_strm *zstrm, |
148 | const void *src, unsigned int src_len, void *dst) | |
e7e1ef43 | 149 | { |
ebaf9ab5 SS |
150 | unsigned int dst_len = PAGE_SIZE; |
151 | ||
152 | return crypto_comp_decompress(zstrm->tfm, | |
153 | src, src_len, | |
154 | dst, &dst_len); | |
e7e1ef43 SS |
155 | } |
156 | ||
1dd6c834 | 157 | int zcomp_cpu_up_prepare(unsigned int cpu, struct hlist_node *node) |
da9556a2 | 158 | { |
1dd6c834 | 159 | struct zcomp *comp = hlist_entry(node, struct zcomp, node); |
da9556a2 | 160 | struct zcomp_strm *zstrm; |
ed19f192 | 161 | int ret; |
da9556a2 | 162 | |
ed19f192 | 163 | zstrm = per_cpu_ptr(comp->stream, cpu); |
19f545b6 MG |
164 | local_lock_init(&zstrm->lock); |
165 | ||
ed19f192 SAS |
166 | ret = zcomp_strm_init(zstrm, comp); |
167 | if (ret) | |
1dd6c834 | 168 | pr_err("Can't allocate a compression stream\n"); |
ed19f192 | 169 | return ret; |
da9556a2 SS |
170 | } |
171 | ||
1dd6c834 | 172 | int zcomp_cpu_dead(unsigned int cpu, struct hlist_node *node) |
da9556a2 | 173 | { |
1dd6c834 AMG |
174 | struct zcomp *comp = hlist_entry(node, struct zcomp, node); |
175 | struct zcomp_strm *zstrm; | |
da9556a2 | 176 | |
ed19f192 SAS |
177 | zstrm = per_cpu_ptr(comp->stream, cpu); |
178 | zcomp_strm_free(zstrm); | |
1dd6c834 | 179 | return 0; |
da9556a2 SS |
180 | } |
181 | ||
182 | static int zcomp_init(struct zcomp *comp) | |
183 | { | |
da9556a2 SS |
184 | int ret; |
185 | ||
ed19f192 | 186 | comp->stream = alloc_percpu(struct zcomp_strm); |
da9556a2 SS |
187 | if (!comp->stream) |
188 | return -ENOMEM; | |
189 | ||
1dd6c834 AMG |
190 | ret = cpuhp_state_add_instance(CPUHP_ZCOMP_PREPARE, &comp->node); |
191 | if (ret < 0) | |
192 | goto cleanup; | |
da9556a2 SS |
193 | return 0; |
194 | ||
195 | cleanup: | |
1dd6c834 AMG |
196 | free_percpu(comp->stream); |
197 | return ret; | |
da9556a2 SS |
198 | } |
199 | ||
e7e1ef43 SS |
200 | void zcomp_destroy(struct zcomp *comp) |
201 | { | |
1dd6c834 | 202 | cpuhp_state_remove_instance(CPUHP_ZCOMP_PREPARE, &comp->node); |
da9556a2 | 203 | free_percpu(comp->stream); |
e7e1ef43 SS |
204 | kfree(comp); |
205 | } | |
206 | ||
207 | /* | |
208 | * search available compressors for requested algorithm. | |
fcfa8d95 SS |
209 | * allocate new zcomp and initialize it. return compressing |
210 | * backend pointer or ERR_PTR if things went bad. ERR_PTR(-EINVAL) | |
211 | * if requested algorithm is not supported, ERR_PTR(-ENOMEM) in | |
3aaf14da | 212 | * case of allocation error, or any other error potentially |
da9556a2 | 213 | * returned by zcomp_init(). |
e7e1ef43 | 214 | */ |
da9556a2 | 215 | struct zcomp *zcomp_create(const char *compress) |
e7e1ef43 SS |
216 | { |
217 | struct zcomp *comp; | |
3aaf14da | 218 | int error; |
e7e1ef43 | 219 | |
415403be | 220 | if (!zcomp_available_algorithm(compress)) |
fcfa8d95 | 221 | return ERR_PTR(-EINVAL); |
e7e1ef43 SS |
222 | |
223 | comp = kzalloc(sizeof(struct zcomp), GFP_KERNEL); | |
224 | if (!comp) | |
fcfa8d95 | 225 | return ERR_PTR(-ENOMEM); |
e7e1ef43 | 226 | |
415403be | 227 | comp->name = compress; |
da9556a2 | 228 | error = zcomp_init(comp); |
3aaf14da | 229 | if (error) { |
e7e1ef43 | 230 | kfree(comp); |
3aaf14da | 231 | return ERR_PTR(error); |
e7e1ef43 SS |
232 | } |
233 | return comp; | |
234 | } |