Commit | Line | Data |
---|---|---|
e7e1ef43 SS |
1 | /* |
2 | * Copyright (C) 2014 Sergey Senozhatsky. | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or | |
5 | * modify it under the terms of the GNU General Public License | |
6 | * as published by the Free Software Foundation; either version | |
7 | * 2 of the License, or (at your option) any later version. | |
8 | */ | |
9 | ||
10 | #include <linux/kernel.h> | |
11 | #include <linux/string.h> | |
fcfa8d95 | 12 | #include <linux/err.h> |
e7e1ef43 SS |
13 | #include <linux/slab.h> |
14 | #include <linux/wait.h> | |
15 | #include <linux/sched.h> | |
da9556a2 | 16 | #include <linux/cpu.h> |
ebaf9ab5 | 17 | #include <linux/crypto.h> |
e7e1ef43 SS |
18 | |
19 | #include "zcomp.h" | |
e7e1ef43 | 20 | |
ebaf9ab5 SS |
21 | static const char * const backends[] = { |
22 | "lzo", | |
ce1ed9f9 | 23 | #if IS_ENABLED(CONFIG_CRYPTO_LZ4) |
ebaf9ab5 | 24 | "lz4", |
eb9f56d8 | 25 | #endif |
eb9f56d8 SS |
26 | #if IS_ENABLED(CONFIG_CRYPTO_LZ4HC) |
27 | "lz4hc", | |
28 | #endif | |
29 | #if IS_ENABLED(CONFIG_CRYPTO_842) | |
30 | "842", | |
5ef3a8b1 SS |
31 | #endif |
32 | #if IS_ENABLED(CONFIG_CRYPTO_ZSTD) | |
33 | "zstd", | |
6e76668e | 34 | #endif |
e46b8a03 SS |
35 | NULL |
36 | }; | |
37 | ||
ebaf9ab5 | 38 | static void zcomp_strm_free(struct zcomp_strm *zstrm) |
e7e1ef43 | 39 | { |
ebaf9ab5 SS |
40 | if (!IS_ERR_OR_NULL(zstrm->tfm)) |
41 | crypto_free_comp(zstrm->tfm); | |
e7e1ef43 SS |
42 | free_pages((unsigned long)zstrm->buffer, 1); |
43 | kfree(zstrm); | |
44 | } | |
45 | ||
46 | /* | |
ebaf9ab5 | 47 | * allocate new zcomp_strm structure with ->tfm initialized by |
e7e1ef43 SS |
48 | * backend, return NULL on error |
49 | */ | |
16d37725 | 50 | static struct zcomp_strm *zcomp_strm_alloc(struct zcomp *comp) |
e7e1ef43 | 51 | { |
16d37725 | 52 | struct zcomp_strm *zstrm = kmalloc(sizeof(*zstrm), GFP_KERNEL); |
e7e1ef43 SS |
53 | if (!zstrm) |
54 | return NULL; | |
55 | ||
ebaf9ab5 | 56 | zstrm->tfm = crypto_alloc_comp(comp->name, 0, 0); |
e7e1ef43 SS |
57 | /* |
58 | * allocate 2 pages. 1 for compressed data, plus 1 extra for the | |
59 | * case when compressed size is larger than the original one | |
60 | */ | |
16d37725 | 61 | zstrm->buffer = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 1); |
ebaf9ab5 SS |
62 | if (IS_ERR_OR_NULL(zstrm->tfm) || !zstrm->buffer) { |
63 | zcomp_strm_free(zstrm); | |
e7e1ef43 SS |
64 | zstrm = NULL; |
65 | } | |
66 | return zstrm; | |
67 | } | |
68 | ||
415403be SS |
69 | bool zcomp_available_algorithm(const char *comp) |
70 | { | |
ed8a5553 | 71 | int i; |
415403be | 72 | |
ed8a5553 AS |
73 | i = __sysfs_match_string(backends, -1, comp); |
74 | if (i >= 0) | |
75 | return true; | |
415403be SS |
76 | |
77 | /* | |
78 | * Crypto does not ignore a trailing new line symbol, | |
79 | * so make sure you don't supply a string containing | |
80 | * one. | |
81 | * This also means that we permit zcomp initialisation | |
82 | * with any compressing algorithm known to crypto api. | |
83 | */ | |
84 | return crypto_has_comp(comp, 0, 0) == 1; | |
85 | } | |
86 | ||
e46b8a03 SS |
87 | /* show available compressors */ |
88 | ssize_t zcomp_available_show(const char *comp, char *buf) | |
89 | { | |
415403be | 90 | bool known_algorithm = false; |
e46b8a03 SS |
91 | ssize_t sz = 0; |
92 | int i = 0; | |
93 | ||
415403be SS |
94 | for (; backends[i]; i++) { |
95 | if (!strcmp(comp, backends[i])) { | |
96 | known_algorithm = true; | |
56b4e8cb | 97 | sz += scnprintf(buf + sz, PAGE_SIZE - sz - 2, |
ebaf9ab5 | 98 | "[%s] ", backends[i]); |
415403be | 99 | } else { |
56b4e8cb | 100 | sz += scnprintf(buf + sz, PAGE_SIZE - sz - 2, |
ebaf9ab5 | 101 | "%s ", backends[i]); |
415403be | 102 | } |
e46b8a03 | 103 | } |
415403be SS |
104 | |
105 | /* | |
106 | * Out-of-tree module known to crypto api or a missing | |
107 | * entry in `backends'. | |
108 | */ | |
109 | if (!known_algorithm && crypto_has_comp(comp, 0, 0) == 1) | |
110 | sz += scnprintf(buf + sz, PAGE_SIZE - sz - 2, | |
111 | "[%s] ", comp); | |
112 | ||
56b4e8cb | 113 | sz += scnprintf(buf + sz, PAGE_SIZE - sz, "\n"); |
e46b8a03 SS |
114 | return sz; |
115 | } | |
116 | ||
2aea8493 | 117 | struct zcomp_strm *zcomp_stream_get(struct zcomp *comp) |
e7e1ef43 | 118 | { |
da9556a2 | 119 | return *get_cpu_ptr(comp->stream); |
e7e1ef43 SS |
120 | } |
121 | ||
2aea8493 | 122 | void zcomp_stream_put(struct zcomp *comp) |
e7e1ef43 | 123 | { |
da9556a2 | 124 | put_cpu_ptr(comp->stream); |
e7e1ef43 SS |
125 | } |
126 | ||
ebaf9ab5 SS |
127 | int zcomp_compress(struct zcomp_strm *zstrm, |
128 | const void *src, unsigned int *dst_len) | |
e7e1ef43 | 129 | { |
ebaf9ab5 SS |
130 | /* |
131 | * Our dst memory (zstrm->buffer) is always `2 * PAGE_SIZE' sized | |
132 | * because sometimes we can endup having a bigger compressed data | |
133 | * due to various reasons: for example compression algorithms tend | |
134 | * to add some padding to the compressed buffer. Speaking of padding, | |
135 | * comp algorithm `842' pads the compressed length to multiple of 8 | |
136 | * and returns -ENOSP when the dst memory is not big enough, which | |
137 | * is not something that ZRAM wants to see. We can handle the | |
138 | * `compressed_size > PAGE_SIZE' case easily in ZRAM, but when we | |
139 | * receive -ERRNO from the compressing backend we can't help it | |
140 | * anymore. To make `842' happy we need to tell the exact size of | |
141 | * the dst buffer, zram_drv will take care of the fact that | |
142 | * compressed buffer is too big. | |
143 | */ | |
144 | *dst_len = PAGE_SIZE * 2; | |
145 | ||
146 | return crypto_comp_compress(zstrm->tfm, | |
147 | src, PAGE_SIZE, | |
148 | zstrm->buffer, dst_len); | |
e7e1ef43 SS |
149 | } |
150 | ||
ebaf9ab5 SS |
151 | int zcomp_decompress(struct zcomp_strm *zstrm, |
152 | const void *src, unsigned int src_len, void *dst) | |
e7e1ef43 | 153 | { |
ebaf9ab5 SS |
154 | unsigned int dst_len = PAGE_SIZE; |
155 | ||
156 | return crypto_comp_decompress(zstrm->tfm, | |
157 | src, src_len, | |
158 | dst, &dst_len); | |
e7e1ef43 SS |
159 | } |
160 | ||
1dd6c834 | 161 | int zcomp_cpu_up_prepare(unsigned int cpu, struct hlist_node *node) |
da9556a2 | 162 | { |
1dd6c834 | 163 | struct zcomp *comp = hlist_entry(node, struct zcomp, node); |
da9556a2 SS |
164 | struct zcomp_strm *zstrm; |
165 | ||
1dd6c834 AMG |
166 | if (WARN_ON(*per_cpu_ptr(comp->stream, cpu))) |
167 | return 0; | |
168 | ||
169 | zstrm = zcomp_strm_alloc(comp); | |
170 | if (IS_ERR_OR_NULL(zstrm)) { | |
171 | pr_err("Can't allocate a compression stream\n"); | |
172 | return -ENOMEM; | |
da9556a2 | 173 | } |
1dd6c834 AMG |
174 | *per_cpu_ptr(comp->stream, cpu) = zstrm; |
175 | return 0; | |
da9556a2 SS |
176 | } |
177 | ||
1dd6c834 | 178 | int zcomp_cpu_dead(unsigned int cpu, struct hlist_node *node) |
da9556a2 | 179 | { |
1dd6c834 AMG |
180 | struct zcomp *comp = hlist_entry(node, struct zcomp, node); |
181 | struct zcomp_strm *zstrm; | |
da9556a2 | 182 | |
1dd6c834 AMG |
183 | zstrm = *per_cpu_ptr(comp->stream, cpu); |
184 | if (!IS_ERR_OR_NULL(zstrm)) | |
185 | zcomp_strm_free(zstrm); | |
186 | *per_cpu_ptr(comp->stream, cpu) = NULL; | |
187 | return 0; | |
da9556a2 SS |
188 | } |
189 | ||
190 | static int zcomp_init(struct zcomp *comp) | |
191 | { | |
da9556a2 SS |
192 | int ret; |
193 | ||
da9556a2 SS |
194 | comp->stream = alloc_percpu(struct zcomp_strm *); |
195 | if (!comp->stream) | |
196 | return -ENOMEM; | |
197 | ||
1dd6c834 AMG |
198 | ret = cpuhp_state_add_instance(CPUHP_ZCOMP_PREPARE, &comp->node); |
199 | if (ret < 0) | |
200 | goto cleanup; | |
da9556a2 SS |
201 | return 0; |
202 | ||
203 | cleanup: | |
1dd6c834 AMG |
204 | free_percpu(comp->stream); |
205 | return ret; | |
da9556a2 SS |
206 | } |
207 | ||
e7e1ef43 SS |
208 | void zcomp_destroy(struct zcomp *comp) |
209 | { | |
1dd6c834 | 210 | cpuhp_state_remove_instance(CPUHP_ZCOMP_PREPARE, &comp->node); |
da9556a2 | 211 | free_percpu(comp->stream); |
e7e1ef43 SS |
212 | kfree(comp); |
213 | } | |
214 | ||
215 | /* | |
216 | * search available compressors for requested algorithm. | |
fcfa8d95 SS |
217 | * allocate new zcomp and initialize it. return compressing |
218 | * backend pointer or ERR_PTR if things went bad. ERR_PTR(-EINVAL) | |
219 | * if requested algorithm is not supported, ERR_PTR(-ENOMEM) in | |
3aaf14da | 220 | * case of allocation error, or any other error potentially |
da9556a2 | 221 | * returned by zcomp_init(). |
e7e1ef43 | 222 | */ |
da9556a2 | 223 | struct zcomp *zcomp_create(const char *compress) |
e7e1ef43 SS |
224 | { |
225 | struct zcomp *comp; | |
3aaf14da | 226 | int error; |
e7e1ef43 | 227 | |
415403be | 228 | if (!zcomp_available_algorithm(compress)) |
fcfa8d95 | 229 | return ERR_PTR(-EINVAL); |
e7e1ef43 SS |
230 | |
231 | comp = kzalloc(sizeof(struct zcomp), GFP_KERNEL); | |
232 | if (!comp) | |
fcfa8d95 | 233 | return ERR_PTR(-ENOMEM); |
e7e1ef43 | 234 | |
415403be | 235 | comp->name = compress; |
da9556a2 | 236 | error = zcomp_init(comp); |
3aaf14da | 237 | if (error) { |
e7e1ef43 | 238 | kfree(comp); |
3aaf14da | 239 | return ERR_PTR(error); |
e7e1ef43 SS |
240 | } |
241 | return comp; | |
242 | } |