Commit | Line | Data |
---|---|---|
09c434b8 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
0b81d077 JK |
2 | /* |
3 | * This contains encryption functions for per-file encryption. | |
4 | * | |
5 | * Copyright (C) 2015, Google, Inc. | |
6 | * Copyright (C) 2015, Motorola Mobility | |
7 | * | |
8 | * Written by Michael Halcrow, 2014. | |
9 | * | |
10 | * Filename encryption additions | |
11 | * Uday Savagaonkar, 2014 | |
12 | * Encryption policy handling additions | |
13 | * Ildar Muslukhov, 2014 | |
14 | * Add fscrypt_pullback_bio_page() | |
15 | * Jaegeuk Kim, 2015. | |
16 | * | |
17 | * This has not yet undergone a rigorous security audit. | |
18 | * | |
19 | * The usage of AES-XTS should conform to recommendations in NIST | |
20 | * Special Publication 800-38E and IEEE P1619/D16. | |
21 | */ | |
22 | ||
0b81d077 JK |
23 | #include <linux/pagemap.h> |
24 | #include <linux/mempool.h> | |
25 | #include <linux/module.h> | |
26 | #include <linux/scatterlist.h> | |
27 | #include <linux/ratelimit.h> | |
0b81d077 | 28 | #include <linux/dcache.h> |
03a8bb0e | 29 | #include <linux/namei.h> |
b7e7cf7a | 30 | #include <crypto/aes.h> |
a575784c | 31 | #include <crypto/skcipher.h> |
cc4e0df0 | 32 | #include "fscrypt_private.h" |
0b81d077 JK |
33 | |
34 | static unsigned int num_prealloc_crypto_pages = 32; | |
35 | static unsigned int num_prealloc_crypto_ctxs = 128; | |
36 | ||
37 | module_param(num_prealloc_crypto_pages, uint, 0444); | |
38 | MODULE_PARM_DESC(num_prealloc_crypto_pages, | |
39 | "Number of crypto pages to preallocate"); | |
40 | module_param(num_prealloc_crypto_ctxs, uint, 0444); | |
41 | MODULE_PARM_DESC(num_prealloc_crypto_ctxs, | |
42 | "Number of crypto contexts to preallocate"); | |
43 | ||
44 | static mempool_t *fscrypt_bounce_page_pool = NULL; | |
45 | ||
46 | static LIST_HEAD(fscrypt_free_ctxs); | |
47 | static DEFINE_SPINLOCK(fscrypt_ctx_lock); | |
48 | ||
0cb8dae4 | 49 | static struct workqueue_struct *fscrypt_read_workqueue; |
0b81d077 JK |
50 | static DEFINE_MUTEX(fscrypt_init_mutex); |
51 | ||
52 | static struct kmem_cache *fscrypt_ctx_cachep; | |
53 | struct kmem_cache *fscrypt_info_cachep; | |
54 | ||
0cb8dae4 EB |
55 | void fscrypt_enqueue_decrypt_work(struct work_struct *work) |
56 | { | |
57 | queue_work(fscrypt_read_workqueue, work); | |
58 | } | |
59 | EXPORT_SYMBOL(fscrypt_enqueue_decrypt_work); | |
60 | ||
0b81d077 | 61 | /** |
2a415a02 EB |
62 | * fscrypt_release_ctx() - Release a decryption context |
63 | * @ctx: The decryption context to release. | |
0b81d077 | 64 | * |
2a415a02 EB |
65 | * If the decryption context was allocated from the pre-allocated pool, return |
66 | * it to that pool. Else, free it. | |
0b81d077 JK |
67 | */ |
68 | void fscrypt_release_ctx(struct fscrypt_ctx *ctx) | |
69 | { | |
70 | unsigned long flags; | |
71 | ||
0b81d077 JK |
72 | if (ctx->flags & FS_CTX_REQUIRES_FREE_ENCRYPT_FL) { |
73 | kmem_cache_free(fscrypt_ctx_cachep, ctx); | |
74 | } else { | |
75 | spin_lock_irqsave(&fscrypt_ctx_lock, flags); | |
76 | list_add(&ctx->free_list, &fscrypt_free_ctxs); | |
77 | spin_unlock_irqrestore(&fscrypt_ctx_lock, flags); | |
78 | } | |
79 | } | |
80 | EXPORT_SYMBOL(fscrypt_release_ctx); | |
81 | ||
82 | /** | |
2a415a02 | 83 | * fscrypt_get_ctx() - Get a decryption context |
b32e4482 | 84 | * @gfp_flags: The gfp flag for memory allocation |
0b81d077 | 85 | * |
2a415a02 | 86 | * Allocate and initialize a decryption context. |
0b81d077 | 87 | * |
2a415a02 | 88 | * Return: A new decryption context on success; an ERR_PTR() otherwise. |
0b81d077 | 89 | */ |
cd0265fc | 90 | struct fscrypt_ctx *fscrypt_get_ctx(gfp_t gfp_flags) |
0b81d077 | 91 | { |
cd0265fc | 92 | struct fscrypt_ctx *ctx; |
0b81d077 JK |
93 | unsigned long flags; |
94 | ||
0b81d077 | 95 | /* |
d2d0727b EB |
96 | * First try getting a ctx from the free list so that we don't have to |
97 | * call into the slab allocator. | |
0b81d077 JK |
98 | */ |
99 | spin_lock_irqsave(&fscrypt_ctx_lock, flags); | |
100 | ctx = list_first_entry_or_null(&fscrypt_free_ctxs, | |
101 | struct fscrypt_ctx, free_list); | |
102 | if (ctx) | |
103 | list_del(&ctx->free_list); | |
104 | spin_unlock_irqrestore(&fscrypt_ctx_lock, flags); | |
105 | if (!ctx) { | |
b32e4482 | 106 | ctx = kmem_cache_zalloc(fscrypt_ctx_cachep, gfp_flags); |
0b81d077 JK |
107 | if (!ctx) |
108 | return ERR_PTR(-ENOMEM); | |
109 | ctx->flags |= FS_CTX_REQUIRES_FREE_ENCRYPT_FL; | |
110 | } else { | |
111 | ctx->flags &= ~FS_CTX_REQUIRES_FREE_ENCRYPT_FL; | |
112 | } | |
0b81d077 JK |
113 | return ctx; |
114 | } | |
115 | EXPORT_SYMBOL(fscrypt_get_ctx); | |
116 | ||
d2d0727b EB |
117 | struct page *fscrypt_alloc_bounce_page(gfp_t gfp_flags) |
118 | { | |
119 | return mempool_alloc(fscrypt_bounce_page_pool, gfp_flags); | |
120 | } | |
121 | ||
122 | /** | |
123 | * fscrypt_free_bounce_page() - free a ciphertext bounce page | |
124 | * | |
53bc1d85 EB |
125 | * Free a bounce page that was allocated by fscrypt_encrypt_pagecache_blocks(), |
126 | * or by fscrypt_alloc_bounce_page() directly. | |
d2d0727b EB |
127 | */ |
128 | void fscrypt_free_bounce_page(struct page *bounce_page) | |
129 | { | |
130 | if (!bounce_page) | |
131 | return; | |
132 | set_page_private(bounce_page, (unsigned long)NULL); | |
133 | ClearPagePrivate(bounce_page); | |
134 | mempool_free(bounce_page, fscrypt_bounce_page_pool); | |
135 | } | |
136 | EXPORT_SYMBOL(fscrypt_free_bounce_page); | |
137 | ||
8094c3ce EB |
138 | void fscrypt_generate_iv(union fscrypt_iv *iv, u64 lblk_num, |
139 | const struct fscrypt_info *ci) | |
140 | { | |
141 | memset(iv, 0, ci->ci_mode->ivsize); | |
142 | iv->lblk_num = cpu_to_le64(lblk_num); | |
143 | ||
5dae460c | 144 | if (fscrypt_is_direct_key_policy(&ci->ci_policy)) |
8094c3ce EB |
145 | memcpy(iv->nonce, ci->ci_nonce, FS_KEY_DERIVATION_NONCE_SIZE); |
146 | ||
147 | if (ci->ci_essiv_tfm != NULL) | |
148 | crypto_cipher_encrypt_one(ci->ci_essiv_tfm, iv->raw, iv->raw); | |
149 | } | |
150 | ||
f47fcbb2 EB |
151 | /* Encrypt or decrypt a single filesystem block of file contents */ |
152 | int fscrypt_crypt_block(const struct inode *inode, fscrypt_direction_t rw, | |
153 | u64 lblk_num, struct page *src_page, | |
154 | struct page *dest_page, unsigned int len, | |
155 | unsigned int offs, gfp_t gfp_flags) | |
0b81d077 | 156 | { |
8094c3ce | 157 | union fscrypt_iv iv; |
d407574e | 158 | struct skcipher_request *req = NULL; |
d0082e1a | 159 | DECLARE_CRYPTO_WAIT(wait); |
0b81d077 JK |
160 | struct scatterlist dst, src; |
161 | struct fscrypt_info *ci = inode->i_crypt_info; | |
d407574e | 162 | struct crypto_skcipher *tfm = ci->ci_ctfm; |
0b81d077 JK |
163 | int res = 0; |
164 | ||
eeacfdc6 EB |
165 | if (WARN_ON_ONCE(len <= 0)) |
166 | return -EINVAL; | |
167 | if (WARN_ON_ONCE(len % FS_CRYPTO_BLOCK_SIZE != 0)) | |
168 | return -EINVAL; | |
1400451f | 169 | |
8094c3ce | 170 | fscrypt_generate_iv(&iv, lblk_num, ci); |
b7e7cf7a | 171 | |
b32e4482 | 172 | req = skcipher_request_alloc(tfm, gfp_flags); |
c90fd775 | 173 | if (!req) |
0b81d077 | 174 | return -ENOMEM; |
0b81d077 | 175 | |
d407574e | 176 | skcipher_request_set_callback( |
0b81d077 | 177 | req, CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP, |
d0082e1a | 178 | crypto_req_done, &wait); |
0b81d077 | 179 | |
0b81d077 | 180 | sg_init_table(&dst, 1); |
1400451f | 181 | sg_set_page(&dst, dest_page, len, offs); |
0b81d077 | 182 | sg_init_table(&src, 1); |
1400451f | 183 | sg_set_page(&src, src_page, len, offs); |
b7e7cf7a | 184 | skcipher_request_set_crypt(req, &src, &dst, len, &iv); |
0b81d077 | 185 | if (rw == FS_DECRYPT) |
d0082e1a | 186 | res = crypto_wait_req(crypto_skcipher_decrypt(req), &wait); |
0b81d077 | 187 | else |
d0082e1a | 188 | res = crypto_wait_req(crypto_skcipher_encrypt(req), &wait); |
d407574e | 189 | skcipher_request_free(req); |
0b81d077 | 190 | if (res) { |
886da8b3 EB |
191 | fscrypt_err(inode, "%scryption failed for block %llu: %d", |
192 | (rw == FS_DECRYPT ? "De" : "En"), lblk_num, res); | |
0b81d077 JK |
193 | return res; |
194 | } | |
195 | return 0; | |
196 | } | |
197 | ||
0b81d077 | 198 | /** |
53bc1d85 EB |
199 | * fscrypt_encrypt_pagecache_blocks() - Encrypt filesystem blocks from a pagecache page |
200 | * @page: The locked pagecache page containing the block(s) to encrypt | |
201 | * @len: Total size of the block(s) to encrypt. Must be a nonzero | |
202 | * multiple of the filesystem's block size. | |
203 | * @offs: Byte offset within @page of the first block to encrypt. Must be | |
204 | * a multiple of the filesystem's block size. | |
205 | * @gfp_flags: Memory allocation flags | |
206 | * | |
207 | * A new bounce page is allocated, and the specified block(s) are encrypted into | |
208 | * it. In the bounce page, the ciphertext block(s) will be located at the same | |
209 | * offsets at which the plaintext block(s) were located in the source page; any | |
210 | * other parts of the bounce page will be left uninitialized. However, normally | |
211 | * blocksize == PAGE_SIZE and the whole page is encrypted at once. | |
0b81d077 | 212 | * |
53bc1d85 | 213 | * This is for use by the filesystem's ->writepages() method. |
0b81d077 | 214 | * |
53bc1d85 | 215 | * Return: the new encrypted bounce page on success; an ERR_PTR() on failure |
0b81d077 | 216 | */ |
53bc1d85 EB |
217 | struct page *fscrypt_encrypt_pagecache_blocks(struct page *page, |
218 | unsigned int len, | |
219 | unsigned int offs, | |
220 | gfp_t gfp_flags) | |
7821d4dd | 221 | |
0b81d077 | 222 | { |
53bc1d85 EB |
223 | const struct inode *inode = page->mapping->host; |
224 | const unsigned int blockbits = inode->i_blkbits; | |
225 | const unsigned int blocksize = 1 << blockbits; | |
03569f2f | 226 | struct page *ciphertext_page; |
53bc1d85 EB |
227 | u64 lblk_num = ((u64)page->index << (PAGE_SHIFT - blockbits)) + |
228 | (offs >> blockbits); | |
229 | unsigned int i; | |
0b81d077 JK |
230 | int err; |
231 | ||
eeacfdc6 EB |
232 | if (WARN_ON_ONCE(!PageLocked(page))) |
233 | return ERR_PTR(-EINVAL); | |
bd7b8290 | 234 | |
53bc1d85 EB |
235 | if (WARN_ON_ONCE(len <= 0 || !IS_ALIGNED(len | offs, blocksize))) |
236 | return ERR_PTR(-EINVAL); | |
237 | ||
d2d0727b EB |
238 | ciphertext_page = fscrypt_alloc_bounce_page(gfp_flags); |
239 | if (!ciphertext_page) | |
240 | return ERR_PTR(-ENOMEM); | |
0b81d077 | 241 | |
53bc1d85 EB |
242 | for (i = offs; i < offs + len; i += blocksize, lblk_num++) { |
243 | err = fscrypt_crypt_block(inode, FS_ENCRYPT, lblk_num, | |
244 | page, ciphertext_page, | |
245 | blocksize, i, gfp_flags); | |
246 | if (err) { | |
247 | fscrypt_free_bounce_page(ciphertext_page); | |
248 | return ERR_PTR(err); | |
249 | } | |
0b81d077 | 250 | } |
9e532772 | 251 | SetPagePrivate(ciphertext_page); |
d2d0727b | 252 | set_page_private(ciphertext_page, (unsigned long)page); |
0b81d077 JK |
253 | return ciphertext_page; |
254 | } | |
53bc1d85 | 255 | EXPORT_SYMBOL(fscrypt_encrypt_pagecache_blocks); |
0b81d077 | 256 | |
03569f2f EB |
257 | /** |
258 | * fscrypt_encrypt_block_inplace() - Encrypt a filesystem block in-place | |
259 | * @inode: The inode to which this block belongs | |
260 | * @page: The page containing the block to encrypt | |
261 | * @len: Size of block to encrypt. Doesn't need to be a multiple of the | |
262 | * fs block size, but must be a multiple of FS_CRYPTO_BLOCK_SIZE. | |
263 | * @offs: Byte offset within @page at which the block to encrypt begins | |
264 | * @lblk_num: Filesystem logical block number of the block, i.e. the 0-based | |
265 | * number of the block within the file | |
266 | * @gfp_flags: Memory allocation flags | |
267 | * | |
268 | * Encrypt a possibly-compressed filesystem block that is located in an | |
269 | * arbitrary page, not necessarily in the original pagecache page. The @inode | |
270 | * and @lblk_num must be specified, as they can't be determined from @page. | |
271 | * | |
272 | * Return: 0 on success; -errno on failure | |
273 | */ | |
274 | int fscrypt_encrypt_block_inplace(const struct inode *inode, struct page *page, | |
275 | unsigned int len, unsigned int offs, | |
276 | u64 lblk_num, gfp_t gfp_flags) | |
277 | { | |
278 | return fscrypt_crypt_block(inode, FS_ENCRYPT, lblk_num, page, page, | |
279 | len, offs, gfp_flags); | |
280 | } | |
281 | EXPORT_SYMBOL(fscrypt_encrypt_block_inplace); | |
282 | ||
0b81d077 | 283 | /** |
aa8bc1ac EB |
284 | * fscrypt_decrypt_pagecache_blocks() - Decrypt filesystem blocks in a pagecache page |
285 | * @page: The locked pagecache page containing the block(s) to decrypt | |
286 | * @len: Total size of the block(s) to decrypt. Must be a nonzero | |
287 | * multiple of the filesystem's block size. | |
288 | * @offs: Byte offset within @page of the first block to decrypt. Must be | |
289 | * a multiple of the filesystem's block size. | |
0b81d077 | 290 | * |
aa8bc1ac EB |
291 | * The specified block(s) are decrypted in-place within the pagecache page, |
292 | * which must still be locked and not uptodate. Normally, blocksize == | |
293 | * PAGE_SIZE and the whole page is decrypted at once. | |
0b81d077 | 294 | * |
aa8bc1ac | 295 | * This is for use by the filesystem's ->readpages() method. |
0b81d077 | 296 | * |
aa8bc1ac | 297 | * Return: 0 on success; -errno on failure |
0b81d077 | 298 | */ |
aa8bc1ac EB |
299 | int fscrypt_decrypt_pagecache_blocks(struct page *page, unsigned int len, |
300 | unsigned int offs) | |
0b81d077 | 301 | { |
aa8bc1ac EB |
302 | const struct inode *inode = page->mapping->host; |
303 | const unsigned int blockbits = inode->i_blkbits; | |
304 | const unsigned int blocksize = 1 << blockbits; | |
305 | u64 lblk_num = ((u64)page->index << (PAGE_SHIFT - blockbits)) + | |
306 | (offs >> blockbits); | |
307 | unsigned int i; | |
308 | int err; | |
309 | ||
41adbcb7 | 310 | if (WARN_ON_ONCE(!PageLocked(page))) |
eeacfdc6 | 311 | return -EINVAL; |
bd7b8290 | 312 | |
aa8bc1ac EB |
313 | if (WARN_ON_ONCE(len <= 0 || !IS_ALIGNED(len | offs, blocksize))) |
314 | return -EINVAL; | |
315 | ||
316 | for (i = offs; i < offs + len; i += blocksize, lblk_num++) { | |
317 | err = fscrypt_crypt_block(inode, FS_DECRYPT, lblk_num, page, | |
318 | page, blocksize, i, GFP_NOFS); | |
319 | if (err) | |
320 | return err; | |
321 | } | |
322 | return 0; | |
0b81d077 | 323 | } |
aa8bc1ac | 324 | EXPORT_SYMBOL(fscrypt_decrypt_pagecache_blocks); |
0b81d077 | 325 | |
41adbcb7 EB |
326 | /** |
327 | * fscrypt_decrypt_block_inplace() - Decrypt a filesystem block in-place | |
328 | * @inode: The inode to which this block belongs | |
329 | * @page: The page containing the block to decrypt | |
330 | * @len: Size of block to decrypt. Doesn't need to be a multiple of the | |
331 | * fs block size, but must be a multiple of FS_CRYPTO_BLOCK_SIZE. | |
332 | * @offs: Byte offset within @page at which the block to decrypt begins | |
333 | * @lblk_num: Filesystem logical block number of the block, i.e. the 0-based | |
334 | * number of the block within the file | |
335 | * | |
336 | * Decrypt a possibly-compressed filesystem block that is located in an | |
337 | * arbitrary page, not necessarily in the original pagecache page. The @inode | |
338 | * and @lblk_num must be specified, as they can't be determined from @page. | |
339 | * | |
340 | * Return: 0 on success; -errno on failure | |
341 | */ | |
342 | int fscrypt_decrypt_block_inplace(const struct inode *inode, struct page *page, | |
343 | unsigned int len, unsigned int offs, | |
344 | u64 lblk_num) | |
345 | { | |
346 | return fscrypt_crypt_block(inode, FS_DECRYPT, lblk_num, page, page, | |
347 | len, offs, GFP_NOFS); | |
348 | } | |
349 | EXPORT_SYMBOL(fscrypt_decrypt_block_inplace); | |
350 | ||
0b81d077 | 351 | /* |
6cc24868 EB |
352 | * Validate dentries in encrypted directories to make sure we aren't potentially |
353 | * caching stale dentries after a key has been added. | |
0b81d077 JK |
354 | */ |
355 | static int fscrypt_d_revalidate(struct dentry *dentry, unsigned int flags) | |
356 | { | |
d7d75352 | 357 | struct dentry *dir; |
6cc24868 EB |
358 | int err; |
359 | int valid; | |
360 | ||
361 | /* | |
362 | * Plaintext names are always valid, since fscrypt doesn't support | |
363 | * reverting to ciphertext names without evicting the directory's inode | |
364 | * -- which implies eviction of the dentries in the directory. | |
365 | */ | |
366 | if (!(dentry->d_flags & DCACHE_ENCRYPTED_NAME)) | |
367 | return 1; | |
368 | ||
369 | /* | |
370 | * Ciphertext name; valid if the directory's key is still unavailable. | |
371 | * | |
372 | * Although fscrypt forbids rename() on ciphertext names, we still must | |
373 | * use dget_parent() here rather than use ->d_parent directly. That's | |
374 | * because a corrupted fs image may contain directory hard links, which | |
375 | * the VFS handles by moving the directory's dentry tree in the dcache | |
376 | * each time ->lookup() finds the directory and it already has a dentry | |
377 | * elsewhere. Thus ->d_parent can be changing, and we must safely grab | |
378 | * a reference to some ->d_parent to prevent it from being freed. | |
379 | */ | |
0b81d077 | 380 | |
03a8bb0e JK |
381 | if (flags & LOOKUP_RCU) |
382 | return -ECHILD; | |
383 | ||
d7d75352 | 384 | dir = dget_parent(dentry); |
6cc24868 EB |
385 | err = fscrypt_get_encryption_info(d_inode(dir)); |
386 | valid = !fscrypt_has_encryption_key(d_inode(dir)); | |
d7d75352 | 387 | dput(dir); |
0b81d077 | 388 | |
6cc24868 EB |
389 | if (err < 0) |
390 | return err; | |
391 | ||
392 | return valid; | |
0b81d077 JK |
393 | } |
394 | ||
395 | const struct dentry_operations fscrypt_d_ops = { | |
396 | .d_revalidate = fscrypt_d_revalidate, | |
397 | }; | |
0b81d077 | 398 | |
0b81d077 JK |
399 | static void fscrypt_destroy(void) |
400 | { | |
401 | struct fscrypt_ctx *pos, *n; | |
402 | ||
403 | list_for_each_entry_safe(pos, n, &fscrypt_free_ctxs, free_list) | |
404 | kmem_cache_free(fscrypt_ctx_cachep, pos); | |
405 | INIT_LIST_HEAD(&fscrypt_free_ctxs); | |
406 | mempool_destroy(fscrypt_bounce_page_pool); | |
407 | fscrypt_bounce_page_pool = NULL; | |
408 | } | |
409 | ||
410 | /** | |
411 | * fscrypt_initialize() - allocate major buffers for fs encryption. | |
f32d7ac2 | 412 | * @cop_flags: fscrypt operations flags |
0b81d077 JK |
413 | * |
414 | * We only call this when we start accessing encrypted files, since it | |
415 | * results in memory getting allocated that wouldn't otherwise be used. | |
416 | * | |
417 | * Return: Zero on success, non-zero otherwise. | |
418 | */ | |
f32d7ac2 | 419 | int fscrypt_initialize(unsigned int cop_flags) |
0b81d077 JK |
420 | { |
421 | int i, res = -ENOMEM; | |
422 | ||
a0b3bc85 EB |
423 | /* No need to allocate a bounce page pool if this FS won't use it. */ |
424 | if (cop_flags & FS_CFLG_OWN_PAGES) | |
0b81d077 JK |
425 | return 0; |
426 | ||
427 | mutex_lock(&fscrypt_init_mutex); | |
428 | if (fscrypt_bounce_page_pool) | |
429 | goto already_initialized; | |
430 | ||
431 | for (i = 0; i < num_prealloc_crypto_ctxs; i++) { | |
432 | struct fscrypt_ctx *ctx; | |
433 | ||
434 | ctx = kmem_cache_zalloc(fscrypt_ctx_cachep, GFP_NOFS); | |
435 | if (!ctx) | |
436 | goto fail; | |
437 | list_add(&ctx->free_list, &fscrypt_free_ctxs); | |
438 | } | |
439 | ||
440 | fscrypt_bounce_page_pool = | |
441 | mempool_create_page_pool(num_prealloc_crypto_pages, 0); | |
442 | if (!fscrypt_bounce_page_pool) | |
443 | goto fail; | |
444 | ||
445 | already_initialized: | |
446 | mutex_unlock(&fscrypt_init_mutex); | |
447 | return 0; | |
448 | fail: | |
449 | fscrypt_destroy(); | |
450 | mutex_unlock(&fscrypt_init_mutex); | |
451 | return res; | |
452 | } | |
0b81d077 | 453 | |
886da8b3 | 454 | void fscrypt_msg(const struct inode *inode, const char *level, |
544d08fd EB |
455 | const char *fmt, ...) |
456 | { | |
457 | static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL, | |
458 | DEFAULT_RATELIMIT_BURST); | |
459 | struct va_format vaf; | |
460 | va_list args; | |
461 | ||
462 | if (!__ratelimit(&rs)) | |
463 | return; | |
464 | ||
465 | va_start(args, fmt); | |
466 | vaf.fmt = fmt; | |
467 | vaf.va = &args; | |
886da8b3 EB |
468 | if (inode) |
469 | printk("%sfscrypt (%s, inode %lu): %pV\n", | |
470 | level, inode->i_sb->s_id, inode->i_ino, &vaf); | |
544d08fd EB |
471 | else |
472 | printk("%sfscrypt: %pV\n", level, &vaf); | |
473 | va_end(args); | |
474 | } | |
475 | ||
0b81d077 JK |
476 | /** |
477 | * fscrypt_init() - Set up for fs encryption. | |
478 | */ | |
479 | static int __init fscrypt_init(void) | |
480 | { | |
22d94f49 EB |
481 | int err = -ENOMEM; |
482 | ||
36dd26e0 EB |
483 | /* |
484 | * Use an unbound workqueue to allow bios to be decrypted in parallel | |
485 | * even when they happen to complete on the same CPU. This sacrifices | |
486 | * locality, but it's worthwhile since decryption is CPU-intensive. | |
487 | * | |
488 | * Also use a high-priority workqueue to prioritize decryption work, | |
489 | * which blocks reads from completing, over regular application tasks. | |
490 | */ | |
0b81d077 | 491 | fscrypt_read_workqueue = alloc_workqueue("fscrypt_read_queue", |
36dd26e0 EB |
492 | WQ_UNBOUND | WQ_HIGHPRI, |
493 | num_online_cpus()); | |
0b81d077 JK |
494 | if (!fscrypt_read_workqueue) |
495 | goto fail; | |
496 | ||
497 | fscrypt_ctx_cachep = KMEM_CACHE(fscrypt_ctx, SLAB_RECLAIM_ACCOUNT); | |
498 | if (!fscrypt_ctx_cachep) | |
499 | goto fail_free_queue; | |
500 | ||
501 | fscrypt_info_cachep = KMEM_CACHE(fscrypt_info, SLAB_RECLAIM_ACCOUNT); | |
502 | if (!fscrypt_info_cachep) | |
503 | goto fail_free_ctx; | |
504 | ||
22d94f49 EB |
505 | err = fscrypt_init_keyring(); |
506 | if (err) | |
507 | goto fail_free_info; | |
508 | ||
0b81d077 JK |
509 | return 0; |
510 | ||
22d94f49 EB |
511 | fail_free_info: |
512 | kmem_cache_destroy(fscrypt_info_cachep); | |
0b81d077 JK |
513 | fail_free_ctx: |
514 | kmem_cache_destroy(fscrypt_ctx_cachep); | |
515 | fail_free_queue: | |
516 | destroy_workqueue(fscrypt_read_workqueue); | |
517 | fail: | |
22d94f49 | 518 | return err; |
0b81d077 | 519 | } |
75798f85 | 520 | late_initcall(fscrypt_init) |