2 * This contains encryption functions for per-file encryption.
4 * Copyright (C) 2015, Google, Inc.
5 * Copyright (C) 2015, Motorola Mobility
7 * Written by Michael Halcrow, 2014.
9 * Filename encryption additions
10 * Uday Savagaonkar, 2014
11 * Encryption policy handling additions
12 * Ildar Muslukhov, 2014
13 * Add fscrypt_pullback_bio_page()
16 * This has not yet undergone a rigorous security audit.
18 * The usage of AES-XTS should conform to recommendations in NIST
19 * Special Publication 800-38E and IEEE P1619/D16.
22 #include <linux/pagemap.h>
23 #include <linux/mempool.h>
24 #include <linux/module.h>
25 #include <linux/scatterlist.h>
26 #include <linux/ratelimit.h>
27 #include <linux/dcache.h>
28 #include <linux/namei.h>
29 #include <crypto/aes.h>
30 #include <crypto/skcipher.h>
31 #include "fscrypt_private.h"
33 static unsigned int num_prealloc_crypto_pages = 32;
34 static unsigned int num_prealloc_crypto_ctxs = 128;
36 module_param(num_prealloc_crypto_pages, uint, 0444);
37 MODULE_PARM_DESC(num_prealloc_crypto_pages,
38 "Number of crypto pages to preallocate");
39 module_param(num_prealloc_crypto_ctxs, uint, 0444);
40 MODULE_PARM_DESC(num_prealloc_crypto_ctxs,
41 "Number of crypto contexts to preallocate");
43 static mempool_t *fscrypt_bounce_page_pool = NULL;
45 static LIST_HEAD(fscrypt_free_ctxs);
46 static DEFINE_SPINLOCK(fscrypt_ctx_lock);
48 static struct workqueue_struct *fscrypt_read_workqueue;
49 static DEFINE_MUTEX(fscrypt_init_mutex);
51 static struct kmem_cache *fscrypt_ctx_cachep;
52 struct kmem_cache *fscrypt_info_cachep;
54 void fscrypt_enqueue_decrypt_work(struct work_struct *work)
56 queue_work(fscrypt_read_workqueue, work);
58 EXPORT_SYMBOL(fscrypt_enqueue_decrypt_work);
61 * fscrypt_release_ctx() - Releases an encryption context
62 * @ctx: The encryption context to release.
64 * If the encryption context was allocated from the pre-allocated pool, returns
65 * it to that pool. Else, frees it.
67 * If there's a bounce page in the context, this frees that.
69 void fscrypt_release_ctx(struct fscrypt_ctx *ctx)
73 if (ctx->flags & FS_CTX_HAS_BOUNCE_BUFFER_FL && ctx->w.bounce_page) {
74 mempool_free(ctx->w.bounce_page, fscrypt_bounce_page_pool);
75 ctx->w.bounce_page = NULL;
77 ctx->w.control_page = NULL;
78 if (ctx->flags & FS_CTX_REQUIRES_FREE_ENCRYPT_FL) {
79 kmem_cache_free(fscrypt_ctx_cachep, ctx);
81 spin_lock_irqsave(&fscrypt_ctx_lock, flags);
82 list_add(&ctx->free_list, &fscrypt_free_ctxs);
83 spin_unlock_irqrestore(&fscrypt_ctx_lock, flags);
86 EXPORT_SYMBOL(fscrypt_release_ctx);
89 * fscrypt_get_ctx() - Gets an encryption context
90 * @inode: The inode for which we are doing the crypto
91 * @gfp_flags: The gfp flag for memory allocation
93 * Allocates and initializes an encryption context.
95 * Return: An allocated and initialized encryption context on success; error
96 * value or NULL otherwise.
98 struct fscrypt_ctx *fscrypt_get_ctx(const struct inode *inode, gfp_t gfp_flags)
100 struct fscrypt_ctx *ctx = NULL;
101 struct fscrypt_info *ci = inode->i_crypt_info;
105 return ERR_PTR(-ENOKEY);
108 * We first try getting the ctx from a free list because in
109 * the common case the ctx will have an allocated and
110 * initialized crypto tfm, so it's probably a worthwhile
111 * optimization. For the bounce page, we first try getting it
112 * from the kernel allocator because that's just about as fast
113 * as getting it from a list and because a cache of free pages
114 * should generally be a "last resort" option for a filesystem
115 * to be able to do its job.
117 spin_lock_irqsave(&fscrypt_ctx_lock, flags);
118 ctx = list_first_entry_or_null(&fscrypt_free_ctxs,
119 struct fscrypt_ctx, free_list);
121 list_del(&ctx->free_list);
122 spin_unlock_irqrestore(&fscrypt_ctx_lock, flags);
124 ctx = kmem_cache_zalloc(fscrypt_ctx_cachep, gfp_flags);
126 return ERR_PTR(-ENOMEM);
127 ctx->flags |= FS_CTX_REQUIRES_FREE_ENCRYPT_FL;
129 ctx->flags &= ~FS_CTX_REQUIRES_FREE_ENCRYPT_FL;
131 ctx->flags &= ~FS_CTX_HAS_BOUNCE_BUFFER_FL;
134 EXPORT_SYMBOL(fscrypt_get_ctx);
136 void fscrypt_generate_iv(union fscrypt_iv *iv, u64 lblk_num,
137 const struct fscrypt_info *ci)
139 memset(iv, 0, ci->ci_mode->ivsize);
140 iv->lblk_num = cpu_to_le64(lblk_num);
142 if (ci->ci_flags & FS_POLICY_FLAG_DIRECT_KEY)
143 memcpy(iv->nonce, ci->ci_nonce, FS_KEY_DERIVATION_NONCE_SIZE);
145 if (ci->ci_essiv_tfm != NULL)
146 crypto_cipher_encrypt_one(ci->ci_essiv_tfm, iv->raw, iv->raw);
149 int fscrypt_do_page_crypto(const struct inode *inode, fscrypt_direction_t rw,
150 u64 lblk_num, struct page *src_page,
151 struct page *dest_page, unsigned int len,
152 unsigned int offs, gfp_t gfp_flags)
155 struct skcipher_request *req = NULL;
156 DECLARE_CRYPTO_WAIT(wait);
157 struct scatterlist dst, src;
158 struct fscrypt_info *ci = inode->i_crypt_info;
159 struct crypto_skcipher *tfm = ci->ci_ctfm;
164 fscrypt_generate_iv(&iv, lblk_num, ci);
166 req = skcipher_request_alloc(tfm, gfp_flags);
170 skcipher_request_set_callback(
171 req, CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP,
172 crypto_req_done, &wait);
174 sg_init_table(&dst, 1);
175 sg_set_page(&dst, dest_page, len, offs);
176 sg_init_table(&src, 1);
177 sg_set_page(&src, src_page, len, offs);
178 skcipher_request_set_crypt(req, &src, &dst, len, &iv);
179 if (rw == FS_DECRYPT)
180 res = crypto_wait_req(crypto_skcipher_decrypt(req), &wait);
182 res = crypto_wait_req(crypto_skcipher_encrypt(req), &wait);
183 skcipher_request_free(req);
185 fscrypt_err(inode->i_sb,
186 "%scryption failed for inode %lu, block %llu: %d",
187 (rw == FS_DECRYPT ? "de" : "en"),
188 inode->i_ino, lblk_num, res);
194 struct page *fscrypt_alloc_bounce_page(struct fscrypt_ctx *ctx,
197 ctx->w.bounce_page = mempool_alloc(fscrypt_bounce_page_pool, gfp_flags);
198 if (ctx->w.bounce_page == NULL)
199 return ERR_PTR(-ENOMEM);
200 ctx->flags |= FS_CTX_HAS_BOUNCE_BUFFER_FL;
201 return ctx->w.bounce_page;
205 * fscypt_encrypt_page() - Encrypts a page
206 * @inode: The inode for which the encryption should take place
207 * @page: The page to encrypt. Must be locked for bounce-page
209 * @len: Length of data to encrypt in @page and encrypted
210 * data in returned page.
211 * @offs: Offset of data within @page and returned
212 * page holding encrypted data.
213 * @lblk_num: Logical block number. This must be unique for multiple
214 * calls with same inode, except when overwriting
215 * previously written data.
216 * @gfp_flags: The gfp flag for memory allocation
218 * Encrypts @page using the ctx encryption context. Performs encryption
219 * either in-place or into a newly allocated bounce page.
220 * Called on the page write path.
222 * Bounce page allocation is the default.
223 * In this case, the contents of @page are encrypted and stored in an
224 * allocated bounce page. @page has to be locked and the caller must call
225 * fscrypt_restore_control_page() on the returned ciphertext page to
226 * release the bounce buffer and the encryption context.
228 * In-place encryption is used by setting the FS_CFLG_OWN_PAGES flag in
229 * fscrypt_operations. Here, the input-page is returned with its content
232 * Return: A page with the encrypted content on success. Else, an
233 * error value or NULL.
235 struct page *fscrypt_encrypt_page(const struct inode *inode,
239 u64 lblk_num, gfp_t gfp_flags)
242 struct fscrypt_ctx *ctx;
243 struct page *ciphertext_page = page;
246 BUG_ON(len % FS_CRYPTO_BLOCK_SIZE != 0);
248 if (inode->i_sb->s_cop->flags & FS_CFLG_OWN_PAGES) {
249 /* with inplace-encryption we just encrypt the page */
250 err = fscrypt_do_page_crypto(inode, FS_ENCRYPT, lblk_num, page,
251 ciphertext_page, len, offs,
256 return ciphertext_page;
259 BUG_ON(!PageLocked(page));
261 ctx = fscrypt_get_ctx(inode, gfp_flags);
263 return (struct page *)ctx;
265 /* The encryption operation will require a bounce page. */
266 ciphertext_page = fscrypt_alloc_bounce_page(ctx, gfp_flags);
267 if (IS_ERR(ciphertext_page))
270 ctx->w.control_page = page;
271 err = fscrypt_do_page_crypto(inode, FS_ENCRYPT, lblk_num,
272 page, ciphertext_page, len, offs,
275 ciphertext_page = ERR_PTR(err);
278 SetPagePrivate(ciphertext_page);
279 set_page_private(ciphertext_page, (unsigned long)ctx);
280 lock_page(ciphertext_page);
281 return ciphertext_page;
284 fscrypt_release_ctx(ctx);
285 return ciphertext_page;
287 EXPORT_SYMBOL(fscrypt_encrypt_page);
290 * fscrypt_decrypt_page() - Decrypts a page in-place
291 * @inode: The corresponding inode for the page to decrypt.
292 * @page: The page to decrypt. Must be locked in case
293 * it is a writeback page (FS_CFLG_OWN_PAGES unset).
294 * @len: Number of bytes in @page to be decrypted.
295 * @offs: Start of data in @page.
296 * @lblk_num: Logical block number.
298 * Decrypts page in-place using the ctx encryption context.
300 * Called from the read completion callback.
302 * Return: Zero on success, non-zero otherwise.
304 int fscrypt_decrypt_page(const struct inode *inode, struct page *page,
305 unsigned int len, unsigned int offs, u64 lblk_num)
307 if (!(inode->i_sb->s_cop->flags & FS_CFLG_OWN_PAGES))
308 BUG_ON(!PageLocked(page));
310 return fscrypt_do_page_crypto(inode, FS_DECRYPT, lblk_num, page, page,
311 len, offs, GFP_NOFS);
313 EXPORT_SYMBOL(fscrypt_decrypt_page);
316 * Validate dentries for encrypted directories to make sure we aren't
317 * potentially caching stale data after a key has been added or
320 static int fscrypt_d_revalidate(struct dentry *dentry, unsigned int flags)
323 int dir_has_key, cached_with_key;
325 if (flags & LOOKUP_RCU)
328 dir = dget_parent(dentry);
329 if (!IS_ENCRYPTED(d_inode(dir))) {
334 spin_lock(&dentry->d_lock);
335 cached_with_key = dentry->d_flags & DCACHE_ENCRYPTED_WITH_KEY;
336 spin_unlock(&dentry->d_lock);
337 dir_has_key = (d_inode(dir)->i_crypt_info != NULL);
341 * If the dentry was cached without the key, and it is a
342 * negative dentry, it might be a valid name. We can't check
343 * if the key has since been made available due to locking
344 * reasons, so we fail the validation so ext4_lookup() can do
347 * We also fail the validation if the dentry was created with
348 * the key present, but we no longer have the key, or vice versa.
350 if ((!cached_with_key && d_is_negative(dentry)) ||
351 (!cached_with_key && dir_has_key) ||
352 (cached_with_key && !dir_has_key))
357 const struct dentry_operations fscrypt_d_ops = {
358 .d_revalidate = fscrypt_d_revalidate,
361 void fscrypt_restore_control_page(struct page *page)
363 struct fscrypt_ctx *ctx;
365 ctx = (struct fscrypt_ctx *)page_private(page);
366 set_page_private(page, (unsigned long)NULL);
367 ClearPagePrivate(page);
369 fscrypt_release_ctx(ctx);
371 EXPORT_SYMBOL(fscrypt_restore_control_page);
373 static void fscrypt_destroy(void)
375 struct fscrypt_ctx *pos, *n;
377 list_for_each_entry_safe(pos, n, &fscrypt_free_ctxs, free_list)
378 kmem_cache_free(fscrypt_ctx_cachep, pos);
379 INIT_LIST_HEAD(&fscrypt_free_ctxs);
380 mempool_destroy(fscrypt_bounce_page_pool);
381 fscrypt_bounce_page_pool = NULL;
385 * fscrypt_initialize() - allocate major buffers for fs encryption.
386 * @cop_flags: fscrypt operations flags
388 * We only call this when we start accessing encrypted files, since it
389 * results in memory getting allocated that wouldn't otherwise be used.
391 * Return: Zero on success, non-zero otherwise.
393 int fscrypt_initialize(unsigned int cop_flags)
395 int i, res = -ENOMEM;
397 /* No need to allocate a bounce page pool if this FS won't use it. */
398 if (cop_flags & FS_CFLG_OWN_PAGES)
401 mutex_lock(&fscrypt_init_mutex);
402 if (fscrypt_bounce_page_pool)
403 goto already_initialized;
405 for (i = 0; i < num_prealloc_crypto_ctxs; i++) {
406 struct fscrypt_ctx *ctx;
408 ctx = kmem_cache_zalloc(fscrypt_ctx_cachep, GFP_NOFS);
411 list_add(&ctx->free_list, &fscrypt_free_ctxs);
414 fscrypt_bounce_page_pool =
415 mempool_create_page_pool(num_prealloc_crypto_pages, 0);
416 if (!fscrypt_bounce_page_pool)
420 mutex_unlock(&fscrypt_init_mutex);
424 mutex_unlock(&fscrypt_init_mutex);
428 void fscrypt_msg(struct super_block *sb, const char *level,
429 const char *fmt, ...)
431 static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
432 DEFAULT_RATELIMIT_BURST);
433 struct va_format vaf;
436 if (!__ratelimit(&rs))
443 printk("%sfscrypt (%s): %pV\n", level, sb->s_id, &vaf);
445 printk("%sfscrypt: %pV\n", level, &vaf);
450 * fscrypt_init() - Set up for fs encryption.
452 static int __init fscrypt_init(void)
455 * Use an unbound workqueue to allow bios to be decrypted in parallel
456 * even when they happen to complete on the same CPU. This sacrifices
457 * locality, but it's worthwhile since decryption is CPU-intensive.
459 * Also use a high-priority workqueue to prioritize decryption work,
460 * which blocks reads from completing, over regular application tasks.
462 fscrypt_read_workqueue = alloc_workqueue("fscrypt_read_queue",
463 WQ_UNBOUND | WQ_HIGHPRI,
465 if (!fscrypt_read_workqueue)
468 fscrypt_ctx_cachep = KMEM_CACHE(fscrypt_ctx, SLAB_RECLAIM_ACCOUNT);
469 if (!fscrypt_ctx_cachep)
470 goto fail_free_queue;
472 fscrypt_info_cachep = KMEM_CACHE(fscrypt_info, SLAB_RECLAIM_ACCOUNT);
473 if (!fscrypt_info_cachep)
479 kmem_cache_destroy(fscrypt_ctx_cachep);
481 destroy_workqueue(fscrypt_read_workqueue);
485 module_init(fscrypt_init)
488 * fscrypt_exit() - Shutdown the fs encryption system
490 static void __exit fscrypt_exit(void)
494 if (fscrypt_read_workqueue)
495 destroy_workqueue(fscrypt_read_workqueue);
496 kmem_cache_destroy(fscrypt_ctx_cachep);
497 kmem_cache_destroy(fscrypt_info_cachep);
499 fscrypt_essiv_cleanup();
501 module_exit(fscrypt_exit);
503 MODULE_LICENSE("GPL");