Commit | Line | Data |
---|---|---|
09c434b8 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
0b81d077 JK |
2 | /* |
3 | * This contains encryption functions for per-file encryption. | |
4 | * | |
5 | * Copyright (C) 2015, Google, Inc. | |
6 | * Copyright (C) 2015, Motorola Mobility | |
7 | * | |
8 | * Written by Michael Halcrow, 2014. | |
9 | * | |
10 | * Filename encryption additions | |
11 | * Uday Savagaonkar, 2014 | |
12 | * Encryption policy handling additions | |
13 | * Ildar Muslukhov, 2014 | |
14 | * Add fscrypt_pullback_bio_page() | |
15 | * Jaegeuk Kim, 2015. | |
16 | * | |
17 | * This has not yet undergone a rigorous security audit. | |
18 | * | |
19 | * The usage of AES-XTS should conform to recommendations in NIST | |
20 | * Special Publication 800-38E and IEEE P1619/D16. | |
21 | */ | |
22 | ||
0b81d077 JK |
23 | #include <linux/pagemap.h> |
24 | #include <linux/mempool.h> | |
25 | #include <linux/module.h> | |
26 | #include <linux/scatterlist.h> | |
27 | #include <linux/ratelimit.h> | |
a575784c | 28 | #include <crypto/skcipher.h> |
cc4e0df0 | 29 | #include "fscrypt_private.h" |
0b81d077 JK |
30 | |
31 | static unsigned int num_prealloc_crypto_pages = 32; | |
0b81d077 JK |
32 | |
33 | module_param(num_prealloc_crypto_pages, uint, 0444); | |
34 | MODULE_PARM_DESC(num_prealloc_crypto_pages, | |
35 | "Number of crypto pages to preallocate"); | |
0b81d077 JK |
36 | |
37 | static mempool_t *fscrypt_bounce_page_pool = NULL; | |
38 | ||
0cb8dae4 | 39 | static struct workqueue_struct *fscrypt_read_workqueue; |
0b81d077 JK |
40 | static DEFINE_MUTEX(fscrypt_init_mutex); |
41 | ||
0b81d077 JK |
42 | struct kmem_cache *fscrypt_info_cachep; |
43 | ||
0cb8dae4 EB |
44 | void fscrypt_enqueue_decrypt_work(struct work_struct *work) |
45 | { | |
46 | queue_work(fscrypt_read_workqueue, work); | |
47 | } | |
48 | EXPORT_SYMBOL(fscrypt_enqueue_decrypt_work); | |
49 | ||
d2d0727b EB |
50 | struct page *fscrypt_alloc_bounce_page(gfp_t gfp_flags) |
51 | { | |
52 | return mempool_alloc(fscrypt_bounce_page_pool, gfp_flags); | |
53 | } | |
54 | ||
55 | /** | |
56 | * fscrypt_free_bounce_page() - free a ciphertext bounce page | |
d2fe9754 | 57 | * @bounce_page: the bounce page to free, or NULL |
d2d0727b | 58 | * |
53bc1d85 EB |
59 | * Free a bounce page that was allocated by fscrypt_encrypt_pagecache_blocks(), |
60 | * or by fscrypt_alloc_bounce_page() directly. | |
d2d0727b EB |
61 | */ |
62 | void fscrypt_free_bounce_page(struct page *bounce_page) | |
63 | { | |
64 | if (!bounce_page) | |
65 | return; | |
66 | set_page_private(bounce_page, (unsigned long)NULL); | |
67 | ClearPagePrivate(bounce_page); | |
68 | mempool_free(bounce_page, fscrypt_bounce_page_pool); | |
69 | } | |
70 | EXPORT_SYMBOL(fscrypt_free_bounce_page); | |
71 | ||
c6c89783 EB |
72 | /* |
73 | * Generate the IV for the given logical block number within the given file. | |
74 | * For filenames encryption, lblk_num == 0. | |
75 | * | |
76 | * Keep this in sync with fscrypt_limit_io_blocks(). fscrypt_limit_io_blocks() | |
77 | * needs to know about any IV generation methods where the low bits of IV don't | |
78 | * simply contain the lblk_num (e.g., IV_INO_LBLK_32). | |
79 | */ | |
8094c3ce EB |
80 | void fscrypt_generate_iv(union fscrypt_iv *iv, u64 lblk_num, |
81 | const struct fscrypt_info *ci) | |
82 | { | |
b103fb76 EB |
83 | u8 flags = fscrypt_policy_flags(&ci->ci_policy); |
84 | ||
8094c3ce | 85 | memset(iv, 0, ci->ci_mode->ivsize); |
8094c3ce | 86 | |
b103fb76 | 87 | if (flags & FSCRYPT_POLICY_FLAG_IV_INO_LBLK_64) { |
e3b1078b EB |
88 | WARN_ON_ONCE(lblk_num > U32_MAX); |
89 | WARN_ON_ONCE(ci->ci_inode->i_ino > U32_MAX); | |
b103fb76 | 90 | lblk_num |= (u64)ci->ci_inode->i_ino << 32; |
e3b1078b EB |
91 | } else if (flags & FSCRYPT_POLICY_FLAG_IV_INO_LBLK_32) { |
92 | WARN_ON_ONCE(lblk_num > U32_MAX); | |
93 | lblk_num = (u32)(ci->ci_hashed_ino + lblk_num); | |
b103fb76 | 94 | } else if (flags & FSCRYPT_POLICY_FLAG_DIRECT_KEY) { |
1d6217a4 | 95 | memcpy(iv->nonce, ci->ci_nonce, FSCRYPT_FILE_NONCE_SIZE); |
b103fb76 EB |
96 | } |
97 | iv->lblk_num = cpu_to_le64(lblk_num); | |
8094c3ce EB |
98 | } |
99 | ||
f47fcbb2 EB |
100 | /* Encrypt or decrypt a single filesystem block of file contents */ |
101 | int fscrypt_crypt_block(const struct inode *inode, fscrypt_direction_t rw, | |
102 | u64 lblk_num, struct page *src_page, | |
103 | struct page *dest_page, unsigned int len, | |
104 | unsigned int offs, gfp_t gfp_flags) | |
0b81d077 | 105 | { |
8094c3ce | 106 | union fscrypt_iv iv; |
d407574e | 107 | struct skcipher_request *req = NULL; |
d0082e1a | 108 | DECLARE_CRYPTO_WAIT(wait); |
0b81d077 JK |
109 | struct scatterlist dst, src; |
110 | struct fscrypt_info *ci = inode->i_crypt_info; | |
5fee3609 | 111 | struct crypto_skcipher *tfm = ci->ci_enc_key.tfm; |
0b81d077 JK |
112 | int res = 0; |
113 | ||
eeacfdc6 EB |
114 | if (WARN_ON_ONCE(len <= 0)) |
115 | return -EINVAL; | |
63cec138 | 116 | if (WARN_ON_ONCE(len % FSCRYPT_CONTENTS_ALIGNMENT != 0)) |
eeacfdc6 | 117 | return -EINVAL; |
1400451f | 118 | |
8094c3ce | 119 | fscrypt_generate_iv(&iv, lblk_num, ci); |
b7e7cf7a | 120 | |
b32e4482 | 121 | req = skcipher_request_alloc(tfm, gfp_flags); |
c90fd775 | 122 | if (!req) |
0b81d077 | 123 | return -ENOMEM; |
0b81d077 | 124 | |
d407574e | 125 | skcipher_request_set_callback( |
0b81d077 | 126 | req, CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP, |
d0082e1a | 127 | crypto_req_done, &wait); |
0b81d077 | 128 | |
0b81d077 | 129 | sg_init_table(&dst, 1); |
1400451f | 130 | sg_set_page(&dst, dest_page, len, offs); |
0b81d077 | 131 | sg_init_table(&src, 1); |
1400451f | 132 | sg_set_page(&src, src_page, len, offs); |
b7e7cf7a | 133 | skcipher_request_set_crypt(req, &src, &dst, len, &iv); |
0b81d077 | 134 | if (rw == FS_DECRYPT) |
d0082e1a | 135 | res = crypto_wait_req(crypto_skcipher_decrypt(req), &wait); |
0b81d077 | 136 | else |
d0082e1a | 137 | res = crypto_wait_req(crypto_skcipher_encrypt(req), &wait); |
d407574e | 138 | skcipher_request_free(req); |
0b81d077 | 139 | if (res) { |
886da8b3 EB |
140 | fscrypt_err(inode, "%scryption failed for block %llu: %d", |
141 | (rw == FS_DECRYPT ? "De" : "En"), lblk_num, res); | |
0b81d077 JK |
142 | return res; |
143 | } | |
144 | return 0; | |
145 | } | |
146 | ||
0b81d077 | 147 | /** |
d2fe9754 EB |
148 | * fscrypt_encrypt_pagecache_blocks() - Encrypt filesystem blocks from a |
149 | * pagecache page | |
53bc1d85 EB |
150 | * @page: The locked pagecache page containing the block(s) to encrypt |
151 | * @len: Total size of the block(s) to encrypt. Must be a nonzero | |
152 | * multiple of the filesystem's block size. | |
153 | * @offs: Byte offset within @page of the first block to encrypt. Must be | |
154 | * a multiple of the filesystem's block size. | |
2d8f7f11 | 155 | * @gfp_flags: Memory allocation flags. See details below. |
53bc1d85 EB |
156 | * |
157 | * A new bounce page is allocated, and the specified block(s) are encrypted into | |
158 | * it. In the bounce page, the ciphertext block(s) will be located at the same | |
159 | * offsets at which the plaintext block(s) were located in the source page; any | |
160 | * other parts of the bounce page will be left uninitialized. However, normally | |
161 | * blocksize == PAGE_SIZE and the whole page is encrypted at once. | |
0b81d077 | 162 | * |
53bc1d85 | 163 | * This is for use by the filesystem's ->writepages() method. |
0b81d077 | 164 | * |
2d8f7f11 EB |
165 | * The bounce page allocation is mempool-backed, so it will always succeed when |
166 | * @gfp_flags includes __GFP_DIRECT_RECLAIM, e.g. when it's GFP_NOFS. However, | |
167 | * only the first page of each bio can be allocated this way. To prevent | |
168 | * deadlocks, for any additional pages a mask like GFP_NOWAIT must be used. | |
169 | * | |
53bc1d85 | 170 | * Return: the new encrypted bounce page on success; an ERR_PTR() on failure |
0b81d077 | 171 | */ |
53bc1d85 EB |
172 | struct page *fscrypt_encrypt_pagecache_blocks(struct page *page, |
173 | unsigned int len, | |
174 | unsigned int offs, | |
175 | gfp_t gfp_flags) | |
7821d4dd | 176 | |
0b81d077 | 177 | { |
53bc1d85 EB |
178 | const struct inode *inode = page->mapping->host; |
179 | const unsigned int blockbits = inode->i_blkbits; | |
180 | const unsigned int blocksize = 1 << blockbits; | |
03569f2f | 181 | struct page *ciphertext_page; |
53bc1d85 EB |
182 | u64 lblk_num = ((u64)page->index << (PAGE_SHIFT - blockbits)) + |
183 | (offs >> blockbits); | |
184 | unsigned int i; | |
0b81d077 JK |
185 | int err; |
186 | ||
eeacfdc6 EB |
187 | if (WARN_ON_ONCE(!PageLocked(page))) |
188 | return ERR_PTR(-EINVAL); | |
bd7b8290 | 189 | |
53bc1d85 EB |
190 | if (WARN_ON_ONCE(len <= 0 || !IS_ALIGNED(len | offs, blocksize))) |
191 | return ERR_PTR(-EINVAL); | |
192 | ||
d2d0727b EB |
193 | ciphertext_page = fscrypt_alloc_bounce_page(gfp_flags); |
194 | if (!ciphertext_page) | |
195 | return ERR_PTR(-ENOMEM); | |
0b81d077 | 196 | |
53bc1d85 EB |
197 | for (i = offs; i < offs + len; i += blocksize, lblk_num++) { |
198 | err = fscrypt_crypt_block(inode, FS_ENCRYPT, lblk_num, | |
199 | page, ciphertext_page, | |
200 | blocksize, i, gfp_flags); | |
201 | if (err) { | |
202 | fscrypt_free_bounce_page(ciphertext_page); | |
203 | return ERR_PTR(err); | |
204 | } | |
0b81d077 | 205 | } |
9e532772 | 206 | SetPagePrivate(ciphertext_page); |
d2d0727b | 207 | set_page_private(ciphertext_page, (unsigned long)page); |
0b81d077 JK |
208 | return ciphertext_page; |
209 | } | |
53bc1d85 | 210 | EXPORT_SYMBOL(fscrypt_encrypt_pagecache_blocks); |
0b81d077 | 211 | |
03569f2f EB |
212 | /** |
213 | * fscrypt_encrypt_block_inplace() - Encrypt a filesystem block in-place | |
214 | * @inode: The inode to which this block belongs | |
215 | * @page: The page containing the block to encrypt | |
63cec138 EB |
216 | * @len: Size of block to encrypt. This must be a multiple of |
217 | * FSCRYPT_CONTENTS_ALIGNMENT. | |
03569f2f EB |
218 | * @offs: Byte offset within @page at which the block to encrypt begins |
219 | * @lblk_num: Filesystem logical block number of the block, i.e. the 0-based | |
220 | * number of the block within the file | |
221 | * @gfp_flags: Memory allocation flags | |
222 | * | |
223 | * Encrypt a possibly-compressed filesystem block that is located in an | |
224 | * arbitrary page, not necessarily in the original pagecache page. The @inode | |
225 | * and @lblk_num must be specified, as they can't be determined from @page. | |
226 | * | |
227 | * Return: 0 on success; -errno on failure | |
228 | */ | |
229 | int fscrypt_encrypt_block_inplace(const struct inode *inode, struct page *page, | |
230 | unsigned int len, unsigned int offs, | |
231 | u64 lblk_num, gfp_t gfp_flags) | |
232 | { | |
233 | return fscrypt_crypt_block(inode, FS_ENCRYPT, lblk_num, page, page, | |
234 | len, offs, gfp_flags); | |
235 | } | |
236 | EXPORT_SYMBOL(fscrypt_encrypt_block_inplace); | |
237 | ||
0b81d077 | 238 | /** |
d2fe9754 | 239 | * fscrypt_decrypt_pagecache_blocks() - Decrypt filesystem blocks in a |
51e4e315 EB |
240 | * pagecache folio |
241 | * @folio: The locked pagecache folio containing the block(s) to decrypt | |
aa8bc1ac EB |
242 | * @len: Total size of the block(s) to decrypt. Must be a nonzero |
243 | * multiple of the filesystem's block size. | |
51e4e315 | 244 | * @offs: Byte offset within @folio of the first block to decrypt. Must be |
aa8bc1ac | 245 | * a multiple of the filesystem's block size. |
0b81d077 | 246 | * |
51e4e315 EB |
247 | * The specified block(s) are decrypted in-place within the pagecache folio, |
248 | * which must still be locked and not uptodate. | |
0b81d077 | 249 | * |
704528d8 | 250 | * This is for use by the filesystem's ->readahead() method. |
0b81d077 | 251 | * |
aa8bc1ac | 252 | * Return: 0 on success; -errno on failure |
0b81d077 | 253 | */ |
51e4e315 EB |
254 | int fscrypt_decrypt_pagecache_blocks(struct folio *folio, size_t len, |
255 | size_t offs) | |
0b81d077 | 256 | { |
51e4e315 | 257 | const struct inode *inode = folio->mapping->host; |
aa8bc1ac EB |
258 | const unsigned int blockbits = inode->i_blkbits; |
259 | const unsigned int blocksize = 1 << blockbits; | |
51e4e315 | 260 | u64 lblk_num = ((u64)folio->index << (PAGE_SHIFT - blockbits)) + |
aa8bc1ac | 261 | (offs >> blockbits); |
51e4e315 | 262 | size_t i; |
aa8bc1ac EB |
263 | int err; |
264 | ||
51e4e315 | 265 | if (WARN_ON_ONCE(!folio_test_locked(folio))) |
eeacfdc6 | 266 | return -EINVAL; |
bd7b8290 | 267 | |
aa8bc1ac EB |
268 | if (WARN_ON_ONCE(len <= 0 || !IS_ALIGNED(len | offs, blocksize))) |
269 | return -EINVAL; | |
270 | ||
271 | for (i = offs; i < offs + len; i += blocksize, lblk_num++) { | |
51e4e315 EB |
272 | struct page *page = folio_page(folio, i >> PAGE_SHIFT); |
273 | ||
aa8bc1ac | 274 | err = fscrypt_crypt_block(inode, FS_DECRYPT, lblk_num, page, |
51e4e315 EB |
275 | page, blocksize, i & ~PAGE_MASK, |
276 | GFP_NOFS); | |
aa8bc1ac EB |
277 | if (err) |
278 | return err; | |
279 | } | |
280 | return 0; | |
0b81d077 | 281 | } |
aa8bc1ac | 282 | EXPORT_SYMBOL(fscrypt_decrypt_pagecache_blocks); |
0b81d077 | 283 | |
41adbcb7 EB |
284 | /** |
285 | * fscrypt_decrypt_block_inplace() - Decrypt a filesystem block in-place | |
286 | * @inode: The inode to which this block belongs | |
287 | * @page: The page containing the block to decrypt | |
63cec138 EB |
288 | * @len: Size of block to decrypt. This must be a multiple of |
289 | * FSCRYPT_CONTENTS_ALIGNMENT. | |
41adbcb7 EB |
290 | * @offs: Byte offset within @page at which the block to decrypt begins |
291 | * @lblk_num: Filesystem logical block number of the block, i.e. the 0-based | |
292 | * number of the block within the file | |
293 | * | |
294 | * Decrypt a possibly-compressed filesystem block that is located in an | |
295 | * arbitrary page, not necessarily in the original pagecache page. The @inode | |
296 | * and @lblk_num must be specified, as they can't be determined from @page. | |
297 | * | |
298 | * Return: 0 on success; -errno on failure | |
299 | */ | |
300 | int fscrypt_decrypt_block_inplace(const struct inode *inode, struct page *page, | |
301 | unsigned int len, unsigned int offs, | |
302 | u64 lblk_num) | |
303 | { | |
304 | return fscrypt_crypt_block(inode, FS_DECRYPT, lblk_num, page, page, | |
305 | len, offs, GFP_NOFS); | |
306 | } | |
307 | EXPORT_SYMBOL(fscrypt_decrypt_block_inplace); | |
308 | ||
0b81d077 JK |
309 | /** |
310 | * fscrypt_initialize() - allocate major buffers for fs encryption. | |
83e57e47 | 311 | * @sb: the filesystem superblock |
0b81d077 JK |
312 | * |
313 | * We only call this when we start accessing encrypted files, since it | |
314 | * results in memory getting allocated that wouldn't otherwise be used. | |
315 | * | |
1565bdad | 316 | * Return: 0 on success; -errno on failure |
0b81d077 | 317 | */ |
83e57e47 | 318 | int fscrypt_initialize(struct super_block *sb) |
0b81d077 | 319 | { |
1565bdad | 320 | int err = 0; |
83e57e47 EB |
321 | mempool_t *pool; |
322 | ||
323 | /* pairs with smp_store_release() below */ | |
324 | if (likely(smp_load_acquire(&fscrypt_bounce_page_pool))) | |
325 | return 0; | |
0b81d077 | 326 | |
a0b3bc85 | 327 | /* No need to allocate a bounce page pool if this FS won't use it. */ |
83e57e47 | 328 | if (sb->s_cop->flags & FS_CFLG_OWN_PAGES) |
0b81d077 JK |
329 | return 0; |
330 | ||
331 | mutex_lock(&fscrypt_init_mutex); | |
332 | if (fscrypt_bounce_page_pool) | |
1565bdad | 333 | goto out_unlock; |
0b81d077 | 334 | |
1565bdad | 335 | err = -ENOMEM; |
83e57e47 EB |
336 | pool = mempool_create_page_pool(num_prealloc_crypto_pages, 0); |
337 | if (!pool) | |
1565bdad | 338 | goto out_unlock; |
83e57e47 EB |
339 | /* pairs with smp_load_acquire() above */ |
340 | smp_store_release(&fscrypt_bounce_page_pool, pool); | |
1565bdad EB |
341 | err = 0; |
342 | out_unlock: | |
0b81d077 | 343 | mutex_unlock(&fscrypt_init_mutex); |
1565bdad | 344 | return err; |
0b81d077 | 345 | } |
0b81d077 | 346 | |
886da8b3 | 347 | void fscrypt_msg(const struct inode *inode, const char *level, |
544d08fd EB |
348 | const char *fmt, ...) |
349 | { | |
350 | static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL, | |
351 | DEFAULT_RATELIMIT_BURST); | |
352 | struct va_format vaf; | |
353 | va_list args; | |
354 | ||
355 | if (!__ratelimit(&rs)) | |
356 | return; | |
357 | ||
358 | va_start(args, fmt); | |
359 | vaf.fmt = fmt; | |
360 | vaf.va = &args; | |
ae9ff8ad | 361 | if (inode && inode->i_ino) |
886da8b3 EB |
362 | printk("%sfscrypt (%s, inode %lu): %pV\n", |
363 | level, inode->i_sb->s_id, inode->i_ino, &vaf); | |
ae9ff8ad EB |
364 | else if (inode) |
365 | printk("%sfscrypt (%s): %pV\n", level, inode->i_sb->s_id, &vaf); | |
544d08fd EB |
366 | else |
367 | printk("%sfscrypt: %pV\n", level, &vaf); | |
368 | va_end(args); | |
369 | } | |
370 | ||
0b81d077 JK |
371 | /** |
372 | * fscrypt_init() - Set up for fs encryption. | |
d2fe9754 EB |
373 | * |
374 | * Return: 0 on success; -errno on failure | |
0b81d077 JK |
375 | */ |
376 | static int __init fscrypt_init(void) | |
377 | { | |
22d94f49 EB |
378 | int err = -ENOMEM; |
379 | ||
36dd26e0 EB |
380 | /* |
381 | * Use an unbound workqueue to allow bios to be decrypted in parallel | |
382 | * even when they happen to complete on the same CPU. This sacrifices | |
383 | * locality, but it's worthwhile since decryption is CPU-intensive. | |
384 | * | |
385 | * Also use a high-priority workqueue to prioritize decryption work, | |
386 | * which blocks reads from completing, over regular application tasks. | |
387 | */ | |
0b81d077 | 388 | fscrypt_read_workqueue = alloc_workqueue("fscrypt_read_queue", |
36dd26e0 EB |
389 | WQ_UNBOUND | WQ_HIGHPRI, |
390 | num_online_cpus()); | |
0b81d077 JK |
391 | if (!fscrypt_read_workqueue) |
392 | goto fail; | |
393 | ||
0b81d077 JK |
394 | fscrypt_info_cachep = KMEM_CACHE(fscrypt_info, SLAB_RECLAIM_ACCOUNT); |
395 | if (!fscrypt_info_cachep) | |
1565bdad | 396 | goto fail_free_queue; |
0b81d077 | 397 | |
22d94f49 EB |
398 | err = fscrypt_init_keyring(); |
399 | if (err) | |
400 | goto fail_free_info; | |
401 | ||
0b81d077 JK |
402 | return 0; |
403 | ||
22d94f49 EB |
404 | fail_free_info: |
405 | kmem_cache_destroy(fscrypt_info_cachep); | |
0b81d077 JK |
406 | fail_free_queue: |
407 | destroy_workqueue(fscrypt_read_workqueue); | |
408 | fail: | |
22d94f49 | 409 | return err; |
0b81d077 | 410 | } |
75798f85 | 411 | late_initcall(fscrypt_init) |