| 1 | // SPDX-License-Identifier: GPL-2.0-only |
| 2 | /* |
| 3 | * This contains encryption functions for per-file encryption. |
| 4 | * |
| 5 | * Copyright (C) 2015, Google, Inc. |
| 6 | * Copyright (C) 2015, Motorola Mobility |
| 7 | * |
| 8 | * Written by Michael Halcrow, 2014. |
| 9 | * |
| 10 | * Filename encryption additions |
| 11 | * Uday Savagaonkar, 2014 |
| 12 | * Encryption policy handling additions |
| 13 | * Ildar Muslukhov, 2014 |
| 14 | * Add fscrypt_pullback_bio_page() |
| 15 | * Jaegeuk Kim, 2015. |
| 16 | * |
| 17 | * This has not yet undergone a rigorous security audit. |
| 18 | * |
| 19 | * The usage of AES-XTS should conform to recommendations in NIST |
| 20 | * Special Publication 800-38E and IEEE P1619/D16. |
| 21 | */ |
| 22 | |
| 23 | #include <linux/pagemap.h> |
| 24 | #include <linux/mempool.h> |
| 25 | #include <linux/module.h> |
| 26 | #include <linux/scatterlist.h> |
| 27 | #include <linux/ratelimit.h> |
| 28 | #include <crypto/skcipher.h> |
| 29 | #include "fscrypt_private.h" |
| 30 | |
| 31 | static unsigned int num_prealloc_crypto_pages = 32; |
| 32 | |
| 33 | module_param(num_prealloc_crypto_pages, uint, 0444); |
| 34 | MODULE_PARM_DESC(num_prealloc_crypto_pages, |
| 35 | "Number of crypto pages to preallocate"); |
| 36 | |
| 37 | static mempool_t *fscrypt_bounce_page_pool = NULL; |
| 38 | |
| 39 | static struct workqueue_struct *fscrypt_read_workqueue; |
| 40 | static DEFINE_MUTEX(fscrypt_init_mutex); |
| 41 | |
| 42 | struct kmem_cache *fscrypt_inode_info_cachep; |
| 43 | |
| 44 | void fscrypt_enqueue_decrypt_work(struct work_struct *work) |
| 45 | { |
| 46 | queue_work(fscrypt_read_workqueue, work); |
| 47 | } |
| 48 | EXPORT_SYMBOL(fscrypt_enqueue_decrypt_work); |
| 49 | |
| 50 | struct page *fscrypt_alloc_bounce_page(gfp_t gfp_flags) |
| 51 | { |
| 52 | if (WARN_ON_ONCE(!fscrypt_bounce_page_pool)) { |
| 53 | /* |
| 54 | * Oops, the filesystem called a function that uses the bounce |
| 55 | * page pool, but it didn't set needs_bounce_pages. |
| 56 | */ |
| 57 | return NULL; |
| 58 | } |
| 59 | return mempool_alloc(fscrypt_bounce_page_pool, gfp_flags); |
| 60 | } |
| 61 | |
| 62 | /** |
| 63 | * fscrypt_free_bounce_page() - free a ciphertext bounce page |
| 64 | * @bounce_page: the bounce page to free, or NULL |
| 65 | * |
| 66 | * Free a bounce page that was allocated by fscrypt_encrypt_pagecache_blocks(), |
| 67 | * or by fscrypt_alloc_bounce_page() directly. |
| 68 | */ |
| 69 | void fscrypt_free_bounce_page(struct page *bounce_page) |
| 70 | { |
| 71 | if (!bounce_page) |
| 72 | return; |
| 73 | set_page_private(bounce_page, (unsigned long)NULL); |
| 74 | ClearPagePrivate(bounce_page); |
| 75 | mempool_free(bounce_page, fscrypt_bounce_page_pool); |
| 76 | } |
| 77 | EXPORT_SYMBOL(fscrypt_free_bounce_page); |
| 78 | |
| 79 | /* |
| 80 | * Generate the IV for the given data unit index within the given file. |
| 81 | * For filenames encryption, index == 0. |
| 82 | * |
| 83 | * Keep this in sync with fscrypt_limit_io_blocks(). fscrypt_limit_io_blocks() |
| 84 | * needs to know about any IV generation methods where the low bits of IV don't |
| 85 | * simply contain the data unit index (e.g., IV_INO_LBLK_32). |
| 86 | */ |
| 87 | void fscrypt_generate_iv(union fscrypt_iv *iv, u64 index, |
| 88 | const struct fscrypt_inode_info *ci) |
| 89 | { |
| 90 | u8 flags = fscrypt_policy_flags(&ci->ci_policy); |
| 91 | |
| 92 | memset(iv, 0, ci->ci_mode->ivsize); |
| 93 | |
| 94 | if (flags & FSCRYPT_POLICY_FLAG_IV_INO_LBLK_64) { |
| 95 | WARN_ON_ONCE(index > U32_MAX); |
| 96 | WARN_ON_ONCE(ci->ci_inode->i_ino > U32_MAX); |
| 97 | index |= (u64)ci->ci_inode->i_ino << 32; |
| 98 | } else if (flags & FSCRYPT_POLICY_FLAG_IV_INO_LBLK_32) { |
| 99 | WARN_ON_ONCE(index > U32_MAX); |
| 100 | index = (u32)(ci->ci_hashed_ino + index); |
| 101 | } else if (flags & FSCRYPT_POLICY_FLAG_DIRECT_KEY) { |
| 102 | memcpy(iv->nonce, ci->ci_nonce, FSCRYPT_FILE_NONCE_SIZE); |
| 103 | } |
| 104 | iv->index = cpu_to_le64(index); |
| 105 | } |
| 106 | |
| 107 | /* Encrypt or decrypt a single "data unit" of file contents. */ |
| 108 | int fscrypt_crypt_data_unit(const struct fscrypt_inode_info *ci, |
| 109 | fscrypt_direction_t rw, u64 index, |
| 110 | struct page *src_page, struct page *dest_page, |
| 111 | unsigned int len, unsigned int offs, |
| 112 | gfp_t gfp_flags) |
| 113 | { |
| 114 | union fscrypt_iv iv; |
| 115 | struct skcipher_request *req = NULL; |
| 116 | DECLARE_CRYPTO_WAIT(wait); |
| 117 | struct scatterlist dst, src; |
| 118 | struct crypto_skcipher *tfm = ci->ci_enc_key.tfm; |
| 119 | int res = 0; |
| 120 | |
| 121 | if (WARN_ON_ONCE(len <= 0)) |
| 122 | return -EINVAL; |
| 123 | if (WARN_ON_ONCE(len % FSCRYPT_CONTENTS_ALIGNMENT != 0)) |
| 124 | return -EINVAL; |
| 125 | |
| 126 | fscrypt_generate_iv(&iv, index, ci); |
| 127 | |
| 128 | req = skcipher_request_alloc(tfm, gfp_flags); |
| 129 | if (!req) |
| 130 | return -ENOMEM; |
| 131 | |
| 132 | skcipher_request_set_callback( |
| 133 | req, CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP, |
| 134 | crypto_req_done, &wait); |
| 135 | |
| 136 | sg_init_table(&dst, 1); |
| 137 | sg_set_page(&dst, dest_page, len, offs); |
| 138 | sg_init_table(&src, 1); |
| 139 | sg_set_page(&src, src_page, len, offs); |
| 140 | skcipher_request_set_crypt(req, &src, &dst, len, &iv); |
| 141 | if (rw == FS_DECRYPT) |
| 142 | res = crypto_wait_req(crypto_skcipher_decrypt(req), &wait); |
| 143 | else |
| 144 | res = crypto_wait_req(crypto_skcipher_encrypt(req), &wait); |
| 145 | skcipher_request_free(req); |
| 146 | if (res) { |
| 147 | fscrypt_err(ci->ci_inode, |
| 148 | "%scryption failed for data unit %llu: %d", |
| 149 | (rw == FS_DECRYPT ? "De" : "En"), index, res); |
| 150 | return res; |
| 151 | } |
| 152 | return 0; |
| 153 | } |
| 154 | |
| 155 | /** |
| 156 | * fscrypt_encrypt_pagecache_blocks() - Encrypt data from a pagecache page |
| 157 | * @page: the locked pagecache page containing the data to encrypt |
| 158 | * @len: size of the data to encrypt, in bytes |
| 159 | * @offs: offset within @page of the data to encrypt, in bytes |
| 160 | * @gfp_flags: memory allocation flags; see details below |
| 161 | * |
| 162 | * This allocates a new bounce page and encrypts the given data into it. The |
| 163 | * length and offset of the data must be aligned to the file's crypto data unit |
| 164 | * size. Alignment to the filesystem block size fulfills this requirement, as |
| 165 | * the filesystem block size is always a multiple of the data unit size. |
| 166 | * |
| 167 | * In the bounce page, the ciphertext data will be located at the same offset at |
| 168 | * which the plaintext data was located in the source page. Any other parts of |
| 169 | * the bounce page will be left uninitialized. |
| 170 | * |
| 171 | * This is for use by the filesystem's ->writepages() method. |
| 172 | * |
| 173 | * The bounce page allocation is mempool-backed, so it will always succeed when |
| 174 | * @gfp_flags includes __GFP_DIRECT_RECLAIM, e.g. when it's GFP_NOFS. However, |
| 175 | * only the first page of each bio can be allocated this way. To prevent |
| 176 | * deadlocks, for any additional pages a mask like GFP_NOWAIT must be used. |
| 177 | * |
| 178 | * Return: the new encrypted bounce page on success; an ERR_PTR() on failure |
| 179 | */ |
| 180 | struct page *fscrypt_encrypt_pagecache_blocks(struct page *page, |
| 181 | unsigned int len, |
| 182 | unsigned int offs, |
| 183 | gfp_t gfp_flags) |
| 184 | |
| 185 | { |
| 186 | const struct inode *inode = page->mapping->host; |
| 187 | const struct fscrypt_inode_info *ci = inode->i_crypt_info; |
| 188 | const unsigned int du_bits = ci->ci_data_unit_bits; |
| 189 | const unsigned int du_size = 1U << du_bits; |
| 190 | struct page *ciphertext_page; |
| 191 | u64 index = ((u64)page->index << (PAGE_SHIFT - du_bits)) + |
| 192 | (offs >> du_bits); |
| 193 | unsigned int i; |
| 194 | int err; |
| 195 | |
| 196 | if (WARN_ON_ONCE(!PageLocked(page))) |
| 197 | return ERR_PTR(-EINVAL); |
| 198 | |
| 199 | if (WARN_ON_ONCE(len <= 0 || !IS_ALIGNED(len | offs, du_size))) |
| 200 | return ERR_PTR(-EINVAL); |
| 201 | |
| 202 | ciphertext_page = fscrypt_alloc_bounce_page(gfp_flags); |
| 203 | if (!ciphertext_page) |
| 204 | return ERR_PTR(-ENOMEM); |
| 205 | |
| 206 | for (i = offs; i < offs + len; i += du_size, index++) { |
| 207 | err = fscrypt_crypt_data_unit(ci, FS_ENCRYPT, index, |
| 208 | page, ciphertext_page, |
| 209 | du_size, i, gfp_flags); |
| 210 | if (err) { |
| 211 | fscrypt_free_bounce_page(ciphertext_page); |
| 212 | return ERR_PTR(err); |
| 213 | } |
| 214 | } |
| 215 | SetPagePrivate(ciphertext_page); |
| 216 | set_page_private(ciphertext_page, (unsigned long)page); |
| 217 | return ciphertext_page; |
| 218 | } |
| 219 | EXPORT_SYMBOL(fscrypt_encrypt_pagecache_blocks); |
| 220 | |
| 221 | /** |
| 222 | * fscrypt_encrypt_block_inplace() - Encrypt a filesystem block in-place |
| 223 | * @inode: The inode to which this block belongs |
| 224 | * @page: The page containing the block to encrypt |
| 225 | * @len: Size of block to encrypt. This must be a multiple of |
| 226 | * FSCRYPT_CONTENTS_ALIGNMENT. |
| 227 | * @offs: Byte offset within @page at which the block to encrypt begins |
| 228 | * @lblk_num: Filesystem logical block number of the block, i.e. the 0-based |
| 229 | * number of the block within the file |
| 230 | * @gfp_flags: Memory allocation flags |
| 231 | * |
| 232 | * Encrypt a possibly-compressed filesystem block that is located in an |
| 233 | * arbitrary page, not necessarily in the original pagecache page. The @inode |
| 234 | * and @lblk_num must be specified, as they can't be determined from @page. |
| 235 | * |
| 236 | * This is not compatible with fscrypt_operations::supports_subblock_data_units. |
| 237 | * |
| 238 | * Return: 0 on success; -errno on failure |
| 239 | */ |
| 240 | int fscrypt_encrypt_block_inplace(const struct inode *inode, struct page *page, |
| 241 | unsigned int len, unsigned int offs, |
| 242 | u64 lblk_num, gfp_t gfp_flags) |
| 243 | { |
| 244 | if (WARN_ON_ONCE(inode->i_sb->s_cop->supports_subblock_data_units)) |
| 245 | return -EOPNOTSUPP; |
| 246 | return fscrypt_crypt_data_unit(inode->i_crypt_info, FS_ENCRYPT, |
| 247 | lblk_num, page, page, len, offs, |
| 248 | gfp_flags); |
| 249 | } |
| 250 | EXPORT_SYMBOL(fscrypt_encrypt_block_inplace); |
| 251 | |
| 252 | /** |
| 253 | * fscrypt_decrypt_pagecache_blocks() - Decrypt data from a pagecache folio |
| 254 | * @folio: the pagecache folio containing the data to decrypt |
| 255 | * @len: size of the data to decrypt, in bytes |
| 256 | * @offs: offset within @folio of the data to decrypt, in bytes |
| 257 | * |
| 258 | * Decrypt data that has just been read from an encrypted file. The data must |
| 259 | * be located in a pagecache folio that is still locked and not yet uptodate. |
| 260 | * The length and offset of the data must be aligned to the file's crypto data |
| 261 | * unit size. Alignment to the filesystem block size fulfills this requirement, |
| 262 | * as the filesystem block size is always a multiple of the data unit size. |
| 263 | * |
| 264 | * Return: 0 on success; -errno on failure |
| 265 | */ |
| 266 | int fscrypt_decrypt_pagecache_blocks(struct folio *folio, size_t len, |
| 267 | size_t offs) |
| 268 | { |
| 269 | const struct inode *inode = folio->mapping->host; |
| 270 | const struct fscrypt_inode_info *ci = inode->i_crypt_info; |
| 271 | const unsigned int du_bits = ci->ci_data_unit_bits; |
| 272 | const unsigned int du_size = 1U << du_bits; |
| 273 | u64 index = ((u64)folio->index << (PAGE_SHIFT - du_bits)) + |
| 274 | (offs >> du_bits); |
| 275 | size_t i; |
| 276 | int err; |
| 277 | |
| 278 | if (WARN_ON_ONCE(!folio_test_locked(folio))) |
| 279 | return -EINVAL; |
| 280 | |
| 281 | if (WARN_ON_ONCE(len <= 0 || !IS_ALIGNED(len | offs, du_size))) |
| 282 | return -EINVAL; |
| 283 | |
| 284 | for (i = offs; i < offs + len; i += du_size, index++) { |
| 285 | struct page *page = folio_page(folio, i >> PAGE_SHIFT); |
| 286 | |
| 287 | err = fscrypt_crypt_data_unit(ci, FS_DECRYPT, index, page, |
| 288 | page, du_size, i & ~PAGE_MASK, |
| 289 | GFP_NOFS); |
| 290 | if (err) |
| 291 | return err; |
| 292 | } |
| 293 | return 0; |
| 294 | } |
| 295 | EXPORT_SYMBOL(fscrypt_decrypt_pagecache_blocks); |
| 296 | |
| 297 | /** |
| 298 | * fscrypt_decrypt_block_inplace() - Decrypt a filesystem block in-place |
| 299 | * @inode: The inode to which this block belongs |
| 300 | * @page: The page containing the block to decrypt |
| 301 | * @len: Size of block to decrypt. This must be a multiple of |
| 302 | * FSCRYPT_CONTENTS_ALIGNMENT. |
| 303 | * @offs: Byte offset within @page at which the block to decrypt begins |
| 304 | * @lblk_num: Filesystem logical block number of the block, i.e. the 0-based |
| 305 | * number of the block within the file |
| 306 | * |
| 307 | * Decrypt a possibly-compressed filesystem block that is located in an |
| 308 | * arbitrary page, not necessarily in the original pagecache page. The @inode |
| 309 | * and @lblk_num must be specified, as they can't be determined from @page. |
| 310 | * |
| 311 | * This is not compatible with fscrypt_operations::supports_subblock_data_units. |
| 312 | * |
| 313 | * Return: 0 on success; -errno on failure |
| 314 | */ |
| 315 | int fscrypt_decrypt_block_inplace(const struct inode *inode, struct page *page, |
| 316 | unsigned int len, unsigned int offs, |
| 317 | u64 lblk_num) |
| 318 | { |
| 319 | if (WARN_ON_ONCE(inode->i_sb->s_cop->supports_subblock_data_units)) |
| 320 | return -EOPNOTSUPP; |
| 321 | return fscrypt_crypt_data_unit(inode->i_crypt_info, FS_DECRYPT, |
| 322 | lblk_num, page, page, len, offs, |
| 323 | GFP_NOFS); |
| 324 | } |
| 325 | EXPORT_SYMBOL(fscrypt_decrypt_block_inplace); |
| 326 | |
| 327 | /** |
| 328 | * fscrypt_initialize() - allocate major buffers for fs encryption. |
| 329 | * @sb: the filesystem superblock |
| 330 | * |
| 331 | * We only call this when we start accessing encrypted files, since it |
| 332 | * results in memory getting allocated that wouldn't otherwise be used. |
| 333 | * |
| 334 | * Return: 0 on success; -errno on failure |
| 335 | */ |
| 336 | int fscrypt_initialize(struct super_block *sb) |
| 337 | { |
| 338 | int err = 0; |
| 339 | mempool_t *pool; |
| 340 | |
| 341 | /* pairs with smp_store_release() below */ |
| 342 | if (likely(smp_load_acquire(&fscrypt_bounce_page_pool))) |
| 343 | return 0; |
| 344 | |
| 345 | /* No need to allocate a bounce page pool if this FS won't use it. */ |
| 346 | if (!sb->s_cop->needs_bounce_pages) |
| 347 | return 0; |
| 348 | |
| 349 | mutex_lock(&fscrypt_init_mutex); |
| 350 | if (fscrypt_bounce_page_pool) |
| 351 | goto out_unlock; |
| 352 | |
| 353 | err = -ENOMEM; |
| 354 | pool = mempool_create_page_pool(num_prealloc_crypto_pages, 0); |
| 355 | if (!pool) |
| 356 | goto out_unlock; |
| 357 | /* pairs with smp_load_acquire() above */ |
| 358 | smp_store_release(&fscrypt_bounce_page_pool, pool); |
| 359 | err = 0; |
| 360 | out_unlock: |
| 361 | mutex_unlock(&fscrypt_init_mutex); |
| 362 | return err; |
| 363 | } |
| 364 | |
| 365 | void fscrypt_msg(const struct inode *inode, const char *level, |
| 366 | const char *fmt, ...) |
| 367 | { |
| 368 | static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL, |
| 369 | DEFAULT_RATELIMIT_BURST); |
| 370 | struct va_format vaf; |
| 371 | va_list args; |
| 372 | |
| 373 | if (!__ratelimit(&rs)) |
| 374 | return; |
| 375 | |
| 376 | va_start(args, fmt); |
| 377 | vaf.fmt = fmt; |
| 378 | vaf.va = &args; |
| 379 | if (inode && inode->i_ino) |
| 380 | printk("%sfscrypt (%s, inode %lu): %pV\n", |
| 381 | level, inode->i_sb->s_id, inode->i_ino, &vaf); |
| 382 | else if (inode) |
| 383 | printk("%sfscrypt (%s): %pV\n", level, inode->i_sb->s_id, &vaf); |
| 384 | else |
| 385 | printk("%sfscrypt: %pV\n", level, &vaf); |
| 386 | va_end(args); |
| 387 | } |
| 388 | |
| 389 | /** |
| 390 | * fscrypt_init() - Set up for fs encryption. |
| 391 | * |
| 392 | * Return: 0 on success; -errno on failure |
| 393 | */ |
| 394 | static int __init fscrypt_init(void) |
| 395 | { |
| 396 | int err = -ENOMEM; |
| 397 | |
| 398 | /* |
| 399 | * Use an unbound workqueue to allow bios to be decrypted in parallel |
| 400 | * even when they happen to complete on the same CPU. This sacrifices |
| 401 | * locality, but it's worthwhile since decryption is CPU-intensive. |
| 402 | * |
| 403 | * Also use a high-priority workqueue to prioritize decryption work, |
| 404 | * which blocks reads from completing, over regular application tasks. |
| 405 | */ |
| 406 | fscrypt_read_workqueue = alloc_workqueue("fscrypt_read_queue", |
| 407 | WQ_UNBOUND | WQ_HIGHPRI, |
| 408 | num_online_cpus()); |
| 409 | if (!fscrypt_read_workqueue) |
| 410 | goto fail; |
| 411 | |
| 412 | fscrypt_inode_info_cachep = KMEM_CACHE(fscrypt_inode_info, |
| 413 | SLAB_RECLAIM_ACCOUNT); |
| 414 | if (!fscrypt_inode_info_cachep) |
| 415 | goto fail_free_queue; |
| 416 | |
| 417 | err = fscrypt_init_keyring(); |
| 418 | if (err) |
| 419 | goto fail_free_inode_info; |
| 420 | |
| 421 | return 0; |
| 422 | |
| 423 | fail_free_inode_info: |
| 424 | kmem_cache_destroy(fscrypt_inode_info_cachep); |
| 425 | fail_free_queue: |
| 426 | destroy_workqueue(fscrypt_read_workqueue); |
| 427 | fail: |
| 428 | return err; |
| 429 | } |
| 430 | late_initcall(fscrypt_init) |