2 * linux/fs/f2fs/crypto.c
4 * Copied from linux/fs/ext4/crypto.c
6 * Copyright (C) 2015, Google, Inc.
7 * Copyright (C) 2015, Motorola Mobility
9 * This contains encryption functions for f2fs
11 * Written by Michael Halcrow, 2014.
13 * Filename encryption additions
14 * Uday Savagaonkar, 2014
15 * Encryption policy handling additions
16 * Ildar Muslukhov, 2014
17 * Remove ext4_encrypted_zeroout(),
18 * add f2fs_restore_and_release_control_page()
21 * This has not yet undergone a rigorous security audit.
23 * The usage of AES-XTS should conform to recommendations in NIST
24 * Special Publication 800-38E and IEEE P1619/D16.
26 #include <crypto/skcipher.h>
27 #include <keys/user-type.h>
28 #include <keys/encrypted-type.h>
29 #include <linux/ecryptfs.h>
30 #include <linux/gfp.h>
31 #include <linux/kernel.h>
32 #include <linux/key.h>
33 #include <linux/list.h>
34 #include <linux/mempool.h>
35 #include <linux/module.h>
36 #include <linux/mutex.h>
37 #include <linux/random.h>
38 #include <linux/scatterlist.h>
39 #include <linux/spinlock_types.h>
40 #include <linux/f2fs_fs.h>
41 #include <linux/ratelimit.h>
42 #include <linux/bio.h>
47 /* Encryption added and removed here! (L: */
49 static unsigned int num_prealloc_crypto_pages = 32;
50 static unsigned int num_prealloc_crypto_ctxs = 128;
52 module_param(num_prealloc_crypto_pages, uint, 0444);
53 MODULE_PARM_DESC(num_prealloc_crypto_pages,
54 "Number of crypto pages to preallocate");
55 module_param(num_prealloc_crypto_ctxs, uint, 0444);
56 MODULE_PARM_DESC(num_prealloc_crypto_ctxs,
57 "Number of crypto contexts to preallocate");
59 static mempool_t *f2fs_bounce_page_pool;
61 static LIST_HEAD(f2fs_free_crypto_ctxs);
62 static DEFINE_SPINLOCK(f2fs_crypto_ctx_lock);
64 static struct workqueue_struct *f2fs_read_workqueue;
65 static DEFINE_MUTEX(crypto_init);
67 static struct kmem_cache *f2fs_crypto_ctx_cachep;
68 struct kmem_cache *f2fs_crypt_info_cachep;
71 * f2fs_release_crypto_ctx() - Releases an encryption context
72 * @ctx: The encryption context to release.
74 * If the encryption context was allocated from the pre-allocated pool, returns
75 * it to that pool. Else, frees it.
77 * If there's a bounce page in the context, this frees that.
79 void f2fs_release_crypto_ctx(struct f2fs_crypto_ctx *ctx)
83 if (ctx->flags & F2FS_WRITE_PATH_FL && ctx->w.bounce_page) {
84 mempool_free(ctx->w.bounce_page, f2fs_bounce_page_pool);
85 ctx->w.bounce_page = NULL;
87 ctx->w.control_page = NULL;
88 if (ctx->flags & F2FS_CTX_REQUIRES_FREE_ENCRYPT_FL) {
89 kmem_cache_free(f2fs_crypto_ctx_cachep, ctx);
91 spin_lock_irqsave(&f2fs_crypto_ctx_lock, flags);
92 list_add(&ctx->free_list, &f2fs_free_crypto_ctxs);
93 spin_unlock_irqrestore(&f2fs_crypto_ctx_lock, flags);
98 * f2fs_get_crypto_ctx() - Gets an encryption context
99 * @inode: The inode for which we are doing the crypto
101 * Allocates and initializes an encryption context.
103 * Return: An allocated and initialized encryption context on success; error
104 * value or NULL otherwise.
106 struct f2fs_crypto_ctx *f2fs_get_crypto_ctx(struct inode *inode)
108 struct f2fs_crypto_ctx *ctx = NULL;
110 struct f2fs_crypt_info *ci = F2FS_I(inode)->i_crypt_info;
113 return ERR_PTR(-ENOKEY);
116 * We first try getting the ctx from a free list because in
117 * the common case the ctx will have an allocated and
118 * initialized crypto tfm, so it's probably a worthwhile
119 * optimization. For the bounce page, we first try getting it
120 * from the kernel allocator because that's just about as fast
121 * as getting it from a list and because a cache of free pages
122 * should generally be a "last resort" option for a filesystem
123 * to be able to do its job.
125 spin_lock_irqsave(&f2fs_crypto_ctx_lock, flags);
126 ctx = list_first_entry_or_null(&f2fs_free_crypto_ctxs,
127 struct f2fs_crypto_ctx, free_list);
129 list_del(&ctx->free_list);
130 spin_unlock_irqrestore(&f2fs_crypto_ctx_lock, flags);
132 ctx = kmem_cache_zalloc(f2fs_crypto_ctx_cachep, GFP_NOFS);
134 return ERR_PTR(-ENOMEM);
135 ctx->flags |= F2FS_CTX_REQUIRES_FREE_ENCRYPT_FL;
137 ctx->flags &= ~F2FS_CTX_REQUIRES_FREE_ENCRYPT_FL;
139 ctx->flags &= ~F2FS_WRITE_PATH_FL;
144 * Call f2fs_decrypt on every single page, reusing the encryption
147 static void completion_pages(struct work_struct *work)
149 struct f2fs_crypto_ctx *ctx =
150 container_of(work, struct f2fs_crypto_ctx, r.work);
151 struct bio *bio = ctx->r.bio;
155 bio_for_each_segment_all(bv, bio, i) {
156 struct page *page = bv->bv_page;
157 int ret = f2fs_decrypt(ctx, page);
163 SetPageUptodate(page);
166 f2fs_release_crypto_ctx(ctx);
170 void f2fs_end_io_crypto_work(struct f2fs_crypto_ctx *ctx, struct bio *bio)
172 INIT_WORK(&ctx->r.work, completion_pages);
174 queue_work(f2fs_read_workqueue, &ctx->r.work);
177 static void f2fs_crypto_destroy(void)
179 struct f2fs_crypto_ctx *pos, *n;
181 list_for_each_entry_safe(pos, n, &f2fs_free_crypto_ctxs, free_list)
182 kmem_cache_free(f2fs_crypto_ctx_cachep, pos);
183 INIT_LIST_HEAD(&f2fs_free_crypto_ctxs);
184 if (f2fs_bounce_page_pool)
185 mempool_destroy(f2fs_bounce_page_pool);
186 f2fs_bounce_page_pool = NULL;
190 * f2fs_crypto_initialize() - Set up for f2fs encryption.
192 * We only call this when we start accessing encrypted files, since it
193 * results in memory getting allocated that wouldn't otherwise be used.
195 * Return: Zero on success, non-zero otherwise.
197 int f2fs_crypto_initialize(void)
199 int i, res = -ENOMEM;
201 if (f2fs_bounce_page_pool)
204 mutex_lock(&crypto_init);
205 if (f2fs_bounce_page_pool)
206 goto already_initialized;
208 for (i = 0; i < num_prealloc_crypto_ctxs; i++) {
209 struct f2fs_crypto_ctx *ctx;
211 ctx = kmem_cache_zalloc(f2fs_crypto_ctx_cachep, GFP_KERNEL);
214 list_add(&ctx->free_list, &f2fs_free_crypto_ctxs);
217 /* must be allocated at the last step to avoid race condition above */
218 f2fs_bounce_page_pool =
219 mempool_create_page_pool(num_prealloc_crypto_pages, 0);
220 if (!f2fs_bounce_page_pool)
224 mutex_unlock(&crypto_init);
227 f2fs_crypto_destroy();
228 mutex_unlock(&crypto_init);
233 * f2fs_exit_crypto() - Shutdown the f2fs encryption system
235 void f2fs_exit_crypto(void)
237 f2fs_crypto_destroy();
239 if (f2fs_read_workqueue)
240 destroy_workqueue(f2fs_read_workqueue);
241 if (f2fs_crypto_ctx_cachep)
242 kmem_cache_destroy(f2fs_crypto_ctx_cachep);
243 if (f2fs_crypt_info_cachep)
244 kmem_cache_destroy(f2fs_crypt_info_cachep);
247 int __init f2fs_init_crypto(void)
251 f2fs_read_workqueue = alloc_workqueue("f2fs_crypto", WQ_HIGHPRI, 0);
252 if (!f2fs_read_workqueue)
255 f2fs_crypto_ctx_cachep = KMEM_CACHE(f2fs_crypto_ctx,
256 SLAB_RECLAIM_ACCOUNT);
257 if (!f2fs_crypto_ctx_cachep)
260 f2fs_crypt_info_cachep = KMEM_CACHE(f2fs_crypt_info,
261 SLAB_RECLAIM_ACCOUNT);
262 if (!f2fs_crypt_info_cachep)
271 void f2fs_restore_and_release_control_page(struct page **page)
273 struct f2fs_crypto_ctx *ctx;
274 struct page *bounce_page;
276 /* The bounce data pages are unmapped. */
277 if ((*page)->mapping)
280 /* The bounce data page is unmapped. */
282 ctx = (struct f2fs_crypto_ctx *)page_private(bounce_page);
284 /* restore control page */
285 *page = ctx->w.control_page;
287 f2fs_restore_control_page(bounce_page);
290 void f2fs_restore_control_page(struct page *data_page)
292 struct f2fs_crypto_ctx *ctx =
293 (struct f2fs_crypto_ctx *)page_private(data_page);
295 set_page_private(data_page, (unsigned long)NULL);
296 ClearPagePrivate(data_page);
297 unlock_page(data_page);
298 f2fs_release_crypto_ctx(ctx);
302 * f2fs_crypt_complete() - The completion callback for page encryption
303 * @req: The asynchronous encryption request context
304 * @res: The result of the encryption operation
306 static void f2fs_crypt_complete(struct crypto_async_request *req, int res)
308 struct f2fs_completion_result *ecr = req->data;
310 if (res == -EINPROGRESS)
313 complete(&ecr->completion);
321 static int f2fs_page_crypto(struct f2fs_crypto_ctx *ctx,
325 struct page *src_page,
326 struct page *dest_page)
328 u8 xts_tweak[F2FS_XTS_TWEAK_SIZE];
329 struct skcipher_request *req = NULL;
330 DECLARE_F2FS_COMPLETION_RESULT(ecr);
331 struct scatterlist dst, src;
332 struct f2fs_crypt_info *ci = F2FS_I(inode)->i_crypt_info;
333 struct crypto_skcipher *tfm = ci->ci_ctfm;
336 req = skcipher_request_alloc(tfm, GFP_NOFS);
338 printk_ratelimited(KERN_ERR
339 "%s: crypto_request_alloc() failed\n",
343 skcipher_request_set_callback(
344 req, CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP,
345 f2fs_crypt_complete, &ecr);
347 BUILD_BUG_ON(F2FS_XTS_TWEAK_SIZE < sizeof(index));
348 memcpy(xts_tweak, &index, sizeof(index));
349 memset(&xts_tweak[sizeof(index)], 0,
350 F2FS_XTS_TWEAK_SIZE - sizeof(index));
352 sg_init_table(&dst, 1);
353 sg_set_page(&dst, dest_page, PAGE_CACHE_SIZE, 0);
354 sg_init_table(&src, 1);
355 sg_set_page(&src, src_page, PAGE_CACHE_SIZE, 0);
356 skcipher_request_set_crypt(req, &src, &dst, PAGE_CACHE_SIZE,
358 if (rw == F2FS_DECRYPT)
359 res = crypto_skcipher_decrypt(req);
361 res = crypto_skcipher_encrypt(req);
362 if (res == -EINPROGRESS || res == -EBUSY) {
363 BUG_ON(req->base.data != &ecr);
364 wait_for_completion(&ecr.completion);
367 skcipher_request_free(req);
369 printk_ratelimited(KERN_ERR
370 "%s: crypto_skcipher_encrypt() returned %d\n",
377 static struct page *alloc_bounce_page(struct f2fs_crypto_ctx *ctx)
379 ctx->w.bounce_page = mempool_alloc(f2fs_bounce_page_pool, GFP_NOWAIT);
380 if (ctx->w.bounce_page == NULL)
381 return ERR_PTR(-ENOMEM);
382 ctx->flags |= F2FS_WRITE_PATH_FL;
383 return ctx->w.bounce_page;
387 * f2fs_encrypt() - Encrypts a page
388 * @inode: The inode for which the encryption should take place
389 * @plaintext_page: The page to encrypt. Must be locked.
391 * Allocates a ciphertext page and encrypts plaintext_page into it using the ctx
392 * encryption context.
394 * Called on the page write path. The caller must call
395 * f2fs_restore_control_page() on the returned ciphertext page to
396 * release the bounce buffer and the encryption context.
398 * Return: An allocated page with the encrypted content on success. Else, an
399 * error value or NULL.
401 struct page *f2fs_encrypt(struct inode *inode,
402 struct page *plaintext_page)
404 struct f2fs_crypto_ctx *ctx;
405 struct page *ciphertext_page = NULL;
408 BUG_ON(!PageLocked(plaintext_page));
410 ctx = f2fs_get_crypto_ctx(inode);
412 return (struct page *)ctx;
414 /* The encryption operation will require a bounce page. */
415 ciphertext_page = alloc_bounce_page(ctx);
416 if (IS_ERR(ciphertext_page))
419 ctx->w.control_page = plaintext_page;
420 err = f2fs_page_crypto(ctx, inode, F2FS_ENCRYPT, plaintext_page->index,
421 plaintext_page, ciphertext_page);
423 ciphertext_page = ERR_PTR(err);
427 SetPagePrivate(ciphertext_page);
428 set_page_private(ciphertext_page, (unsigned long)ctx);
429 lock_page(ciphertext_page);
430 return ciphertext_page;
433 f2fs_release_crypto_ctx(ctx);
434 return ciphertext_page;
438 * f2fs_decrypt() - Decrypts a page in-place
439 * @ctx: The encryption context.
440 * @page: The page to decrypt. Must be locked.
442 * Decrypts page in-place using the ctx encryption context.
444 * Called from the read completion callback.
446 * Return: Zero on success, non-zero otherwise.
448 int f2fs_decrypt(struct f2fs_crypto_ctx *ctx, struct page *page)
450 BUG_ON(!PageLocked(page));
452 return f2fs_page_crypto(ctx, page->mapping->host,
453 F2FS_DECRYPT, page->index, page, page);
457 * Convenience function which takes care of allocating and
458 * deallocating the encryption context
460 int f2fs_decrypt_one(struct inode *inode, struct page *page)
462 struct f2fs_crypto_ctx *ctx = f2fs_get_crypto_ctx(inode);
467 ret = f2fs_decrypt(ctx, page);
468 f2fs_release_crypto_ctx(ctx);
472 bool f2fs_valid_contents_enc_mode(uint32_t mode)
474 return (mode == F2FS_ENCRYPTION_MODE_AES_256_XTS);
478 * f2fs_validate_encryption_key_size() - Validate the encryption key size
479 * @mode: The key mode.
480 * @size: The key size to validate.
482 * Return: The validated key size for @mode. Zero if invalid.
484 uint32_t f2fs_validate_encryption_key_size(uint32_t mode, uint32_t size)
486 if (size == f2fs_encryption_key_size(mode))