2 * 2007+ Copyright (c) Evgeniy Polyakov <zbr@ioremap.net>
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
16 #include <linux/crypto.h>
17 #include <linux/highmem.h>
18 #include <linux/kthread.h>
19 #include <linux/pagemap.h>
20 #include <linux/slab.h>
24 static struct crypto_hash *pohmelfs_init_hash(struct pohmelfs_sb *psb)
27 struct crypto_hash *hash;
29 hash = crypto_alloc_hash(psb->hash_string, 0, CRYPTO_ALG_ASYNC);
32 dprintk("%s: idx: %u: failed to allocate hash '%s', err: %d.\n",
33 __func__, psb->idx, psb->hash_string, err);
37 psb->crypto_attached_size = crypto_hash_digestsize(hash);
39 if (!psb->hash_keysize)
42 err = crypto_hash_setkey(hash, psb->hash_key, psb->hash_keysize);
44 dprintk("%s: idx: %u: failed to set key for hash '%s', err: %d.\n",
45 __func__, psb->idx, psb->hash_string, err);
52 crypto_free_hash(hash);
57 static struct crypto_ablkcipher *pohmelfs_init_cipher(struct pohmelfs_sb *psb)
60 struct crypto_ablkcipher *cipher;
62 if (!psb->cipher_keysize)
65 cipher = crypto_alloc_ablkcipher(psb->cipher_string, 0, 0);
67 err = PTR_ERR(cipher);
68 dprintk("%s: idx: %u: failed to allocate cipher '%s', err: %d.\n",
69 __func__, psb->idx, psb->cipher_string, err);
73 crypto_ablkcipher_clear_flags(cipher, ~0);
75 err = crypto_ablkcipher_setkey(cipher, psb->cipher_key, psb->cipher_keysize);
77 dprintk("%s: idx: %u: failed to set key for cipher '%s', err: %d.\n",
78 __func__, psb->idx, psb->cipher_string, err);
85 crypto_free_ablkcipher(cipher);
90 int pohmelfs_crypto_engine_init(struct pohmelfs_crypto_engine *e, struct pohmelfs_sb *psb)
97 e->data = kmalloc(e->size, GFP_KERNEL);
103 if (psb->hash_string) {
104 e->hash = pohmelfs_init_hash(psb);
105 if (IS_ERR(e->hash)) {
106 err = PTR_ERR(e->hash);
112 if (psb->cipher_string) {
113 e->cipher = pohmelfs_init_cipher(psb);
114 if (IS_ERR(e->cipher)) {
115 err = PTR_ERR(e->cipher);
117 goto err_out_free_hash;
124 crypto_free_hash(e->hash);
131 void pohmelfs_crypto_engine_exit(struct pohmelfs_crypto_engine *e)
133 crypto_free_hash(e->hash);
134 crypto_free_ablkcipher(e->cipher);
138 static void pohmelfs_crypto_complete(struct crypto_async_request *req, int err)
140 struct pohmelfs_crypto_completion *c = req->data;
142 if (err == -EINPROGRESS)
145 dprintk("%s: req: %p, err: %d.\n", __func__, req, err);
147 complete(&c->complete);
150 static int pohmelfs_crypto_process(struct ablkcipher_request *req,
151 struct scatterlist *sg_dst, struct scatterlist *sg_src,
152 void *iv, int enc, unsigned long timeout)
154 struct pohmelfs_crypto_completion complete;
157 init_completion(&complete.complete);
158 complete.error = -EINPROGRESS;
160 ablkcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
161 pohmelfs_crypto_complete, &complete);
163 ablkcipher_request_set_crypt(req, sg_src, sg_dst, sg_src->length, iv);
166 err = crypto_ablkcipher_encrypt(req);
168 err = crypto_ablkcipher_decrypt(req);
173 err = wait_for_completion_interruptible_timeout(&complete.complete,
178 err = complete.error;
187 int pohmelfs_crypto_process_input_data(struct pohmelfs_crypto_engine *e, u64 cmd_iv,
188 void *data, struct page *page, unsigned int size)
191 struct scatterlist sg;
193 if (!e->cipher && !e->hash)
196 dprintk("%s: eng: %p, iv: %llx, data: %p, page: %p/%lu, size: %u.\n",
197 __func__, e, cmd_iv, data, page, (page) ? page->index : 0, size);
200 sg_init_one(&sg, data, size);
202 sg_init_table(&sg, 1);
203 sg_set_page(&sg, page, size, 0);
207 struct ablkcipher_request *req = e->data + crypto_hash_digestsize(e->hash);
210 memset(iv, 0, sizeof(iv));
211 memcpy(iv, &cmd_iv, sizeof(cmd_iv));
213 ablkcipher_request_set_tfm(req, e->cipher);
215 err = pohmelfs_crypto_process(req, &sg, &sg, iv, 0, e->timeout);
221 struct hash_desc desc;
222 void *dst = e->data + e->size/2;
227 err = crypto_hash_init(&desc);
231 err = crypto_hash_update(&desc, &sg, size);
235 err = crypto_hash_final(&desc, dst);
239 err = !!memcmp(dst, e->data, crypto_hash_digestsize(e->hash));
242 #ifdef CONFIG_POHMELFS_DEBUG
244 unsigned char *recv = e->data, *calc = dst;
246 dprintk("%s: eng: %p, hash: %p, cipher: %p: iv : %llx, hash mismatch (recv/calc): ",
247 __func__, e, e->hash, e->cipher, cmd_iv);
248 for (i = 0; i < crypto_hash_digestsize(e->hash); ++i) {
250 dprintka("%02x ", recv[i]);
251 if (recv[i] != calc[i]) {
252 dprintka("| calc byte: %02x.\n", calc[i]);
256 dprintka("%02x/%02x ", recv[i], calc[i]);
263 dprintk("%s: eng: %p, hash: %p, cipher: %p: hashes matched.\n",
264 __func__, e, e->hash, e->cipher);
268 dprintk("%s: eng: %p, size: %u, hash: %p, cipher: %p: completed.\n",
269 __func__, e, e->size, e->hash, e->cipher);
274 dprintk("%s: eng: %p, hash: %p, cipher: %p: err: %d.\n",
275 __func__, e, e->hash, e->cipher, err);
279 static int pohmelfs_trans_iter(struct netfs_trans *t, struct pohmelfs_crypto_engine *e,
280 int (*iterator) (struct pohmelfs_crypto_engine *e,
281 struct scatterlist *dst,
282 struct scatterlist *src))
284 void *data = t->iovec.iov_base + sizeof(struct netfs_cmd) + t->psb->crypto_attached_size;
285 unsigned int size = t->iovec.iov_len - sizeof(struct netfs_cmd) - t->psb->crypto_attached_size;
286 struct netfs_cmd *cmd = data;
287 unsigned int sz, pages = t->attached_pages, i, csize, cmd_cmd, dpage_idx;
288 struct scatterlist sg_src, sg_dst;
293 cmd_cmd = __be16_to_cpu(cmd->cmd);
294 csize = __be32_to_cpu(cmd->size);
295 cmd->iv = __cpu_to_be64(e->iv);
297 if (cmd_cmd == NETFS_READ_PAGES || cmd_cmd == NETFS_READ_PAGE)
298 csize = __be16_to_cpu(cmd->ext);
300 sz = csize + __be16_to_cpu(cmd->cpad) + sizeof(struct netfs_cmd);
302 dprintk("%s: size: %u, sz: %u, cmd_size: %u, cmd_cpad: %u.\n",
303 __func__, size, sz, __be32_to_cpu(cmd->size), __be16_to_cpu(cmd->cpad));
308 sg_init_one(&sg_src, cmd->data, sz - sizeof(struct netfs_cmd));
309 sg_init_one(&sg_dst, cmd->data, sz - sizeof(struct netfs_cmd));
311 err = iterator(e, &sg_dst, &sg_src);
320 for (i = 0; i < t->page_num; ++i) {
321 struct page *page = t->pages[i];
322 struct page *dpage = e->pages[dpage_idx];
327 sg_init_table(&sg_src, 1);
328 sg_init_table(&sg_dst, 1);
329 sg_set_page(&sg_src, page, page_private(page), 0);
330 sg_set_page(&sg_dst, dpage, page_private(page), 0);
332 err = iterator(e, &sg_dst, &sg_src);
345 static int pohmelfs_encrypt_iterator(struct pohmelfs_crypto_engine *e,
346 struct scatterlist *sg_dst, struct scatterlist *sg_src)
348 struct ablkcipher_request *req = e->data;
351 memset(iv, 0, sizeof(iv));
353 memcpy(iv, &e->iv, sizeof(e->iv));
355 return pohmelfs_crypto_process(req, sg_dst, sg_src, iv, 1, e->timeout);
358 static int pohmelfs_encrypt(struct pohmelfs_crypto_thread *tc)
360 struct netfs_trans *t = tc->trans;
361 struct pohmelfs_crypto_engine *e = &tc->eng;
362 struct ablkcipher_request *req = e->data;
364 memset(req, 0, sizeof(struct ablkcipher_request));
365 ablkcipher_request_set_tfm(req, e->cipher);
367 e->iv = pohmelfs_gen_iv(t);
369 return pohmelfs_trans_iter(t, e, pohmelfs_encrypt_iterator);
372 static int pohmelfs_hash_iterator(struct pohmelfs_crypto_engine *e,
373 struct scatterlist *sg_dst, struct scatterlist *sg_src)
375 return crypto_hash_update(e->data, sg_src, sg_src->length);
378 static int pohmelfs_hash(struct pohmelfs_crypto_thread *tc)
380 struct pohmelfs_crypto_engine *e = &tc->eng;
381 struct hash_desc *desc = e->data;
382 unsigned char *dst = tc->trans->iovec.iov_base + sizeof(struct netfs_cmd);
388 err = crypto_hash_init(desc);
392 err = pohmelfs_trans_iter(tc->trans, e, pohmelfs_hash_iterator);
396 err = crypto_hash_final(desc, dst);
402 dprintk("%s: ", __func__);
403 for (i = 0; i < tc->psb->crypto_attached_size; ++i)
404 dprintka("%02x ", dst[i]);
411 static void pohmelfs_crypto_pages_free(struct pohmelfs_crypto_engine *e)
415 for (i = 0; i < e->page_num; ++i)
416 __free_page(e->pages[i]);
420 static int pohmelfs_crypto_pages_alloc(struct pohmelfs_crypto_engine *e, struct pohmelfs_sb *psb)
424 e->pages = kmalloc(psb->trans_max_pages * sizeof(struct page *), GFP_KERNEL);
428 for (i = 0; i < psb->trans_max_pages; ++i) {
429 e->pages[i] = alloc_page(GFP_KERNEL);
445 static void pohmelfs_sys_crypto_exit_one(struct pohmelfs_crypto_thread *t)
447 struct pohmelfs_sb *psb = t->psb;
450 kthread_stop(t->thread);
452 mutex_lock(&psb->crypto_thread_lock);
453 list_del(&t->thread_entry);
454 psb->crypto_thread_num--;
455 mutex_unlock(&psb->crypto_thread_lock);
457 pohmelfs_crypto_engine_exit(&t->eng);
458 pohmelfs_crypto_pages_free(&t->eng);
462 static int pohmelfs_crypto_finish(struct netfs_trans *t, struct pohmelfs_sb *psb, int err)
464 struct netfs_cmd *cmd = t->iovec.iov_base;
465 netfs_convert_cmd(cmd);
468 err = netfs_trans_finish_send(t, psb);
476 void pohmelfs_crypto_thread_make_ready(struct pohmelfs_crypto_thread *th)
478 struct pohmelfs_sb *psb = th->psb;
483 mutex_lock(&psb->crypto_thread_lock);
484 list_move_tail(&th->thread_entry, &psb->crypto_ready_list);
485 mutex_unlock(&psb->crypto_thread_lock);
489 static int pohmelfs_crypto_thread_trans(struct pohmelfs_crypto_thread *t)
491 struct netfs_trans *trans;
498 err = pohmelfs_hash(t);
504 err = pohmelfs_encrypt(t);
507 trans->eng = &t->eng;
515 pohmelfs_crypto_thread_make_ready(t);
517 pohmelfs_crypto_finish(trans, t->psb, err);
521 static int pohmelfs_crypto_thread_page(struct pohmelfs_crypto_thread *t)
523 struct pohmelfs_crypto_engine *e = &t->eng;
524 struct page *page = t->page;
527 WARN_ON(!PageChecked(page));
529 err = pohmelfs_crypto_process_input_data(e, e->iv, NULL, page, t->size);
531 SetPageUptodate(page);
535 page_cache_release(page);
537 pohmelfs_crypto_thread_make_ready(t);
542 static int pohmelfs_crypto_thread_func(void *data)
544 struct pohmelfs_crypto_thread *t = data;
546 while (!kthread_should_stop()) {
547 wait_event_interruptible(t->wait, kthread_should_stop() ||
548 t->trans || t->page);
550 if (kthread_should_stop())
553 if (!t->trans && !t->page)
556 dprintk("%s: thread: %p, trans: %p, page: %p.\n",
557 __func__, t, t->trans, t->page);
560 pohmelfs_crypto_thread_trans(t);
562 pohmelfs_crypto_thread_page(t);
568 static void pohmelfs_crypto_flush(struct pohmelfs_sb *psb, struct list_head *head)
570 while (!list_empty(head)) {
571 struct pohmelfs_crypto_thread *t = NULL;
573 mutex_lock(&psb->crypto_thread_lock);
574 if (!list_empty(head)) {
575 t = list_first_entry(head, struct pohmelfs_crypto_thread, thread_entry);
576 list_del_init(&t->thread_entry);
578 mutex_unlock(&psb->crypto_thread_lock);
581 pohmelfs_sys_crypto_exit_one(t);
585 static void pohmelfs_sys_crypto_exit(struct pohmelfs_sb *psb)
587 while (!list_empty(&psb->crypto_active_list) || !list_empty(&psb->crypto_ready_list)) {
588 dprintk("%s: crypto_thread_num: %u.\n", __func__, psb->crypto_thread_num);
589 pohmelfs_crypto_flush(psb, &psb->crypto_active_list);
590 pohmelfs_crypto_flush(psb, &psb->crypto_ready_list);
594 static int pohmelfs_sys_crypto_init(struct pohmelfs_sb *psb)
597 struct pohmelfs_crypto_thread *t;
598 struct pohmelfs_config *c;
599 struct netfs_state *st;
602 list_for_each_entry(c, &psb->state_list, config_entry) {
605 err = pohmelfs_crypto_engine_init(&st->eng, psb);
609 dprintk("%s: st: %p, eng: %p, hash: %p, cipher: %p.\n",
610 __func__, st, &st->eng, &st->eng.hash, &st->eng.cipher);
613 for (i = 0; i < psb->crypto_thread_num; ++i) {
615 t = kzalloc(sizeof(struct pohmelfs_crypto_thread), GFP_KERNEL);
617 goto err_out_free_state_engines;
619 init_waitqueue_head(&t->wait);
625 err = pohmelfs_crypto_engine_init(&t->eng, psb);
627 goto err_out_free_state_engines;
629 err = pohmelfs_crypto_pages_alloc(&t->eng, psb);
633 t->thread = kthread_run(pohmelfs_crypto_thread_func, t,
634 "pohmelfs-crypto-%d-%d", psb->idx, i);
635 if (IS_ERR(t->thread)) {
636 err = PTR_ERR(t->thread);
642 psb->crypto_align_size = crypto_ablkcipher_blocksize(t->eng.cipher);
644 mutex_lock(&psb->crypto_thread_lock);
645 list_add_tail(&t->thread_entry, &psb->crypto_ready_list);
646 mutex_unlock(&psb->crypto_thread_lock);
649 psb->crypto_thread_num = i;
653 pohmelfs_sys_crypto_exit_one(t);
654 err_out_free_state_engines:
655 list_for_each_entry(c, &psb->state_list, config_entry) {
657 pohmelfs_crypto_engine_exit(&st->eng);
660 pohmelfs_sys_crypto_exit(psb);
664 void pohmelfs_crypto_exit(struct pohmelfs_sb *psb)
666 pohmelfs_sys_crypto_exit(psb);
668 kfree(psb->hash_string);
669 kfree(psb->cipher_string);
672 static int pohmelfs_crypt_init_complete(struct page **pages, unsigned int page_num,
673 void *private, int err)
675 struct pohmelfs_sb *psb = private;
678 dprintk("%s: err: %d.\n", __func__, err);
685 static int pohmelfs_crypto_init_handshake(struct pohmelfs_sb *psb)
687 struct netfs_trans *t;
688 struct netfs_crypto_capabilities *cap;
689 struct netfs_cmd *cmd;
691 int err = -ENOMEM, size;
693 size = sizeof(struct netfs_crypto_capabilities) +
694 psb->cipher_strlen + psb->hash_strlen + 2; /* 0 bytes */
696 t = netfs_trans_alloc(psb, size, 0, 0);
700 t->complete = pohmelfs_crypt_init_complete;
703 cmd = netfs_trans_current(t);
704 cap = (struct netfs_crypto_capabilities *)(cmd + 1);
705 str = (char *)(cap + 1);
707 cmd->cmd = NETFS_CAPABILITIES;
708 cmd->id = POHMELFS_CRYPTO_CAPABILITIES;
714 netfs_convert_cmd(cmd);
715 netfs_trans_update(cmd, t, size);
717 cap->hash_strlen = psb->hash_strlen;
718 if (cap->hash_strlen) {
719 sprintf(str, "%s", psb->hash_string);
720 str += cap->hash_strlen;
723 cap->cipher_strlen = psb->cipher_strlen;
724 cap->cipher_keysize = psb->cipher_keysize;
725 if (cap->cipher_strlen)
726 sprintf(str, "%s", psb->cipher_string);
728 netfs_convert_crypto_capabilities(cap);
731 err = netfs_trans_finish(t, psb);
735 err = wait_event_interruptible_timeout(psb->wait, (psb->flags != ~0),
736 psb->wait_on_page_timeout);
743 psb->perform_crypto = 1;
747 * At this point NETFS_CAPABILITIES response command
748 * should setup superblock in a way, which is acceptible
749 * for both client and server, so if server refuses connection,
750 * it will send error in transaction response.
762 int pohmelfs_crypto_init(struct pohmelfs_sb *psb)
766 if (!psb->cipher_string && !psb->hash_string)
769 err = pohmelfs_crypto_init_handshake(psb);
773 err = pohmelfs_sys_crypto_init(psb);
780 static int pohmelfs_crypto_thread_get(struct pohmelfs_sb *psb,
781 int (*action)(struct pohmelfs_crypto_thread *t, void *data), void *data)
783 struct pohmelfs_crypto_thread *t = NULL;
787 err = wait_event_interruptible_timeout(psb->wait,
788 !list_empty(&psb->crypto_ready_list),
789 psb->wait_on_page_timeout);
793 mutex_lock(&psb->crypto_thread_lock);
794 if (!list_empty(&psb->crypto_ready_list)) {
795 t = list_entry(psb->crypto_ready_list.prev,
796 struct pohmelfs_crypto_thread,
799 list_move_tail(&t->thread_entry,
800 &psb->crypto_active_list);
806 mutex_unlock(&psb->crypto_thread_lock);
812 static int pohmelfs_trans_crypt_action(struct pohmelfs_crypto_thread *t, void *data)
814 struct netfs_trans *trans = data;
816 netfs_trans_get(trans);
819 dprintk("%s: t: %p, gen: %u, thread: %p.\n", __func__, trans, trans->gen, t);
823 int pohmelfs_trans_crypt(struct netfs_trans *trans, struct pohmelfs_sb *psb)
825 if ((!psb->hash_string && !psb->cipher_string) || !psb->perform_crypto) {
826 netfs_trans_get(trans);
827 return pohmelfs_crypto_finish(trans, psb, 0);
830 return pohmelfs_crypto_thread_get(psb, pohmelfs_trans_crypt_action, trans);
833 struct pohmelfs_crypto_input_action_data {
835 struct pohmelfs_crypto_engine *e;
840 static int pohmelfs_crypt_input_page_action(struct pohmelfs_crypto_thread *t, void *data)
842 struct pohmelfs_crypto_input_action_data *act = data;
844 memcpy(t->eng.data, act->e->data, t->psb->crypto_attached_size);
853 int pohmelfs_crypto_process_input_page(struct pohmelfs_crypto_engine *e,
854 struct page *page, unsigned int size, u64 iv)
856 struct inode *inode = page->mapping->host;
857 struct pohmelfs_crypto_input_action_data act;
865 err = pohmelfs_crypto_thread_get(POHMELFS_SB(inode->i_sb),
866 pohmelfs_crypt_input_page_action, &act);
873 SetPageUptodate(page);
874 page_cache_release(page);