2 * Support for Intel AES-NI instructions. This file contains glue
3 * code, the real AES implementation is in intel-aes_asm.S.
5 * Copyright (C) 2008, Intel Corp.
6 * Author: Huang Ying <ying.huang@intel.com>
8 * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD
9 * interface for 64-bit kernels.
10 * Authors: Adrian Hoban <adrian.hoban@intel.com>
11 * Gabriele Paoloni <gabriele.paoloni@intel.com>
12 * Tadeusz Struk (tadeusz.struk@intel.com)
13 * Aidan O'Mahony (aidan.o.mahony@intel.com)
14 * Copyright (c) 2010, Intel Corporation.
16 * This program is free software; you can redistribute it and/or modify
17 * it under the terms of the GNU General Public License as published by
18 * the Free Software Foundation; either version 2 of the License, or
19 * (at your option) any later version.
22 #include <linux/hardirq.h>
23 #include <linux/types.h>
24 #include <linux/crypto.h>
25 #include <linux/module.h>
26 #include <linux/err.h>
27 #include <crypto/algapi.h>
28 #include <crypto/aes.h>
29 #include <crypto/cryptd.h>
30 #include <crypto/ctr.h>
31 #include <crypto/b128ops.h>
32 #include <crypto/lrw.h>
33 #include <crypto/xts.h>
34 #include <asm/cpu_device_id.h>
36 #include <asm/crypto/aes.h>
37 #include <asm/crypto/ablk_helper.h>
38 #include <crypto/scatterwalk.h>
39 #include <crypto/internal/aead.h>
40 #include <linux/workqueue.h>
41 #include <linux/spinlock.h>
43 #if defined(CONFIG_CRYPTO_PCBC) || defined(CONFIG_CRYPTO_PCBC_MODULE)
47 /* This data is stored at the end of the crypto_tfm struct.
48 * It's a type of per "session" data storage location.
49 * This needs to be 16 byte aligned.
51 struct aesni_rfc4106_gcm_ctx {
53 struct crypto_aes_ctx aes_key_expanded;
55 struct cryptd_aead *cryptd_tfm;
58 struct aesni_gcm_set_hash_subkey_result {
60 struct completion completion;
63 struct aesni_hash_subkey_req_data {
65 struct aesni_gcm_set_hash_subkey_result result;
66 struct scatterlist sg;
69 #define AESNI_ALIGN (16)
70 #define AES_BLOCK_MASK (~(AES_BLOCK_SIZE-1))
71 #define RFC4106_HASH_SUBKEY_SIZE 16
73 struct aesni_lrw_ctx {
74 struct lrw_table_ctx lrw_table;
75 u8 raw_aes_ctx[sizeof(struct crypto_aes_ctx) + AESNI_ALIGN - 1];
78 struct aesni_xts_ctx {
79 u8 raw_tweak_ctx[sizeof(struct crypto_aes_ctx) + AESNI_ALIGN - 1];
80 u8 raw_crypt_ctx[sizeof(struct crypto_aes_ctx) + AESNI_ALIGN - 1];
83 asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
84 unsigned int key_len);
85 asmlinkage void aesni_enc(struct crypto_aes_ctx *ctx, u8 *out,
87 asmlinkage void aesni_dec(struct crypto_aes_ctx *ctx, u8 *out,
89 asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out,
90 const u8 *in, unsigned int len);
91 asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out,
92 const u8 *in, unsigned int len);
93 asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
94 const u8 *in, unsigned int len, u8 *iv);
95 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
96 const u8 *in, unsigned int len, u8 *iv);
98 int crypto_fpu_init(void);
99 void crypto_fpu_exit(void);
102 asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out,
103 const u8 *in, unsigned int len, u8 *iv);
105 /* asmlinkage void aesni_gcm_enc()
106 * void *ctx, AES Key schedule. Starts on a 16 byte boundary.
107 * u8 *out, Ciphertext output. Encrypt in-place is allowed.
108 * const u8 *in, Plaintext input
109 * unsigned long plaintext_len, Length of data in bytes for encryption.
110 * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association)
111 * concatenated with 8 byte Initialisation Vector (from IPSec ESP
112 * Payload) concatenated with 0x00000001. 16-byte aligned pointer.
113 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
114 * const u8 *aad, Additional Authentication Data (AAD)
115 * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this
116 * is going to be 8 or 12 bytes
117 * u8 *auth_tag, Authenticated Tag output.
118 * unsigned long auth_tag_len), Authenticated Tag Length in bytes.
119 * Valid values are 16 (most likely), 12 or 8.
121 asmlinkage void aesni_gcm_enc(void *ctx, u8 *out,
122 const u8 *in, unsigned long plaintext_len, u8 *iv,
123 u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
124 u8 *auth_tag, unsigned long auth_tag_len);
126 /* asmlinkage void aesni_gcm_dec()
127 * void *ctx, AES Key schedule. Starts on a 16 byte boundary.
128 * u8 *out, Plaintext output. Decrypt in-place is allowed.
129 * const u8 *in, Ciphertext input
130 * unsigned long ciphertext_len, Length of data in bytes for decryption.
131 * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association)
132 * concatenated with 8 byte Initialisation Vector (from IPSec ESP
133 * Payload) concatenated with 0x00000001. 16-byte aligned pointer.
134 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
135 * const u8 *aad, Additional Authentication Data (AAD)
136 * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this is going
137 * to be 8 or 12 bytes
138 * u8 *auth_tag, Authenticated Tag output.
139 * unsigned long auth_tag_len) Authenticated Tag Length in bytes.
140 * Valid values are 16 (most likely), 12 or 8.
142 asmlinkage void aesni_gcm_dec(void *ctx, u8 *out,
143 const u8 *in, unsigned long ciphertext_len, u8 *iv,
144 u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
145 u8 *auth_tag, unsigned long auth_tag_len);
148 aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm)
151 (struct aesni_rfc4106_gcm_ctx *)
153 crypto_tfm_ctx(crypto_aead_tfm(tfm)), AESNI_ALIGN);
157 static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx)
159 unsigned long addr = (unsigned long)raw_ctx;
160 unsigned long align = AESNI_ALIGN;
162 if (align <= crypto_tfm_ctx_alignment())
164 return (struct crypto_aes_ctx *)ALIGN(addr, align);
167 static int aes_set_key_common(struct crypto_tfm *tfm, void *raw_ctx,
168 const u8 *in_key, unsigned int key_len)
170 struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx);
171 u32 *flags = &tfm->crt_flags;
174 if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 &&
175 key_len != AES_KEYSIZE_256) {
176 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
180 if (!irq_fpu_usable())
181 err = crypto_aes_expand_key(ctx, in_key, key_len);
184 err = aesni_set_key(ctx, in_key, key_len);
191 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
192 unsigned int key_len)
194 return aes_set_key_common(tfm, crypto_tfm_ctx(tfm), in_key, key_len);
197 static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
199 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
201 if (!irq_fpu_usable())
202 crypto_aes_encrypt_x86(ctx, dst, src);
205 aesni_enc(ctx, dst, src);
210 static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
212 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
214 if (!irq_fpu_usable())
215 crypto_aes_decrypt_x86(ctx, dst, src);
218 aesni_dec(ctx, dst, src);
223 static void __aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
225 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
227 aesni_enc(ctx, dst, src);
230 static void __aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
232 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
234 aesni_dec(ctx, dst, src);
237 static int ecb_encrypt(struct blkcipher_desc *desc,
238 struct scatterlist *dst, struct scatterlist *src,
241 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
242 struct blkcipher_walk walk;
245 blkcipher_walk_init(&walk, dst, src, nbytes);
246 err = blkcipher_walk_virt(desc, &walk);
247 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
250 while ((nbytes = walk.nbytes)) {
251 aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
252 nbytes & AES_BLOCK_MASK);
253 nbytes &= AES_BLOCK_SIZE - 1;
254 err = blkcipher_walk_done(desc, &walk, nbytes);
261 static int ecb_decrypt(struct blkcipher_desc *desc,
262 struct scatterlist *dst, struct scatterlist *src,
265 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
266 struct blkcipher_walk walk;
269 blkcipher_walk_init(&walk, dst, src, nbytes);
270 err = blkcipher_walk_virt(desc, &walk);
271 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
274 while ((nbytes = walk.nbytes)) {
275 aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
276 nbytes & AES_BLOCK_MASK);
277 nbytes &= AES_BLOCK_SIZE - 1;
278 err = blkcipher_walk_done(desc, &walk, nbytes);
285 static int cbc_encrypt(struct blkcipher_desc *desc,
286 struct scatterlist *dst, struct scatterlist *src,
289 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
290 struct blkcipher_walk walk;
293 blkcipher_walk_init(&walk, dst, src, nbytes);
294 err = blkcipher_walk_virt(desc, &walk);
295 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
298 while ((nbytes = walk.nbytes)) {
299 aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
300 nbytes & AES_BLOCK_MASK, walk.iv);
301 nbytes &= AES_BLOCK_SIZE - 1;
302 err = blkcipher_walk_done(desc, &walk, nbytes);
309 static int cbc_decrypt(struct blkcipher_desc *desc,
310 struct scatterlist *dst, struct scatterlist *src,
313 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
314 struct blkcipher_walk walk;
317 blkcipher_walk_init(&walk, dst, src, nbytes);
318 err = blkcipher_walk_virt(desc, &walk);
319 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
322 while ((nbytes = walk.nbytes)) {
323 aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
324 nbytes & AES_BLOCK_MASK, walk.iv);
325 nbytes &= AES_BLOCK_SIZE - 1;
326 err = blkcipher_walk_done(desc, &walk, nbytes);
334 static void ctr_crypt_final(struct crypto_aes_ctx *ctx,
335 struct blkcipher_walk *walk)
337 u8 *ctrblk = walk->iv;
338 u8 keystream[AES_BLOCK_SIZE];
339 u8 *src = walk->src.virt.addr;
340 u8 *dst = walk->dst.virt.addr;
341 unsigned int nbytes = walk->nbytes;
343 aesni_enc(ctx, keystream, ctrblk);
344 crypto_xor(keystream, src, nbytes);
345 memcpy(dst, keystream, nbytes);
346 crypto_inc(ctrblk, AES_BLOCK_SIZE);
349 static int ctr_crypt(struct blkcipher_desc *desc,
350 struct scatterlist *dst, struct scatterlist *src,
353 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
354 struct blkcipher_walk walk;
357 blkcipher_walk_init(&walk, dst, src, nbytes);
358 err = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE);
359 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
362 while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
363 aesni_ctr_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
364 nbytes & AES_BLOCK_MASK, walk.iv);
365 nbytes &= AES_BLOCK_SIZE - 1;
366 err = blkcipher_walk_done(desc, &walk, nbytes);
369 ctr_crypt_final(ctx, &walk);
370 err = blkcipher_walk_done(desc, &walk, 0);
378 static int ablk_ecb_init(struct crypto_tfm *tfm)
380 return ablk_init_common(tfm, "__driver-ecb-aes-aesni");
383 static int ablk_cbc_init(struct crypto_tfm *tfm)
385 return ablk_init_common(tfm, "__driver-cbc-aes-aesni");
389 static int ablk_ctr_init(struct crypto_tfm *tfm)
391 return ablk_init_common(tfm, "__driver-ctr-aes-aesni");
397 static int ablk_pcbc_init(struct crypto_tfm *tfm)
399 return ablk_init_common(tfm, "fpu(pcbc(__driver-aes-aesni))");
403 static void lrw_xts_encrypt_callback(void *ctx, u8 *blks, unsigned int nbytes)
405 aesni_ecb_enc(ctx, blks, blks, nbytes);
408 static void lrw_xts_decrypt_callback(void *ctx, u8 *blks, unsigned int nbytes)
410 aesni_ecb_dec(ctx, blks, blks, nbytes);
413 static int lrw_aesni_setkey(struct crypto_tfm *tfm, const u8 *key,
416 struct aesni_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
419 err = aes_set_key_common(tfm, ctx->raw_aes_ctx, key,
420 keylen - AES_BLOCK_SIZE);
424 return lrw_init_table(&ctx->lrw_table, key + keylen - AES_BLOCK_SIZE);
427 static void lrw_aesni_exit_tfm(struct crypto_tfm *tfm)
429 struct aesni_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
431 lrw_free_table(&ctx->lrw_table);
434 static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
435 struct scatterlist *src, unsigned int nbytes)
437 struct aesni_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
439 struct lrw_crypt_req req = {
441 .tbuflen = sizeof(buf),
443 .table_ctx = &ctx->lrw_table,
444 .crypt_ctx = aes_ctx(ctx->raw_aes_ctx),
445 .crypt_fn = lrw_xts_encrypt_callback,
449 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
452 ret = lrw_crypt(desc, dst, src, nbytes, &req);
458 static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
459 struct scatterlist *src, unsigned int nbytes)
461 struct aesni_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
463 struct lrw_crypt_req req = {
465 .tbuflen = sizeof(buf),
467 .table_ctx = &ctx->lrw_table,
468 .crypt_ctx = aes_ctx(ctx->raw_aes_ctx),
469 .crypt_fn = lrw_xts_decrypt_callback,
473 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
476 ret = lrw_crypt(desc, dst, src, nbytes, &req);
482 static int xts_aesni_setkey(struct crypto_tfm *tfm, const u8 *key,
485 struct aesni_xts_ctx *ctx = crypto_tfm_ctx(tfm);
486 u32 *flags = &tfm->crt_flags;
489 /* key consists of keys of equal size concatenated, therefore
490 * the length must be even
493 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
497 /* first half of xts-key is for crypt */
498 err = aes_set_key_common(tfm, ctx->raw_crypt_ctx, key, keylen / 2);
502 /* second half of xts-key is for tweak */
503 return aes_set_key_common(tfm, ctx->raw_tweak_ctx, key + keylen / 2,
508 static void aesni_xts_tweak(void *ctx, u8 *out, const u8 *in)
510 aesni_enc(ctx, out, in);
513 static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
514 struct scatterlist *src, unsigned int nbytes)
516 struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
518 struct xts_crypt_req req = {
520 .tbuflen = sizeof(buf),
522 .tweak_ctx = aes_ctx(ctx->raw_tweak_ctx),
523 .tweak_fn = aesni_xts_tweak,
524 .crypt_ctx = aes_ctx(ctx->raw_crypt_ctx),
525 .crypt_fn = lrw_xts_encrypt_callback,
529 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
532 ret = xts_crypt(desc, dst, src, nbytes, &req);
538 static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
539 struct scatterlist *src, unsigned int nbytes)
541 struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
543 struct xts_crypt_req req = {
545 .tbuflen = sizeof(buf),
547 .tweak_ctx = aes_ctx(ctx->raw_tweak_ctx),
548 .tweak_fn = aesni_xts_tweak,
549 .crypt_ctx = aes_ctx(ctx->raw_crypt_ctx),
550 .crypt_fn = lrw_xts_decrypt_callback,
554 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
557 ret = xts_crypt(desc, dst, src, nbytes, &req);
564 static int rfc4106_init(struct crypto_tfm *tfm)
566 struct cryptd_aead *cryptd_tfm;
567 struct aesni_rfc4106_gcm_ctx *ctx = (struct aesni_rfc4106_gcm_ctx *)
568 PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN);
569 struct crypto_aead *cryptd_child;
570 struct aesni_rfc4106_gcm_ctx *child_ctx;
571 cryptd_tfm = cryptd_alloc_aead("__driver-gcm-aes-aesni", 0, 0);
572 if (IS_ERR(cryptd_tfm))
573 return PTR_ERR(cryptd_tfm);
575 cryptd_child = cryptd_aead_child(cryptd_tfm);
576 child_ctx = aesni_rfc4106_gcm_ctx_get(cryptd_child);
577 memcpy(child_ctx, ctx, sizeof(*ctx));
578 ctx->cryptd_tfm = cryptd_tfm;
579 tfm->crt_aead.reqsize = sizeof(struct aead_request)
580 + crypto_aead_reqsize(&cryptd_tfm->base);
584 static void rfc4106_exit(struct crypto_tfm *tfm)
586 struct aesni_rfc4106_gcm_ctx *ctx =
587 (struct aesni_rfc4106_gcm_ctx *)
588 PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN);
589 if (!IS_ERR(ctx->cryptd_tfm))
590 cryptd_free_aead(ctx->cryptd_tfm);
595 rfc4106_set_hash_subkey_done(struct crypto_async_request *req, int err)
597 struct aesni_gcm_set_hash_subkey_result *result = req->data;
599 if (err == -EINPROGRESS)
602 complete(&result->completion);
606 rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len)
608 struct crypto_ablkcipher *ctr_tfm;
609 struct ablkcipher_request *req;
611 struct aesni_hash_subkey_req_data *req_data;
613 ctr_tfm = crypto_alloc_ablkcipher("ctr(aes)", 0, 0);
615 return PTR_ERR(ctr_tfm);
617 crypto_ablkcipher_clear_flags(ctr_tfm, ~0);
619 ret = crypto_ablkcipher_setkey(ctr_tfm, key, key_len);
621 goto out_free_ablkcipher;
624 req = ablkcipher_request_alloc(ctr_tfm, GFP_KERNEL);
626 goto out_free_ablkcipher;
628 req_data = kmalloc(sizeof(*req_data), GFP_KERNEL);
630 goto out_free_request;
632 memset(req_data->iv, 0, sizeof(req_data->iv));
634 /* Clear the data in the hash sub key container to zero.*/
635 /* We want to cipher all zeros to create the hash sub key. */
636 memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE);
638 init_completion(&req_data->result.completion);
639 sg_init_one(&req_data->sg, hash_subkey, RFC4106_HASH_SUBKEY_SIZE);
640 ablkcipher_request_set_tfm(req, ctr_tfm);
641 ablkcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP |
642 CRYPTO_TFM_REQ_MAY_BACKLOG,
643 rfc4106_set_hash_subkey_done,
646 ablkcipher_request_set_crypt(req, &req_data->sg,
647 &req_data->sg, RFC4106_HASH_SUBKEY_SIZE, req_data->iv);
649 ret = crypto_ablkcipher_encrypt(req);
650 if (ret == -EINPROGRESS || ret == -EBUSY) {
651 ret = wait_for_completion_interruptible
652 (&req_data->result.completion);
654 ret = req_data->result.err;
658 ablkcipher_request_free(req);
660 crypto_free_ablkcipher(ctr_tfm);
664 static int rfc4106_set_key(struct crypto_aead *parent, const u8 *key,
665 unsigned int key_len)
668 struct crypto_tfm *tfm = crypto_aead_tfm(parent);
669 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent);
670 struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
671 struct aesni_rfc4106_gcm_ctx *child_ctx =
672 aesni_rfc4106_gcm_ctx_get(cryptd_child);
673 u8 *new_key_align, *new_key_mem = NULL;
676 crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
679 /*Account for 4 byte nonce at the end.*/
681 if (key_len != AES_KEYSIZE_128) {
682 crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
686 memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce));
687 /*This must be on a 16 byte boundary!*/
688 if ((unsigned long)(&(ctx->aes_key_expanded.key_enc[0])) % AESNI_ALIGN)
691 if ((unsigned long)key % AESNI_ALIGN) {
692 /*key is not aligned: use an auxuliar aligned pointer*/
693 new_key_mem = kmalloc(key_len+AESNI_ALIGN, GFP_KERNEL);
697 new_key_align = PTR_ALIGN(new_key_mem, AESNI_ALIGN);
698 memcpy(new_key_align, key, key_len);
702 if (!irq_fpu_usable())
703 ret = crypto_aes_expand_key(&(ctx->aes_key_expanded),
707 ret = aesni_set_key(&(ctx->aes_key_expanded), key, key_len);
710 /*This must be on a 16 byte boundary!*/
711 if ((unsigned long)(&(ctx->hash_subkey[0])) % AESNI_ALIGN) {
715 ret = rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
716 memcpy(child_ctx, ctx, sizeof(*ctx));
722 /* This is the Integrity Check Value (aka the authentication tag length and can
723 * be 8, 12 or 16 bytes long. */
724 static int rfc4106_set_authsize(struct crypto_aead *parent,
725 unsigned int authsize)
727 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent);
728 struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
738 crypto_aead_crt(parent)->authsize = authsize;
739 crypto_aead_crt(cryptd_child)->authsize = authsize;
743 static int rfc4106_encrypt(struct aead_request *req)
746 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
747 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
749 if (!irq_fpu_usable()) {
750 struct aead_request *cryptd_req =
751 (struct aead_request *) aead_request_ctx(req);
752 memcpy(cryptd_req, req, sizeof(*req));
753 aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
754 return crypto_aead_encrypt(cryptd_req);
756 struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
758 ret = cryptd_child->base.crt_aead.encrypt(req);
764 static int rfc4106_decrypt(struct aead_request *req)
767 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
768 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
770 if (!irq_fpu_usable()) {
771 struct aead_request *cryptd_req =
772 (struct aead_request *) aead_request_ctx(req);
773 memcpy(cryptd_req, req, sizeof(*req));
774 aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
775 return crypto_aead_decrypt(cryptd_req);
777 struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
779 ret = cryptd_child->base.crt_aead.decrypt(req);
785 static int __driver_rfc4106_encrypt(struct aead_request *req)
787 u8 one_entry_in_sg = 0;
788 u8 *src, *dst, *assoc;
789 __be32 counter = cpu_to_be32(1);
790 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
791 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
792 void *aes_ctx = &(ctx->aes_key_expanded);
793 unsigned long auth_tag_len = crypto_aead_authsize(tfm);
794 u8 iv_tab[16+AESNI_ALIGN];
795 u8* iv = (u8 *) PTR_ALIGN((u8 *)iv_tab, AESNI_ALIGN);
796 struct scatter_walk src_sg_walk;
797 struct scatter_walk assoc_sg_walk;
798 struct scatter_walk dst_sg_walk;
801 /* Assuming we are supporting rfc4106 64-bit extended */
802 /* sequence numbers We need to have the AAD length equal */
803 /* to 8 or 12 bytes */
804 if (unlikely(req->assoclen != 8 && req->assoclen != 12))
807 for (i = 0; i < 4; i++)
808 *(iv+i) = ctx->nonce[i];
809 for (i = 0; i < 8; i++)
810 *(iv+4+i) = req->iv[i];
811 *((__be32 *)(iv+12)) = counter;
813 if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) {
815 scatterwalk_start(&src_sg_walk, req->src);
816 scatterwalk_start(&assoc_sg_walk, req->assoc);
817 src = scatterwalk_map(&src_sg_walk);
818 assoc = scatterwalk_map(&assoc_sg_walk);
820 if (unlikely(req->src != req->dst)) {
821 scatterwalk_start(&dst_sg_walk, req->dst);
822 dst = scatterwalk_map(&dst_sg_walk);
826 /* Allocate memory for src, dst, assoc */
827 src = kmalloc(req->cryptlen + auth_tag_len + req->assoclen,
831 assoc = (src + req->cryptlen + auth_tag_len);
832 scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0);
833 scatterwalk_map_and_copy(assoc, req->assoc, 0,
838 aesni_gcm_enc(aes_ctx, dst, src, (unsigned long)req->cryptlen, iv,
839 ctx->hash_subkey, assoc, (unsigned long)req->assoclen, dst
840 + ((unsigned long)req->cryptlen), auth_tag_len);
842 /* The authTag (aka the Integrity Check Value) needs to be written
843 * back to the packet. */
844 if (one_entry_in_sg) {
845 if (unlikely(req->src != req->dst)) {
846 scatterwalk_unmap(dst);
847 scatterwalk_done(&dst_sg_walk, 0, 0);
849 scatterwalk_unmap(src);
850 scatterwalk_unmap(assoc);
851 scatterwalk_done(&src_sg_walk, 0, 0);
852 scatterwalk_done(&assoc_sg_walk, 0, 0);
854 scatterwalk_map_and_copy(dst, req->dst, 0,
855 req->cryptlen + auth_tag_len, 1);
861 static int __driver_rfc4106_decrypt(struct aead_request *req)
863 u8 one_entry_in_sg = 0;
864 u8 *src, *dst, *assoc;
865 unsigned long tempCipherLen = 0;
866 __be32 counter = cpu_to_be32(1);
868 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
869 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
870 void *aes_ctx = &(ctx->aes_key_expanded);
871 unsigned long auth_tag_len = crypto_aead_authsize(tfm);
872 u8 iv_and_authTag[32+AESNI_ALIGN];
873 u8 *iv = (u8 *) PTR_ALIGN((u8 *)iv_and_authTag, AESNI_ALIGN);
874 u8 *authTag = iv + 16;
875 struct scatter_walk src_sg_walk;
876 struct scatter_walk assoc_sg_walk;
877 struct scatter_walk dst_sg_walk;
880 if (unlikely((req->cryptlen < auth_tag_len) ||
881 (req->assoclen != 8 && req->assoclen != 12)))
883 /* Assuming we are supporting rfc4106 64-bit extended */
884 /* sequence numbers We need to have the AAD length */
885 /* equal to 8 or 12 bytes */
887 tempCipherLen = (unsigned long)(req->cryptlen - auth_tag_len);
889 for (i = 0; i < 4; i++)
890 *(iv+i) = ctx->nonce[i];
891 for (i = 0; i < 8; i++)
892 *(iv+4+i) = req->iv[i];
893 *((__be32 *)(iv+12)) = counter;
895 if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) {
897 scatterwalk_start(&src_sg_walk, req->src);
898 scatterwalk_start(&assoc_sg_walk, req->assoc);
899 src = scatterwalk_map(&src_sg_walk);
900 assoc = scatterwalk_map(&assoc_sg_walk);
902 if (unlikely(req->src != req->dst)) {
903 scatterwalk_start(&dst_sg_walk, req->dst);
904 dst = scatterwalk_map(&dst_sg_walk);
908 /* Allocate memory for src, dst, assoc */
909 src = kmalloc(req->cryptlen + req->assoclen, GFP_ATOMIC);
912 assoc = (src + req->cryptlen + auth_tag_len);
913 scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0);
914 scatterwalk_map_and_copy(assoc, req->assoc, 0,
919 aesni_gcm_dec(aes_ctx, dst, src, tempCipherLen, iv,
920 ctx->hash_subkey, assoc, (unsigned long)req->assoclen,
921 authTag, auth_tag_len);
923 /* Compare generated tag with passed in tag. */
924 retval = memcmp(src + tempCipherLen, authTag, auth_tag_len) ?
927 if (one_entry_in_sg) {
928 if (unlikely(req->src != req->dst)) {
929 scatterwalk_unmap(dst);
930 scatterwalk_done(&dst_sg_walk, 0, 0);
932 scatterwalk_unmap(src);
933 scatterwalk_unmap(assoc);
934 scatterwalk_done(&src_sg_walk, 0, 0);
935 scatterwalk_done(&assoc_sg_walk, 0, 0);
937 scatterwalk_map_and_copy(dst, req->dst, 0, req->cryptlen, 1);
944 static struct crypto_alg aesni_algs[] = { {
946 .cra_driver_name = "aes-aesni",
948 .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
949 .cra_blocksize = AES_BLOCK_SIZE,
950 .cra_ctxsize = sizeof(struct crypto_aes_ctx) +
953 .cra_module = THIS_MODULE,
956 .cia_min_keysize = AES_MIN_KEY_SIZE,
957 .cia_max_keysize = AES_MAX_KEY_SIZE,
958 .cia_setkey = aes_set_key,
959 .cia_encrypt = aes_encrypt,
960 .cia_decrypt = aes_decrypt
964 .cra_name = "__aes-aesni",
965 .cra_driver_name = "__driver-aes-aesni",
967 .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
968 .cra_blocksize = AES_BLOCK_SIZE,
969 .cra_ctxsize = sizeof(struct crypto_aes_ctx) +
972 .cra_module = THIS_MODULE,
975 .cia_min_keysize = AES_MIN_KEY_SIZE,
976 .cia_max_keysize = AES_MAX_KEY_SIZE,
977 .cia_setkey = aes_set_key,
978 .cia_encrypt = __aes_encrypt,
979 .cia_decrypt = __aes_decrypt
983 .cra_name = "__ecb-aes-aesni",
984 .cra_driver_name = "__driver-ecb-aes-aesni",
986 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
987 .cra_blocksize = AES_BLOCK_SIZE,
988 .cra_ctxsize = sizeof(struct crypto_aes_ctx) +
991 .cra_type = &crypto_blkcipher_type,
992 .cra_module = THIS_MODULE,
995 .min_keysize = AES_MIN_KEY_SIZE,
996 .max_keysize = AES_MAX_KEY_SIZE,
997 .setkey = aes_set_key,
998 .encrypt = ecb_encrypt,
999 .decrypt = ecb_decrypt,
1003 .cra_name = "__cbc-aes-aesni",
1004 .cra_driver_name = "__driver-cbc-aes-aesni",
1006 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
1007 .cra_blocksize = AES_BLOCK_SIZE,
1008 .cra_ctxsize = sizeof(struct crypto_aes_ctx) +
1011 .cra_type = &crypto_blkcipher_type,
1012 .cra_module = THIS_MODULE,
1015 .min_keysize = AES_MIN_KEY_SIZE,
1016 .max_keysize = AES_MAX_KEY_SIZE,
1017 .setkey = aes_set_key,
1018 .encrypt = cbc_encrypt,
1019 .decrypt = cbc_decrypt,
1023 .cra_name = "ecb(aes)",
1024 .cra_driver_name = "ecb-aes-aesni",
1025 .cra_priority = 400,
1026 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1027 .cra_blocksize = AES_BLOCK_SIZE,
1028 .cra_ctxsize = sizeof(struct async_helper_ctx),
1030 .cra_type = &crypto_ablkcipher_type,
1031 .cra_module = THIS_MODULE,
1032 .cra_init = ablk_ecb_init,
1033 .cra_exit = ablk_exit,
1036 .min_keysize = AES_MIN_KEY_SIZE,
1037 .max_keysize = AES_MAX_KEY_SIZE,
1038 .setkey = ablk_set_key,
1039 .encrypt = ablk_encrypt,
1040 .decrypt = ablk_decrypt,
1044 .cra_name = "cbc(aes)",
1045 .cra_driver_name = "cbc-aes-aesni",
1046 .cra_priority = 400,
1047 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1048 .cra_blocksize = AES_BLOCK_SIZE,
1049 .cra_ctxsize = sizeof(struct async_helper_ctx),
1051 .cra_type = &crypto_ablkcipher_type,
1052 .cra_module = THIS_MODULE,
1053 .cra_init = ablk_cbc_init,
1054 .cra_exit = ablk_exit,
1057 .min_keysize = AES_MIN_KEY_SIZE,
1058 .max_keysize = AES_MAX_KEY_SIZE,
1059 .ivsize = AES_BLOCK_SIZE,
1060 .setkey = ablk_set_key,
1061 .encrypt = ablk_encrypt,
1062 .decrypt = ablk_decrypt,
1065 #ifdef CONFIG_X86_64
1067 .cra_name = "__ctr-aes-aesni",
1068 .cra_driver_name = "__driver-ctr-aes-aesni",
1070 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
1072 .cra_ctxsize = sizeof(struct crypto_aes_ctx) +
1075 .cra_type = &crypto_blkcipher_type,
1076 .cra_module = THIS_MODULE,
1079 .min_keysize = AES_MIN_KEY_SIZE,
1080 .max_keysize = AES_MAX_KEY_SIZE,
1081 .ivsize = AES_BLOCK_SIZE,
1082 .setkey = aes_set_key,
1083 .encrypt = ctr_crypt,
1084 .decrypt = ctr_crypt,
1088 .cra_name = "ctr(aes)",
1089 .cra_driver_name = "ctr-aes-aesni",
1090 .cra_priority = 400,
1091 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1093 .cra_ctxsize = sizeof(struct async_helper_ctx),
1095 .cra_type = &crypto_ablkcipher_type,
1096 .cra_module = THIS_MODULE,
1097 .cra_init = ablk_ctr_init,
1098 .cra_exit = ablk_exit,
1101 .min_keysize = AES_MIN_KEY_SIZE,
1102 .max_keysize = AES_MAX_KEY_SIZE,
1103 .ivsize = AES_BLOCK_SIZE,
1104 .setkey = ablk_set_key,
1105 .encrypt = ablk_encrypt,
1106 .decrypt = ablk_encrypt,
1111 .cra_name = "__gcm-aes-aesni",
1112 .cra_driver_name = "__driver-gcm-aes-aesni",
1114 .cra_flags = CRYPTO_ALG_TYPE_AEAD,
1116 .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx) +
1119 .cra_type = &crypto_aead_type,
1120 .cra_module = THIS_MODULE,
1123 .encrypt = __driver_rfc4106_encrypt,
1124 .decrypt = __driver_rfc4106_decrypt,
1128 .cra_name = "rfc4106(gcm(aes))",
1129 .cra_driver_name = "rfc4106-gcm-aesni",
1130 .cra_priority = 400,
1131 .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
1133 .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx) +
1136 .cra_type = &crypto_nivaead_type,
1137 .cra_module = THIS_MODULE,
1138 .cra_init = rfc4106_init,
1139 .cra_exit = rfc4106_exit,
1142 .setkey = rfc4106_set_key,
1143 .setauthsize = rfc4106_set_authsize,
1144 .encrypt = rfc4106_encrypt,
1145 .decrypt = rfc4106_decrypt,
1154 .cra_name = "pcbc(aes)",
1155 .cra_driver_name = "pcbc-aes-aesni",
1156 .cra_priority = 400,
1157 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1158 .cra_blocksize = AES_BLOCK_SIZE,
1159 .cra_ctxsize = sizeof(struct async_helper_ctx),
1161 .cra_type = &crypto_ablkcipher_type,
1162 .cra_module = THIS_MODULE,
1163 .cra_init = ablk_pcbc_init,
1164 .cra_exit = ablk_exit,
1167 .min_keysize = AES_MIN_KEY_SIZE,
1168 .max_keysize = AES_MAX_KEY_SIZE,
1169 .ivsize = AES_BLOCK_SIZE,
1170 .setkey = ablk_set_key,
1171 .encrypt = ablk_encrypt,
1172 .decrypt = ablk_decrypt,
1177 .cra_name = "__lrw-aes-aesni",
1178 .cra_driver_name = "__driver-lrw-aes-aesni",
1180 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
1181 .cra_blocksize = AES_BLOCK_SIZE,
1182 .cra_ctxsize = sizeof(struct aesni_lrw_ctx),
1184 .cra_type = &crypto_blkcipher_type,
1185 .cra_module = THIS_MODULE,
1186 .cra_exit = lrw_aesni_exit_tfm,
1189 .min_keysize = AES_MIN_KEY_SIZE + AES_BLOCK_SIZE,
1190 .max_keysize = AES_MAX_KEY_SIZE + AES_BLOCK_SIZE,
1191 .ivsize = AES_BLOCK_SIZE,
1192 .setkey = lrw_aesni_setkey,
1193 .encrypt = lrw_encrypt,
1194 .decrypt = lrw_decrypt,
1198 .cra_name = "__xts-aes-aesni",
1199 .cra_driver_name = "__driver-xts-aes-aesni",
1201 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
1202 .cra_blocksize = AES_BLOCK_SIZE,
1203 .cra_ctxsize = sizeof(struct aesni_xts_ctx),
1205 .cra_type = &crypto_blkcipher_type,
1206 .cra_module = THIS_MODULE,
1209 .min_keysize = 2 * AES_MIN_KEY_SIZE,
1210 .max_keysize = 2 * AES_MAX_KEY_SIZE,
1211 .ivsize = AES_BLOCK_SIZE,
1212 .setkey = xts_aesni_setkey,
1213 .encrypt = xts_encrypt,
1214 .decrypt = xts_decrypt,
1218 .cra_name = "lrw(aes)",
1219 .cra_driver_name = "lrw-aes-aesni",
1220 .cra_priority = 400,
1221 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1222 .cra_blocksize = AES_BLOCK_SIZE,
1223 .cra_ctxsize = sizeof(struct async_helper_ctx),
1225 .cra_type = &crypto_ablkcipher_type,
1226 .cra_module = THIS_MODULE,
1227 .cra_init = ablk_init,
1228 .cra_exit = ablk_exit,
1231 .min_keysize = AES_MIN_KEY_SIZE + AES_BLOCK_SIZE,
1232 .max_keysize = AES_MAX_KEY_SIZE + AES_BLOCK_SIZE,
1233 .ivsize = AES_BLOCK_SIZE,
1234 .setkey = ablk_set_key,
1235 .encrypt = ablk_encrypt,
1236 .decrypt = ablk_decrypt,
1240 .cra_name = "xts(aes)",
1241 .cra_driver_name = "xts-aes-aesni",
1242 .cra_priority = 400,
1243 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1244 .cra_blocksize = AES_BLOCK_SIZE,
1245 .cra_ctxsize = sizeof(struct async_helper_ctx),
1247 .cra_type = &crypto_ablkcipher_type,
1248 .cra_module = THIS_MODULE,
1249 .cra_init = ablk_init,
1250 .cra_exit = ablk_exit,
1253 .min_keysize = 2 * AES_MIN_KEY_SIZE,
1254 .max_keysize = 2 * AES_MAX_KEY_SIZE,
1255 .ivsize = AES_BLOCK_SIZE,
1256 .setkey = ablk_set_key,
1257 .encrypt = ablk_encrypt,
1258 .decrypt = ablk_decrypt,
1264 static const struct x86_cpu_id aesni_cpu_id[] = {
1265 X86_FEATURE_MATCH(X86_FEATURE_AES),
1268 MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id);
1270 static int __init aesni_init(void)
1274 if (!x86_match_cpu(aesni_cpu_id))
1277 err = crypto_fpu_init();
1281 return crypto_register_algs(aesni_algs, ARRAY_SIZE(aesni_algs));
1284 static void __exit aesni_exit(void)
1286 crypto_unregister_algs(aesni_algs, ARRAY_SIZE(aesni_algs));
1291 module_init(aesni_init);
1292 module_exit(aesni_exit);
1294 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized");
1295 MODULE_LICENSE("GPL");
1296 MODULE_ALIAS("aes");