2 * Glue Code for AVX assembler versions of Serpent Cipher
4 * Copyright (C) 2012 Johannes Goetzfried
5 * <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
7 * Glue code based on serpent_sse2_glue.c by:
8 * Copyright (C) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License as published by
12 * the Free Software Foundation; either version 2 of the License, or
13 * (at your option) any later version.
15 * This program is distributed in the hope that it will be useful,
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 * GNU General Public License for more details.
20 * You should have received a copy of the GNU General Public License
21 * along with this program; if not, write to the Free Software
22 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
27 #include <linux/module.h>
28 #include <linux/hardirq.h>
29 #include <linux/types.h>
30 #include <linux/crypto.h>
31 #include <linux/err.h>
32 #include <crypto/algapi.h>
33 #include <crypto/serpent.h>
34 #include <crypto/cryptd.h>
35 #include <crypto/b128ops.h>
36 #include <crypto/ctr.h>
37 #include <crypto/lrw.h>
38 #include <crypto/xts.h>
40 #include <asm/xsave.h>
41 #include <asm/crypto/serpent-avx.h>
42 #include <asm/crypto/ablk_helper.h>
43 #include <asm/crypto/glue_helper.h>
45 static void serpent_crypt_ctr(void *ctx, u128 *dst, const u128 *src, le128 *iv)
49 le128_to_be128(&ctrblk, iv);
52 __serpent_encrypt(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk);
53 u128_xor(dst, src, (u128 *)&ctrblk);
56 static const struct common_glue_ctx serpent_enc = {
58 .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
61 .num_blocks = SERPENT_PARALLEL_BLOCKS,
62 .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_enc_8way_avx) }
65 .fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_encrypt) }
69 static const struct common_glue_ctx serpent_ctr = {
71 .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
74 .num_blocks = SERPENT_PARALLEL_BLOCKS,
75 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_ctr_8way_avx) }
78 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_crypt_ctr) }
82 static const struct common_glue_ctx serpent_dec = {
84 .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
87 .num_blocks = SERPENT_PARALLEL_BLOCKS,
88 .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_dec_8way_avx) }
91 .fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_decrypt) }
95 static const struct common_glue_ctx serpent_dec_cbc = {
97 .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
100 .num_blocks = SERPENT_PARALLEL_BLOCKS,
101 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(serpent_cbc_dec_8way_avx) }
104 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(__serpent_decrypt) }
108 static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
109 struct scatterlist *src, unsigned int nbytes)
111 return glue_ecb_crypt_128bit(&serpent_enc, desc, dst, src, nbytes);
114 static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
115 struct scatterlist *src, unsigned int nbytes)
117 return glue_ecb_crypt_128bit(&serpent_dec, desc, dst, src, nbytes);
120 static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
121 struct scatterlist *src, unsigned int nbytes)
123 return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(__serpent_encrypt), desc,
127 static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
128 struct scatterlist *src, unsigned int nbytes)
130 return glue_cbc_decrypt_128bit(&serpent_dec_cbc, desc, dst, src,
134 static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
135 struct scatterlist *src, unsigned int nbytes)
137 return glue_ctr_crypt_128bit(&serpent_ctr, desc, dst, src, nbytes);
140 static inline bool serpent_fpu_begin(bool fpu_enabled, unsigned int nbytes)
142 return glue_fpu_begin(SERPENT_BLOCK_SIZE, SERPENT_PARALLEL_BLOCKS,
143 NULL, fpu_enabled, nbytes);
146 static inline void serpent_fpu_end(bool fpu_enabled)
148 glue_fpu_end(fpu_enabled);
152 struct serpent_ctx *ctx;
156 static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
158 const unsigned int bsize = SERPENT_BLOCK_SIZE;
159 struct crypt_priv *ctx = priv;
162 ctx->fpu_enabled = serpent_fpu_begin(ctx->fpu_enabled, nbytes);
164 if (nbytes == bsize * SERPENT_PARALLEL_BLOCKS) {
165 serpent_ecb_enc_8way_avx(ctx->ctx, srcdst, srcdst);
169 for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
170 __serpent_encrypt(ctx->ctx, srcdst, srcdst);
173 static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
175 const unsigned int bsize = SERPENT_BLOCK_SIZE;
176 struct crypt_priv *ctx = priv;
179 ctx->fpu_enabled = serpent_fpu_begin(ctx->fpu_enabled, nbytes);
181 if (nbytes == bsize * SERPENT_PARALLEL_BLOCKS) {
182 serpent_ecb_dec_8way_avx(ctx->ctx, srcdst, srcdst);
186 for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
187 __serpent_decrypt(ctx->ctx, srcdst, srcdst);
190 struct serpent_lrw_ctx {
191 struct lrw_table_ctx lrw_table;
192 struct serpent_ctx serpent_ctx;
195 static int lrw_serpent_setkey(struct crypto_tfm *tfm, const u8 *key,
198 struct serpent_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
201 err = __serpent_setkey(&ctx->serpent_ctx, key, keylen -
206 return lrw_init_table(&ctx->lrw_table, key + keylen -
210 static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
211 struct scatterlist *src, unsigned int nbytes)
213 struct serpent_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
214 be128 buf[SERPENT_PARALLEL_BLOCKS];
215 struct crypt_priv crypt_ctx = {
216 .ctx = &ctx->serpent_ctx,
217 .fpu_enabled = false,
219 struct lrw_crypt_req req = {
221 .tbuflen = sizeof(buf),
223 .table_ctx = &ctx->lrw_table,
224 .crypt_ctx = &crypt_ctx,
225 .crypt_fn = encrypt_callback,
229 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
230 ret = lrw_crypt(desc, dst, src, nbytes, &req);
231 serpent_fpu_end(crypt_ctx.fpu_enabled);
236 static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
237 struct scatterlist *src, unsigned int nbytes)
239 struct serpent_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
240 be128 buf[SERPENT_PARALLEL_BLOCKS];
241 struct crypt_priv crypt_ctx = {
242 .ctx = &ctx->serpent_ctx,
243 .fpu_enabled = false,
245 struct lrw_crypt_req req = {
247 .tbuflen = sizeof(buf),
249 .table_ctx = &ctx->lrw_table,
250 .crypt_ctx = &crypt_ctx,
251 .crypt_fn = decrypt_callback,
255 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
256 ret = lrw_crypt(desc, dst, src, nbytes, &req);
257 serpent_fpu_end(crypt_ctx.fpu_enabled);
262 static void lrw_exit_tfm(struct crypto_tfm *tfm)
264 struct serpent_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
266 lrw_free_table(&ctx->lrw_table);
269 struct serpent_xts_ctx {
270 struct serpent_ctx tweak_ctx;
271 struct serpent_ctx crypt_ctx;
274 static int xts_serpent_setkey(struct crypto_tfm *tfm, const u8 *key,
277 struct serpent_xts_ctx *ctx = crypto_tfm_ctx(tfm);
278 u32 *flags = &tfm->crt_flags;
281 /* key consists of keys of equal size concatenated, therefore
282 * the length must be even
285 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
289 /* first half of xts-key is for crypt */
290 err = __serpent_setkey(&ctx->crypt_ctx, key, keylen / 2);
294 /* second half of xts-key is for tweak */
295 return __serpent_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2);
298 static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
299 struct scatterlist *src, unsigned int nbytes)
301 struct serpent_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
302 be128 buf[SERPENT_PARALLEL_BLOCKS];
303 struct crypt_priv crypt_ctx = {
304 .ctx = &ctx->crypt_ctx,
305 .fpu_enabled = false,
307 struct xts_crypt_req req = {
309 .tbuflen = sizeof(buf),
311 .tweak_ctx = &ctx->tweak_ctx,
312 .tweak_fn = XTS_TWEAK_CAST(__serpent_encrypt),
313 .crypt_ctx = &crypt_ctx,
314 .crypt_fn = encrypt_callback,
318 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
319 ret = xts_crypt(desc, dst, src, nbytes, &req);
320 serpent_fpu_end(crypt_ctx.fpu_enabled);
325 static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
326 struct scatterlist *src, unsigned int nbytes)
328 struct serpent_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
329 be128 buf[SERPENT_PARALLEL_BLOCKS];
330 struct crypt_priv crypt_ctx = {
331 .ctx = &ctx->crypt_ctx,
332 .fpu_enabled = false,
334 struct xts_crypt_req req = {
336 .tbuflen = sizeof(buf),
338 .tweak_ctx = &ctx->tweak_ctx,
339 .tweak_fn = XTS_TWEAK_CAST(__serpent_encrypt),
340 .crypt_ctx = &crypt_ctx,
341 .crypt_fn = decrypt_callback,
345 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
346 ret = xts_crypt(desc, dst, src, nbytes, &req);
347 serpent_fpu_end(crypt_ctx.fpu_enabled);
352 static struct crypto_alg serpent_algs[10] = { {
353 .cra_name = "__ecb-serpent-avx",
354 .cra_driver_name = "__driver-ecb-serpent-avx",
356 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
357 .cra_blocksize = SERPENT_BLOCK_SIZE,
358 .cra_ctxsize = sizeof(struct serpent_ctx),
360 .cra_type = &crypto_blkcipher_type,
361 .cra_module = THIS_MODULE,
364 .min_keysize = SERPENT_MIN_KEY_SIZE,
365 .max_keysize = SERPENT_MAX_KEY_SIZE,
366 .setkey = serpent_setkey,
367 .encrypt = ecb_encrypt,
368 .decrypt = ecb_decrypt,
372 .cra_name = "__cbc-serpent-avx",
373 .cra_driver_name = "__driver-cbc-serpent-avx",
375 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
376 .cra_blocksize = SERPENT_BLOCK_SIZE,
377 .cra_ctxsize = sizeof(struct serpent_ctx),
379 .cra_type = &crypto_blkcipher_type,
380 .cra_module = THIS_MODULE,
383 .min_keysize = SERPENT_MIN_KEY_SIZE,
384 .max_keysize = SERPENT_MAX_KEY_SIZE,
385 .setkey = serpent_setkey,
386 .encrypt = cbc_encrypt,
387 .decrypt = cbc_decrypt,
391 .cra_name = "__ctr-serpent-avx",
392 .cra_driver_name = "__driver-ctr-serpent-avx",
394 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
396 .cra_ctxsize = sizeof(struct serpent_ctx),
398 .cra_type = &crypto_blkcipher_type,
399 .cra_module = THIS_MODULE,
402 .min_keysize = SERPENT_MIN_KEY_SIZE,
403 .max_keysize = SERPENT_MAX_KEY_SIZE,
404 .ivsize = SERPENT_BLOCK_SIZE,
405 .setkey = serpent_setkey,
406 .encrypt = ctr_crypt,
407 .decrypt = ctr_crypt,
411 .cra_name = "__lrw-serpent-avx",
412 .cra_driver_name = "__driver-lrw-serpent-avx",
414 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
415 .cra_blocksize = SERPENT_BLOCK_SIZE,
416 .cra_ctxsize = sizeof(struct serpent_lrw_ctx),
418 .cra_type = &crypto_blkcipher_type,
419 .cra_module = THIS_MODULE,
420 .cra_exit = lrw_exit_tfm,
423 .min_keysize = SERPENT_MIN_KEY_SIZE +
425 .max_keysize = SERPENT_MAX_KEY_SIZE +
427 .ivsize = SERPENT_BLOCK_SIZE,
428 .setkey = lrw_serpent_setkey,
429 .encrypt = lrw_encrypt,
430 .decrypt = lrw_decrypt,
434 .cra_name = "__xts-serpent-avx",
435 .cra_driver_name = "__driver-xts-serpent-avx",
437 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
438 .cra_blocksize = SERPENT_BLOCK_SIZE,
439 .cra_ctxsize = sizeof(struct serpent_xts_ctx),
441 .cra_type = &crypto_blkcipher_type,
442 .cra_module = THIS_MODULE,
445 .min_keysize = SERPENT_MIN_KEY_SIZE * 2,
446 .max_keysize = SERPENT_MAX_KEY_SIZE * 2,
447 .ivsize = SERPENT_BLOCK_SIZE,
448 .setkey = xts_serpent_setkey,
449 .encrypt = xts_encrypt,
450 .decrypt = xts_decrypt,
454 .cra_name = "ecb(serpent)",
455 .cra_driver_name = "ecb-serpent-avx",
457 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
458 .cra_blocksize = SERPENT_BLOCK_SIZE,
459 .cra_ctxsize = sizeof(struct async_helper_ctx),
461 .cra_type = &crypto_ablkcipher_type,
462 .cra_module = THIS_MODULE,
463 .cra_init = ablk_init,
464 .cra_exit = ablk_exit,
467 .min_keysize = SERPENT_MIN_KEY_SIZE,
468 .max_keysize = SERPENT_MAX_KEY_SIZE,
469 .setkey = ablk_set_key,
470 .encrypt = ablk_encrypt,
471 .decrypt = ablk_decrypt,
475 .cra_name = "cbc(serpent)",
476 .cra_driver_name = "cbc-serpent-avx",
478 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
479 .cra_blocksize = SERPENT_BLOCK_SIZE,
480 .cra_ctxsize = sizeof(struct async_helper_ctx),
482 .cra_type = &crypto_ablkcipher_type,
483 .cra_module = THIS_MODULE,
484 .cra_init = ablk_init,
485 .cra_exit = ablk_exit,
488 .min_keysize = SERPENT_MIN_KEY_SIZE,
489 .max_keysize = SERPENT_MAX_KEY_SIZE,
490 .ivsize = SERPENT_BLOCK_SIZE,
491 .setkey = ablk_set_key,
492 .encrypt = __ablk_encrypt,
493 .decrypt = ablk_decrypt,
497 .cra_name = "ctr(serpent)",
498 .cra_driver_name = "ctr-serpent-avx",
500 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
502 .cra_ctxsize = sizeof(struct async_helper_ctx),
504 .cra_type = &crypto_ablkcipher_type,
505 .cra_module = THIS_MODULE,
506 .cra_init = ablk_init,
507 .cra_exit = ablk_exit,
510 .min_keysize = SERPENT_MIN_KEY_SIZE,
511 .max_keysize = SERPENT_MAX_KEY_SIZE,
512 .ivsize = SERPENT_BLOCK_SIZE,
513 .setkey = ablk_set_key,
514 .encrypt = ablk_encrypt,
515 .decrypt = ablk_encrypt,
520 .cra_name = "lrw(serpent)",
521 .cra_driver_name = "lrw-serpent-avx",
523 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
524 .cra_blocksize = SERPENT_BLOCK_SIZE,
525 .cra_ctxsize = sizeof(struct async_helper_ctx),
527 .cra_type = &crypto_ablkcipher_type,
528 .cra_module = THIS_MODULE,
529 .cra_init = ablk_init,
530 .cra_exit = ablk_exit,
533 .min_keysize = SERPENT_MIN_KEY_SIZE +
535 .max_keysize = SERPENT_MAX_KEY_SIZE +
537 .ivsize = SERPENT_BLOCK_SIZE,
538 .setkey = ablk_set_key,
539 .encrypt = ablk_encrypt,
540 .decrypt = ablk_decrypt,
544 .cra_name = "xts(serpent)",
545 .cra_driver_name = "xts-serpent-avx",
547 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
548 .cra_blocksize = SERPENT_BLOCK_SIZE,
549 .cra_ctxsize = sizeof(struct async_helper_ctx),
551 .cra_type = &crypto_ablkcipher_type,
552 .cra_module = THIS_MODULE,
553 .cra_init = ablk_init,
554 .cra_exit = ablk_exit,
557 .min_keysize = SERPENT_MIN_KEY_SIZE * 2,
558 .max_keysize = SERPENT_MAX_KEY_SIZE * 2,
559 .ivsize = SERPENT_BLOCK_SIZE,
560 .setkey = ablk_set_key,
561 .encrypt = ablk_encrypt,
562 .decrypt = ablk_decrypt,
567 static int __init serpent_init(void)
571 if (!cpu_has_avx || !cpu_has_osxsave) {
572 printk(KERN_INFO "AVX instructions are not detected.\n");
576 xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
577 if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) {
578 printk(KERN_INFO "AVX detected but unusable.\n");
582 return crypto_register_algs(serpent_algs, ARRAY_SIZE(serpent_algs));
585 static void __exit serpent_exit(void)
587 crypto_unregister_algs(serpent_algs, ARRAY_SIZE(serpent_algs));
590 module_init(serpent_init);
591 module_exit(serpent_exit);
593 MODULE_DESCRIPTION("Serpent Cipher Algorithm, AVX optimized");
594 MODULE_LICENSE("GPL");
595 MODULE_ALIAS("serpent");