2 * Glue Code for AVX assembler version of Twofish Cipher
4 * Copyright (C) 2012 Johannes Goetzfried
5 * <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 2 of the License, or
10 * (at your option) any later version.
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public License
18 * along with this program; if not, write to the Free Software
19 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
24 #include <linux/module.h>
25 #include <linux/hardirq.h>
26 #include <linux/types.h>
27 #include <linux/crypto.h>
28 #include <linux/err.h>
29 #include <crypto/algapi.h>
30 #include <crypto/twofish.h>
31 #include <crypto/cryptd.h>
32 #include <crypto/b128ops.h>
33 #include <crypto/ctr.h>
34 #include <crypto/lrw.h>
35 #include <crypto/xts.h>
38 #include <asm/xsave.h>
39 #include <asm/crypto/twofish.h>
40 #include <asm/crypto/ablk_helper.h>
41 #include <asm/crypto/glue_helper.h>
42 #include <crypto/scatterwalk.h>
43 #include <linux/workqueue.h>
44 #include <linux/spinlock.h>
46 #define TWOFISH_PARALLEL_BLOCKS 8
48 /* 8-way parallel cipher functions */
49 asmlinkage void twofish_ecb_enc_8way(struct twofish_ctx *ctx, u8 *dst,
51 asmlinkage void twofish_ecb_dec_8way(struct twofish_ctx *ctx, u8 *dst,
54 asmlinkage void twofish_cbc_dec_8way(struct twofish_ctx *ctx, u8 *dst,
56 asmlinkage void twofish_ctr_8way(struct twofish_ctx *ctx, u8 *dst,
57 const u8 *src, le128 *iv);
59 static inline void twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst,
62 __twofish_enc_blk_3way(ctx, dst, src, false);
66 static const struct common_glue_ctx twofish_enc = {
68 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
71 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
72 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_ecb_enc_8way) }
75 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_3way) }
78 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk) }
82 static const struct common_glue_ctx twofish_ctr = {
84 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
87 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
88 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_ctr_8way) }
91 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr_3way) }
94 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr) }
98 static const struct common_glue_ctx twofish_dec = {
100 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
103 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
104 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_ecb_dec_8way) }
107 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk_3way) }
110 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk) }
114 static const struct common_glue_ctx twofish_dec_cbc = {
116 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
119 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
120 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_cbc_dec_8way) }
123 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk_cbc_3way) }
126 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk) }
130 static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
131 struct scatterlist *src, unsigned int nbytes)
133 return glue_ecb_crypt_128bit(&twofish_enc, desc, dst, src, nbytes);
136 static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
137 struct scatterlist *src, unsigned int nbytes)
139 return glue_ecb_crypt_128bit(&twofish_dec, desc, dst, src, nbytes);
142 static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
143 struct scatterlist *src, unsigned int nbytes)
145 return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(twofish_enc_blk), desc,
149 static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
150 struct scatterlist *src, unsigned int nbytes)
152 return glue_cbc_decrypt_128bit(&twofish_dec_cbc, desc, dst, src,
156 static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
157 struct scatterlist *src, unsigned int nbytes)
159 return glue_ctr_crypt_128bit(&twofish_ctr, desc, dst, src, nbytes);
162 static inline bool twofish_fpu_begin(bool fpu_enabled, unsigned int nbytes)
164 return glue_fpu_begin(TF_BLOCK_SIZE, TWOFISH_PARALLEL_BLOCKS, NULL,
165 fpu_enabled, nbytes);
168 static inline void twofish_fpu_end(bool fpu_enabled)
170 glue_fpu_end(fpu_enabled);
174 struct twofish_ctx *ctx;
178 static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
180 const unsigned int bsize = TF_BLOCK_SIZE;
181 struct crypt_priv *ctx = priv;
184 ctx->fpu_enabled = twofish_fpu_begin(ctx->fpu_enabled, nbytes);
186 if (nbytes == bsize * TWOFISH_PARALLEL_BLOCKS) {
187 twofish_ecb_enc_8way(ctx->ctx, srcdst, srcdst);
191 for (i = 0; i < nbytes / (bsize * 3); i++, srcdst += bsize * 3)
192 twofish_enc_blk_3way(ctx->ctx, srcdst, srcdst);
196 for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
197 twofish_enc_blk(ctx->ctx, srcdst, srcdst);
200 static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
202 const unsigned int bsize = TF_BLOCK_SIZE;
203 struct crypt_priv *ctx = priv;
206 ctx->fpu_enabled = twofish_fpu_begin(ctx->fpu_enabled, nbytes);
208 if (nbytes == bsize * TWOFISH_PARALLEL_BLOCKS) {
209 twofish_ecb_dec_8way(ctx->ctx, srcdst, srcdst);
213 for (i = 0; i < nbytes / (bsize * 3); i++, srcdst += bsize * 3)
214 twofish_dec_blk_3way(ctx->ctx, srcdst, srcdst);
218 for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
219 twofish_dec_blk(ctx->ctx, srcdst, srcdst);
222 static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
223 struct scatterlist *src, unsigned int nbytes)
225 struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
226 be128 buf[TWOFISH_PARALLEL_BLOCKS];
227 struct crypt_priv crypt_ctx = {
228 .ctx = &ctx->twofish_ctx,
229 .fpu_enabled = false,
231 struct lrw_crypt_req req = {
233 .tbuflen = sizeof(buf),
235 .table_ctx = &ctx->lrw_table,
236 .crypt_ctx = &crypt_ctx,
237 .crypt_fn = encrypt_callback,
241 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
242 ret = lrw_crypt(desc, dst, src, nbytes, &req);
243 twofish_fpu_end(crypt_ctx.fpu_enabled);
248 static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
249 struct scatterlist *src, unsigned int nbytes)
251 struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
252 be128 buf[TWOFISH_PARALLEL_BLOCKS];
253 struct crypt_priv crypt_ctx = {
254 .ctx = &ctx->twofish_ctx,
255 .fpu_enabled = false,
257 struct lrw_crypt_req req = {
259 .tbuflen = sizeof(buf),
261 .table_ctx = &ctx->lrw_table,
262 .crypt_ctx = &crypt_ctx,
263 .crypt_fn = decrypt_callback,
267 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
268 ret = lrw_crypt(desc, dst, src, nbytes, &req);
269 twofish_fpu_end(crypt_ctx.fpu_enabled);
274 static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
275 struct scatterlist *src, unsigned int nbytes)
277 struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
278 be128 buf[TWOFISH_PARALLEL_BLOCKS];
279 struct crypt_priv crypt_ctx = {
280 .ctx = &ctx->crypt_ctx,
281 .fpu_enabled = false,
283 struct xts_crypt_req req = {
285 .tbuflen = sizeof(buf),
287 .tweak_ctx = &ctx->tweak_ctx,
288 .tweak_fn = XTS_TWEAK_CAST(twofish_enc_blk),
289 .crypt_ctx = &crypt_ctx,
290 .crypt_fn = encrypt_callback,
294 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
295 ret = xts_crypt(desc, dst, src, nbytes, &req);
296 twofish_fpu_end(crypt_ctx.fpu_enabled);
301 static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
302 struct scatterlist *src, unsigned int nbytes)
304 struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
305 be128 buf[TWOFISH_PARALLEL_BLOCKS];
306 struct crypt_priv crypt_ctx = {
307 .ctx = &ctx->crypt_ctx,
308 .fpu_enabled = false,
310 struct xts_crypt_req req = {
312 .tbuflen = sizeof(buf),
314 .tweak_ctx = &ctx->tweak_ctx,
315 .tweak_fn = XTS_TWEAK_CAST(twofish_enc_blk),
316 .crypt_ctx = &crypt_ctx,
317 .crypt_fn = decrypt_callback,
321 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
322 ret = xts_crypt(desc, dst, src, nbytes, &req);
323 twofish_fpu_end(crypt_ctx.fpu_enabled);
328 static struct crypto_alg twofish_algs[10] = { {
329 .cra_name = "__ecb-twofish-avx",
330 .cra_driver_name = "__driver-ecb-twofish-avx",
332 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
333 .cra_blocksize = TF_BLOCK_SIZE,
334 .cra_ctxsize = sizeof(struct twofish_ctx),
336 .cra_type = &crypto_blkcipher_type,
337 .cra_module = THIS_MODULE,
340 .min_keysize = TF_MIN_KEY_SIZE,
341 .max_keysize = TF_MAX_KEY_SIZE,
342 .setkey = twofish_setkey,
343 .encrypt = ecb_encrypt,
344 .decrypt = ecb_decrypt,
348 .cra_name = "__cbc-twofish-avx",
349 .cra_driver_name = "__driver-cbc-twofish-avx",
351 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
352 .cra_blocksize = TF_BLOCK_SIZE,
353 .cra_ctxsize = sizeof(struct twofish_ctx),
355 .cra_type = &crypto_blkcipher_type,
356 .cra_module = THIS_MODULE,
359 .min_keysize = TF_MIN_KEY_SIZE,
360 .max_keysize = TF_MAX_KEY_SIZE,
361 .setkey = twofish_setkey,
362 .encrypt = cbc_encrypt,
363 .decrypt = cbc_decrypt,
367 .cra_name = "__ctr-twofish-avx",
368 .cra_driver_name = "__driver-ctr-twofish-avx",
370 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
372 .cra_ctxsize = sizeof(struct twofish_ctx),
374 .cra_type = &crypto_blkcipher_type,
375 .cra_module = THIS_MODULE,
378 .min_keysize = TF_MIN_KEY_SIZE,
379 .max_keysize = TF_MAX_KEY_SIZE,
380 .ivsize = TF_BLOCK_SIZE,
381 .setkey = twofish_setkey,
382 .encrypt = ctr_crypt,
383 .decrypt = ctr_crypt,
387 .cra_name = "__lrw-twofish-avx",
388 .cra_driver_name = "__driver-lrw-twofish-avx",
390 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
391 .cra_blocksize = TF_BLOCK_SIZE,
392 .cra_ctxsize = sizeof(struct twofish_lrw_ctx),
394 .cra_type = &crypto_blkcipher_type,
395 .cra_module = THIS_MODULE,
396 .cra_exit = lrw_twofish_exit_tfm,
399 .min_keysize = TF_MIN_KEY_SIZE +
401 .max_keysize = TF_MAX_KEY_SIZE +
403 .ivsize = TF_BLOCK_SIZE,
404 .setkey = lrw_twofish_setkey,
405 .encrypt = lrw_encrypt,
406 .decrypt = lrw_decrypt,
410 .cra_name = "__xts-twofish-avx",
411 .cra_driver_name = "__driver-xts-twofish-avx",
413 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
414 .cra_blocksize = TF_BLOCK_SIZE,
415 .cra_ctxsize = sizeof(struct twofish_xts_ctx),
417 .cra_type = &crypto_blkcipher_type,
418 .cra_module = THIS_MODULE,
421 .min_keysize = TF_MIN_KEY_SIZE * 2,
422 .max_keysize = TF_MAX_KEY_SIZE * 2,
423 .ivsize = TF_BLOCK_SIZE,
424 .setkey = xts_twofish_setkey,
425 .encrypt = xts_encrypt,
426 .decrypt = xts_decrypt,
430 .cra_name = "ecb(twofish)",
431 .cra_driver_name = "ecb-twofish-avx",
433 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
434 .cra_blocksize = TF_BLOCK_SIZE,
435 .cra_ctxsize = sizeof(struct async_helper_ctx),
437 .cra_type = &crypto_ablkcipher_type,
438 .cra_module = THIS_MODULE,
439 .cra_init = ablk_init,
440 .cra_exit = ablk_exit,
443 .min_keysize = TF_MIN_KEY_SIZE,
444 .max_keysize = TF_MAX_KEY_SIZE,
445 .setkey = ablk_set_key,
446 .encrypt = ablk_encrypt,
447 .decrypt = ablk_decrypt,
451 .cra_name = "cbc(twofish)",
452 .cra_driver_name = "cbc-twofish-avx",
454 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
455 .cra_blocksize = TF_BLOCK_SIZE,
456 .cra_ctxsize = sizeof(struct async_helper_ctx),
458 .cra_type = &crypto_ablkcipher_type,
459 .cra_module = THIS_MODULE,
460 .cra_init = ablk_init,
461 .cra_exit = ablk_exit,
464 .min_keysize = TF_MIN_KEY_SIZE,
465 .max_keysize = TF_MAX_KEY_SIZE,
466 .ivsize = TF_BLOCK_SIZE,
467 .setkey = ablk_set_key,
468 .encrypt = __ablk_encrypt,
469 .decrypt = ablk_decrypt,
473 .cra_name = "ctr(twofish)",
474 .cra_driver_name = "ctr-twofish-avx",
476 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
478 .cra_ctxsize = sizeof(struct async_helper_ctx),
480 .cra_type = &crypto_ablkcipher_type,
481 .cra_module = THIS_MODULE,
482 .cra_init = ablk_init,
483 .cra_exit = ablk_exit,
486 .min_keysize = TF_MIN_KEY_SIZE,
487 .max_keysize = TF_MAX_KEY_SIZE,
488 .ivsize = TF_BLOCK_SIZE,
489 .setkey = ablk_set_key,
490 .encrypt = ablk_encrypt,
491 .decrypt = ablk_encrypt,
496 .cra_name = "lrw(twofish)",
497 .cra_driver_name = "lrw-twofish-avx",
499 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
500 .cra_blocksize = TF_BLOCK_SIZE,
501 .cra_ctxsize = sizeof(struct async_helper_ctx),
503 .cra_type = &crypto_ablkcipher_type,
504 .cra_module = THIS_MODULE,
505 .cra_init = ablk_init,
506 .cra_exit = ablk_exit,
509 .min_keysize = TF_MIN_KEY_SIZE +
511 .max_keysize = TF_MAX_KEY_SIZE +
513 .ivsize = TF_BLOCK_SIZE,
514 .setkey = ablk_set_key,
515 .encrypt = ablk_encrypt,
516 .decrypt = ablk_decrypt,
520 .cra_name = "xts(twofish)",
521 .cra_driver_name = "xts-twofish-avx",
523 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
524 .cra_blocksize = TF_BLOCK_SIZE,
525 .cra_ctxsize = sizeof(struct async_helper_ctx),
527 .cra_type = &crypto_ablkcipher_type,
528 .cra_module = THIS_MODULE,
529 .cra_init = ablk_init,
530 .cra_exit = ablk_exit,
533 .min_keysize = TF_MIN_KEY_SIZE * 2,
534 .max_keysize = TF_MAX_KEY_SIZE * 2,
535 .ivsize = TF_BLOCK_SIZE,
536 .setkey = ablk_set_key,
537 .encrypt = ablk_encrypt,
538 .decrypt = ablk_decrypt,
543 static int __init twofish_init(void)
547 if (!cpu_has_avx || !cpu_has_osxsave) {
548 printk(KERN_INFO "AVX instructions are not detected.\n");
552 xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
553 if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) {
554 printk(KERN_INFO "AVX detected but unusable.\n");
558 return crypto_register_algs(twofish_algs, ARRAY_SIZE(twofish_algs));
561 static void __exit twofish_exit(void)
563 crypto_unregister_algs(twofish_algs, ARRAY_SIZE(twofish_algs));
566 module_init(twofish_init);
567 module_exit(twofish_exit);
569 MODULE_DESCRIPTION("Twofish Cipher Algorithm, AVX optimized");
570 MODULE_LICENSE("GPL");
571 MODULE_ALIAS("twofish");