2 * Glue Code for x86_64/AVX/AES-NI assembler optimized version of Camellia
4 * Copyright © 2012-2013 Jussi Kivilinna <jussi.kivilinna@iki.fi>
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
13 #include <linux/module.h>
14 #include <linux/types.h>
15 #include <linux/crypto.h>
16 #include <linux/err.h>
17 #include <crypto/ablk_helper.h>
18 #include <crypto/algapi.h>
19 #include <crypto/ctr.h>
20 #include <crypto/lrw.h>
21 #include <crypto/xts.h>
22 #include <asm/fpu/api.h>
23 #include <asm/crypto/camellia.h>
24 #include <asm/crypto/glue_helper.h>
26 #define CAMELLIA_AESNI_PARALLEL_BLOCKS 16
28 /* 16-way parallel cipher functions (avx/aes-ni) */
29 asmlinkage void camellia_ecb_enc_16way(struct camellia_ctx *ctx, u8 *dst,
31 EXPORT_SYMBOL_GPL(camellia_ecb_enc_16way);
33 asmlinkage void camellia_ecb_dec_16way(struct camellia_ctx *ctx, u8 *dst,
35 EXPORT_SYMBOL_GPL(camellia_ecb_dec_16way);
37 asmlinkage void camellia_cbc_dec_16way(struct camellia_ctx *ctx, u8 *dst,
39 EXPORT_SYMBOL_GPL(camellia_cbc_dec_16way);
41 asmlinkage void camellia_ctr_16way(struct camellia_ctx *ctx, u8 *dst,
42 const u8 *src, le128 *iv);
43 EXPORT_SYMBOL_GPL(camellia_ctr_16way);
45 asmlinkage void camellia_xts_enc_16way(struct camellia_ctx *ctx, u8 *dst,
46 const u8 *src, le128 *iv);
47 EXPORT_SYMBOL_GPL(camellia_xts_enc_16way);
49 asmlinkage void camellia_xts_dec_16way(struct camellia_ctx *ctx, u8 *dst,
50 const u8 *src, le128 *iv);
51 EXPORT_SYMBOL_GPL(camellia_xts_dec_16way);
53 void camellia_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv)
55 glue_xts_crypt_128bit_one(ctx, dst, src, iv,
56 GLUE_FUNC_CAST(camellia_enc_blk));
58 EXPORT_SYMBOL_GPL(camellia_xts_enc);
60 void camellia_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv)
62 glue_xts_crypt_128bit_one(ctx, dst, src, iv,
63 GLUE_FUNC_CAST(camellia_dec_blk));
65 EXPORT_SYMBOL_GPL(camellia_xts_dec);
67 static const struct common_glue_ctx camellia_enc = {
69 .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
72 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
73 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_ecb_enc_16way) }
76 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_enc_blk_2way) }
79 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_enc_blk) }
83 static const struct common_glue_ctx camellia_ctr = {
85 .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
88 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
89 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_ctr_16way) }
92 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_crypt_ctr_2way) }
95 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_crypt_ctr) }
99 static const struct common_glue_ctx camellia_enc_xts = {
101 .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
104 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
105 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_enc_16way) }
108 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_enc) }
112 static const struct common_glue_ctx camellia_dec = {
114 .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
117 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
118 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_ecb_dec_16way) }
121 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_dec_blk_2way) }
124 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_dec_blk) }
128 static const struct common_glue_ctx camellia_dec_cbc = {
130 .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
133 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
134 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_cbc_dec_16way) }
137 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_decrypt_cbc_2way) }
140 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_dec_blk) }
144 static const struct common_glue_ctx camellia_dec_xts = {
146 .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
149 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
150 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_dec_16way) }
153 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_dec) }
157 static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
158 struct scatterlist *src, unsigned int nbytes)
160 return glue_ecb_crypt_128bit(&camellia_enc, desc, dst, src, nbytes);
163 static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
164 struct scatterlist *src, unsigned int nbytes)
166 return glue_ecb_crypt_128bit(&camellia_dec, desc, dst, src, nbytes);
169 static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
170 struct scatterlist *src, unsigned int nbytes)
172 return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(camellia_enc_blk), desc,
176 static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
177 struct scatterlist *src, unsigned int nbytes)
179 return glue_cbc_decrypt_128bit(&camellia_dec_cbc, desc, dst, src,
183 static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
184 struct scatterlist *src, unsigned int nbytes)
186 return glue_ctr_crypt_128bit(&camellia_ctr, desc, dst, src, nbytes);
189 static inline bool camellia_fpu_begin(bool fpu_enabled, unsigned int nbytes)
191 return glue_fpu_begin(CAMELLIA_BLOCK_SIZE,
192 CAMELLIA_AESNI_PARALLEL_BLOCKS, NULL, fpu_enabled,
196 static inline void camellia_fpu_end(bool fpu_enabled)
198 glue_fpu_end(fpu_enabled);
201 static int camellia_setkey(struct crypto_tfm *tfm, const u8 *in_key,
202 unsigned int key_len)
204 return __camellia_setkey(crypto_tfm_ctx(tfm), in_key, key_len,
209 struct camellia_ctx *ctx;
213 static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
215 const unsigned int bsize = CAMELLIA_BLOCK_SIZE;
216 struct crypt_priv *ctx = priv;
219 ctx->fpu_enabled = camellia_fpu_begin(ctx->fpu_enabled, nbytes);
221 if (nbytes >= CAMELLIA_AESNI_PARALLEL_BLOCKS * bsize) {
222 camellia_ecb_enc_16way(ctx->ctx, srcdst, srcdst);
223 srcdst += bsize * CAMELLIA_AESNI_PARALLEL_BLOCKS;
224 nbytes -= bsize * CAMELLIA_AESNI_PARALLEL_BLOCKS;
227 while (nbytes >= CAMELLIA_PARALLEL_BLOCKS * bsize) {
228 camellia_enc_blk_2way(ctx->ctx, srcdst, srcdst);
229 srcdst += bsize * CAMELLIA_PARALLEL_BLOCKS;
230 nbytes -= bsize * CAMELLIA_PARALLEL_BLOCKS;
233 for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
234 camellia_enc_blk(ctx->ctx, srcdst, srcdst);
237 static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
239 const unsigned int bsize = CAMELLIA_BLOCK_SIZE;
240 struct crypt_priv *ctx = priv;
243 ctx->fpu_enabled = camellia_fpu_begin(ctx->fpu_enabled, nbytes);
245 if (nbytes >= CAMELLIA_AESNI_PARALLEL_BLOCKS * bsize) {
246 camellia_ecb_dec_16way(ctx->ctx, srcdst, srcdst);
247 srcdst += bsize * CAMELLIA_AESNI_PARALLEL_BLOCKS;
248 nbytes -= bsize * CAMELLIA_AESNI_PARALLEL_BLOCKS;
251 while (nbytes >= CAMELLIA_PARALLEL_BLOCKS * bsize) {
252 camellia_dec_blk_2way(ctx->ctx, srcdst, srcdst);
253 srcdst += bsize * CAMELLIA_PARALLEL_BLOCKS;
254 nbytes -= bsize * CAMELLIA_PARALLEL_BLOCKS;
257 for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
258 camellia_dec_blk(ctx->ctx, srcdst, srcdst);
261 static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
262 struct scatterlist *src, unsigned int nbytes)
264 struct camellia_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
265 be128 buf[CAMELLIA_AESNI_PARALLEL_BLOCKS];
266 struct crypt_priv crypt_ctx = {
267 .ctx = &ctx->camellia_ctx,
268 .fpu_enabled = false,
270 struct lrw_crypt_req req = {
272 .tbuflen = sizeof(buf),
274 .table_ctx = &ctx->lrw_table,
275 .crypt_ctx = &crypt_ctx,
276 .crypt_fn = encrypt_callback,
280 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
281 ret = lrw_crypt(desc, dst, src, nbytes, &req);
282 camellia_fpu_end(crypt_ctx.fpu_enabled);
287 static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
288 struct scatterlist *src, unsigned int nbytes)
290 struct camellia_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
291 be128 buf[CAMELLIA_AESNI_PARALLEL_BLOCKS];
292 struct crypt_priv crypt_ctx = {
293 .ctx = &ctx->camellia_ctx,
294 .fpu_enabled = false,
296 struct lrw_crypt_req req = {
298 .tbuflen = sizeof(buf),
300 .table_ctx = &ctx->lrw_table,
301 .crypt_ctx = &crypt_ctx,
302 .crypt_fn = decrypt_callback,
306 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
307 ret = lrw_crypt(desc, dst, src, nbytes, &req);
308 camellia_fpu_end(crypt_ctx.fpu_enabled);
313 static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
314 struct scatterlist *src, unsigned int nbytes)
316 struct camellia_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
318 return glue_xts_crypt_128bit(&camellia_enc_xts, desc, dst, src, nbytes,
319 XTS_TWEAK_CAST(camellia_enc_blk),
320 &ctx->tweak_ctx, &ctx->crypt_ctx);
323 static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
324 struct scatterlist *src, unsigned int nbytes)
326 struct camellia_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
328 return glue_xts_crypt_128bit(&camellia_dec_xts, desc, dst, src, nbytes,
329 XTS_TWEAK_CAST(camellia_enc_blk),
330 &ctx->tweak_ctx, &ctx->crypt_ctx);
333 static struct crypto_alg cmll_algs[10] = { {
334 .cra_name = "__ecb-camellia-aesni",
335 .cra_driver_name = "__driver-ecb-camellia-aesni",
337 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER |
339 .cra_blocksize = CAMELLIA_BLOCK_SIZE,
340 .cra_ctxsize = sizeof(struct camellia_ctx),
342 .cra_type = &crypto_blkcipher_type,
343 .cra_module = THIS_MODULE,
346 .min_keysize = CAMELLIA_MIN_KEY_SIZE,
347 .max_keysize = CAMELLIA_MAX_KEY_SIZE,
348 .setkey = camellia_setkey,
349 .encrypt = ecb_encrypt,
350 .decrypt = ecb_decrypt,
354 .cra_name = "__cbc-camellia-aesni",
355 .cra_driver_name = "__driver-cbc-camellia-aesni",
357 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER |
359 .cra_blocksize = CAMELLIA_BLOCK_SIZE,
360 .cra_ctxsize = sizeof(struct camellia_ctx),
362 .cra_type = &crypto_blkcipher_type,
363 .cra_module = THIS_MODULE,
366 .min_keysize = CAMELLIA_MIN_KEY_SIZE,
367 .max_keysize = CAMELLIA_MAX_KEY_SIZE,
368 .setkey = camellia_setkey,
369 .encrypt = cbc_encrypt,
370 .decrypt = cbc_decrypt,
374 .cra_name = "__ctr-camellia-aesni",
375 .cra_driver_name = "__driver-ctr-camellia-aesni",
377 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER |
380 .cra_ctxsize = sizeof(struct camellia_ctx),
382 .cra_type = &crypto_blkcipher_type,
383 .cra_module = THIS_MODULE,
386 .min_keysize = CAMELLIA_MIN_KEY_SIZE,
387 .max_keysize = CAMELLIA_MAX_KEY_SIZE,
388 .ivsize = CAMELLIA_BLOCK_SIZE,
389 .setkey = camellia_setkey,
390 .encrypt = ctr_crypt,
391 .decrypt = ctr_crypt,
395 .cra_name = "__lrw-camellia-aesni",
396 .cra_driver_name = "__driver-lrw-camellia-aesni",
398 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER |
400 .cra_blocksize = CAMELLIA_BLOCK_SIZE,
401 .cra_ctxsize = sizeof(struct camellia_lrw_ctx),
403 .cra_type = &crypto_blkcipher_type,
404 .cra_module = THIS_MODULE,
405 .cra_exit = lrw_camellia_exit_tfm,
408 .min_keysize = CAMELLIA_MIN_KEY_SIZE +
410 .max_keysize = CAMELLIA_MAX_KEY_SIZE +
412 .ivsize = CAMELLIA_BLOCK_SIZE,
413 .setkey = lrw_camellia_setkey,
414 .encrypt = lrw_encrypt,
415 .decrypt = lrw_decrypt,
419 .cra_name = "__xts-camellia-aesni",
420 .cra_driver_name = "__driver-xts-camellia-aesni",
422 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER |
424 .cra_blocksize = CAMELLIA_BLOCK_SIZE,
425 .cra_ctxsize = sizeof(struct camellia_xts_ctx),
427 .cra_type = &crypto_blkcipher_type,
428 .cra_module = THIS_MODULE,
431 .min_keysize = CAMELLIA_MIN_KEY_SIZE * 2,
432 .max_keysize = CAMELLIA_MAX_KEY_SIZE * 2,
433 .ivsize = CAMELLIA_BLOCK_SIZE,
434 .setkey = xts_camellia_setkey,
435 .encrypt = xts_encrypt,
436 .decrypt = xts_decrypt,
440 .cra_name = "ecb(camellia)",
441 .cra_driver_name = "ecb-camellia-aesni",
443 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
444 .cra_blocksize = CAMELLIA_BLOCK_SIZE,
445 .cra_ctxsize = sizeof(struct async_helper_ctx),
447 .cra_type = &crypto_ablkcipher_type,
448 .cra_module = THIS_MODULE,
449 .cra_init = ablk_init,
450 .cra_exit = ablk_exit,
453 .min_keysize = CAMELLIA_MIN_KEY_SIZE,
454 .max_keysize = CAMELLIA_MAX_KEY_SIZE,
455 .setkey = ablk_set_key,
456 .encrypt = ablk_encrypt,
457 .decrypt = ablk_decrypt,
461 .cra_name = "cbc(camellia)",
462 .cra_driver_name = "cbc-camellia-aesni",
464 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
465 .cra_blocksize = CAMELLIA_BLOCK_SIZE,
466 .cra_ctxsize = sizeof(struct async_helper_ctx),
468 .cra_type = &crypto_ablkcipher_type,
469 .cra_module = THIS_MODULE,
470 .cra_init = ablk_init,
471 .cra_exit = ablk_exit,
474 .min_keysize = CAMELLIA_MIN_KEY_SIZE,
475 .max_keysize = CAMELLIA_MAX_KEY_SIZE,
476 .ivsize = CAMELLIA_BLOCK_SIZE,
477 .setkey = ablk_set_key,
478 .encrypt = __ablk_encrypt,
479 .decrypt = ablk_decrypt,
483 .cra_name = "ctr(camellia)",
484 .cra_driver_name = "ctr-camellia-aesni",
486 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
488 .cra_ctxsize = sizeof(struct async_helper_ctx),
490 .cra_type = &crypto_ablkcipher_type,
491 .cra_module = THIS_MODULE,
492 .cra_init = ablk_init,
493 .cra_exit = ablk_exit,
496 .min_keysize = CAMELLIA_MIN_KEY_SIZE,
497 .max_keysize = CAMELLIA_MAX_KEY_SIZE,
498 .ivsize = CAMELLIA_BLOCK_SIZE,
499 .setkey = ablk_set_key,
500 .encrypt = ablk_encrypt,
501 .decrypt = ablk_encrypt,
506 .cra_name = "lrw(camellia)",
507 .cra_driver_name = "lrw-camellia-aesni",
509 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
510 .cra_blocksize = CAMELLIA_BLOCK_SIZE,
511 .cra_ctxsize = sizeof(struct async_helper_ctx),
513 .cra_type = &crypto_ablkcipher_type,
514 .cra_module = THIS_MODULE,
515 .cra_init = ablk_init,
516 .cra_exit = ablk_exit,
519 .min_keysize = CAMELLIA_MIN_KEY_SIZE +
521 .max_keysize = CAMELLIA_MAX_KEY_SIZE +
523 .ivsize = CAMELLIA_BLOCK_SIZE,
524 .setkey = ablk_set_key,
525 .encrypt = ablk_encrypt,
526 .decrypt = ablk_decrypt,
530 .cra_name = "xts(camellia)",
531 .cra_driver_name = "xts-camellia-aesni",
533 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
534 .cra_blocksize = CAMELLIA_BLOCK_SIZE,
535 .cra_ctxsize = sizeof(struct async_helper_ctx),
537 .cra_type = &crypto_ablkcipher_type,
538 .cra_module = THIS_MODULE,
539 .cra_init = ablk_init,
540 .cra_exit = ablk_exit,
543 .min_keysize = CAMELLIA_MIN_KEY_SIZE * 2,
544 .max_keysize = CAMELLIA_MAX_KEY_SIZE * 2,
545 .ivsize = CAMELLIA_BLOCK_SIZE,
546 .setkey = ablk_set_key,
547 .encrypt = ablk_encrypt,
548 .decrypt = ablk_decrypt,
553 static int __init camellia_aesni_init(void)
555 const char *feature_name;
557 if (!boot_cpu_has(X86_FEATURE_AVX) ||
558 !boot_cpu_has(X86_FEATURE_AES) ||
559 !boot_cpu_has(X86_FEATURE_OSXSAVE)) {
560 pr_info("AVX or AES-NI instructions are not detected.\n");
564 if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
566 pr_info("CPU feature '%s' is not supported.\n", feature_name);
570 return crypto_register_algs(cmll_algs, ARRAY_SIZE(cmll_algs));
573 static void __exit camellia_aesni_fini(void)
575 crypto_unregister_algs(cmll_algs, ARRAY_SIZE(cmll_algs));
578 module_init(camellia_aesni_init);
579 module_exit(camellia_aesni_fini);
581 MODULE_LICENSE("GPL");
582 MODULE_DESCRIPTION("Camellia Cipher Algorithm, AES-NI/AVX optimized");
583 MODULE_ALIAS_CRYPTO("camellia");
584 MODULE_ALIAS_CRYPTO("camellia-asm");