arch/x86/crypto/chacha20_glue.c

   1 /*
   2  * ChaCha20 256-bit cipher algorithm, RFC7539, SIMD glue code
   3  *
   4  * Copyright (C) 2015 Martin Willi
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation; either version 2 of the License, or
   9  * (at your option) any later version.
  10  */
  11
  12 #include <crypto/algapi.h>
  13 #include <crypto/chacha20.h>
  14 #include <crypto/internal/skcipher.h>
  15 #include <linux/kernel.h>
  16 #include <linux/module.h>
  17 #include <asm/fpu/api.h>
  18 #include <asm/simd.h>
  19
  20 #define CHACHA20_STATE_ALIGN 16
  21
  22 asmlinkage void chacha20_block_xor_ssse3(u32 *state, u8 *dst, const u8 *src);
  23 asmlinkage void chacha20_4block_xor_ssse3(u32 *state, u8 *dst, const u8 *src);
  24 #ifdef CONFIG_AS_AVX2
  25 asmlinkage void chacha20_8block_xor_avx2(u32 *state, u8 *dst, const u8 *src);
  26 static bool chacha20_use_avx2;
  27 #endif
  28
  29 static void chacha20_dosimd(u32 *state, u8 *dst, const u8 *src,
  30                             unsigned int bytes)
  31 {
  32         u8 buf[CHACHA20_BLOCK_SIZE];
  33
  34 #ifdef CONFIG_AS_AVX2
  35         if (chacha20_use_avx2) {
  36                 while (bytes >= CHACHA20_BLOCK_SIZE * 8) {
  37                         chacha20_8block_xor_avx2(state, dst, src);
  38                         bytes -= CHACHA20_BLOCK_SIZE * 8;
  39                         src += CHACHA20_BLOCK_SIZE * 8;
  40                         dst += CHACHA20_BLOCK_SIZE * 8;
  41                         state[12] += 8;
  42                 }
  43         }
  44 #endif
  45         while (bytes >= CHACHA20_BLOCK_SIZE * 4) {
  46                 chacha20_4block_xor_ssse3(state, dst, src);
  47                 bytes -= CHACHA20_BLOCK_SIZE * 4;
  48                 src += CHACHA20_BLOCK_SIZE * 4;
  49                 dst += CHACHA20_BLOCK_SIZE * 4;
  50                 state[12] += 4;
  51         }
  52         while (bytes >= CHACHA20_BLOCK_SIZE) {
  53                 chacha20_block_xor_ssse3(state, dst, src);
  54                 bytes -= CHACHA20_BLOCK_SIZE;
  55                 src += CHACHA20_BLOCK_SIZE;
  56                 dst += CHACHA20_BLOCK_SIZE;
  57                 state[12]++;
  58         }
  59         if (bytes) {
  60                 memcpy(buf, src, bytes);
  61                 chacha20_block_xor_ssse3(state, buf, buf);
  62                 memcpy(dst, buf, bytes);
  63         }
  64 }
  65
  66 static int chacha20_simd(struct skcipher_request *req)
  67 {
  68         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
  69         struct chacha20_ctx *ctx = crypto_skcipher_ctx(tfm);
  70         u32 state[16] __aligned(CHACHA20_STATE_ALIGN);
  71         struct skcipher_walk walk;
  72         int err;
  73
  74         if (req->cryptlen <= CHACHA20_BLOCK_SIZE || !may_use_simd())
  75                 return crypto_chacha20_crypt(req);
  76
  77         err = skcipher_walk_virt(&walk, req, true);
  78
  79         crypto_chacha20_init(state, ctx, walk.iv);
  80
  81         kernel_fpu_begin();
  82
  83         while (walk.nbytes >= CHACHA20_BLOCK_SIZE) {
  84                 chacha20_dosimd(state, walk.dst.virt.addr, walk.src.virt.addr,
  85                                 rounddown(walk.nbytes, CHACHA20_BLOCK_SIZE));
  86                 err = skcipher_walk_done(&walk,
  87                                          walk.nbytes % CHACHA20_BLOCK_SIZE);
  88         }
  89
  90         if (walk.nbytes) {
  91                 chacha20_dosimd(state, walk.dst.virt.addr, walk.src.virt.addr,
  92                                 walk.nbytes);
  93                 err = skcipher_walk_done(&walk, 0);
  94         }
  95
  96         kernel_fpu_end();
  97
  98         return err;
  99 }
 100
 101 static struct skcipher_alg alg = {
 102         .base.cra_name          = "chacha20",
 103         .base.cra_driver_name   = "chacha20-simd",
 104         .base.cra_priority      = 300,
 105         .base.cra_blocksize     = 1,
 106         .base.cra_ctxsize       = sizeof(struct chacha20_ctx),
 107         .base.cra_alignmask     = sizeof(u32) - 1,
 108         .base.cra_module        = THIS_MODULE,
 109
 110         .min_keysize            = CHACHA20_KEY_SIZE,
 111         .max_keysize            = CHACHA20_KEY_SIZE,
 112         .ivsize                 = CHACHA20_IV_SIZE,
 113         .chunksize              = CHACHA20_BLOCK_SIZE,
 114         .setkey                 = crypto_chacha20_setkey,
 115         .encrypt                = chacha20_simd,
 116         .decrypt                = chacha20_simd,
 117 };
 118
 119 static int __init chacha20_simd_mod_init(void)
 120 {
 121         if (!boot_cpu_has(X86_FEATURE_SSSE3))
 122                 return -ENODEV;
 123
 124 #ifdef CONFIG_AS_AVX2
 125         chacha20_use_avx2 = boot_cpu_has(X86_FEATURE_AVX) &&
 126                             boot_cpu_has(X86_FEATURE_AVX2) &&
 127                             cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL);
 128 #endif
 129         return crypto_register_skcipher(&alg);
 130 }
 131
 132 static void __exit chacha20_simd_mod_fini(void)
 133 {
 134         crypto_unregister_skcipher(&alg);
 135 }
 136
 137 module_init(chacha20_simd_mod_init);
 138 module_exit(chacha20_simd_mod_fini);
 139
 140 MODULE_LICENSE("GPL");
 141 MODULE_AUTHOR("Martin Willi <martin@strongswan.org>");
 142 MODULE_DESCRIPTION("chacha20 cipher algorithm, SIMD accelerated");
 143 MODULE_ALIAS_CRYPTO("chacha20");
 144 MODULE_ALIAS_CRYPTO("chacha20-simd");