crypto/echainiv.c

   1 /*
   2  * echainiv: Encrypted Chain IV Generator
   3  *
   4  * This generator generates an IV based on a sequence number by xoring it
   5  * with a salt and then encrypting it with the same key as used to encrypt
   6  * the plain text.  This algorithm requires that the block size be equal
   7  * to the IV size.  It is mainly useful for CBC.
   8  *
   9  * This generator can only be used by algorithms where authentication
  10  * is performed after encryption (i.e., authenc).
  11  *
  12  * Copyright (c) 2015 Herbert Xu <herbert@gondor.apana.org.au>
  13  *
  14  * This program is free software; you can redistribute it and/or modify it
  15  * under the terms of the GNU General Public License as published by the Free
  16  * Software Foundation; either version 2 of the License, or (at your option)
  17  * any later version.
  18  *
  19  */
  20
  21 #include <crypto/internal/aead.h>
  22 #include <crypto/null.h>
  23 #include <crypto/rng.h>
  24 #include <crypto/scatterwalk.h>
  25 #include <linux/err.h>
  26 #include <linux/init.h>
  27 #include <linux/kernel.h>
  28 #include <linux/mm.h>
  29 #include <linux/module.h>
  30 #include <linux/percpu.h>
  31 #include <linux/spinlock.h>
  32 #include <linux/string.h>
  33
  34 #define MAX_IV_SIZE 16
  35
  36 struct echainiv_request_ctx {
  37         struct scatterlist src[2];
  38         struct scatterlist dst[2];
  39         struct scatterlist ivbuf[2];
  40         struct scatterlist *ivsg;
  41         struct aead_givcrypt_request subreq;
  42 };
  43
  44 struct echainiv_ctx {
  45         struct crypto_aead *child;
  46         spinlock_t lock;
  47         struct crypto_blkcipher *null;
  48         u8 salt[] __attribute__ ((aligned(__alignof__(u32))));
  49 };
  50
  51 static DEFINE_PER_CPU(u32 [MAX_IV_SIZE / sizeof(u32)], echainiv_iv);
  52
  53 static int echainiv_setkey(struct crypto_aead *tfm,
  54                               const u8 *key, unsigned int keylen)
  55 {
  56         struct echainiv_ctx *ctx = crypto_aead_ctx(tfm);
  57
  58         return crypto_aead_setkey(ctx->child, key, keylen);
  59 }
  60
  61 static int echainiv_setauthsize(struct crypto_aead *tfm,
  62                                   unsigned int authsize)
  63 {
  64         struct echainiv_ctx *ctx = crypto_aead_ctx(tfm);
  65
  66         return crypto_aead_setauthsize(ctx->child, authsize);
  67 }
  68
  69 /* We don't care if we get preempted and read/write IVs from the next CPU. */
  70 static void echainiv_read_iv(u8 *dst, unsigned size)
  71 {
  72         u32 *a = (u32 *)dst;
  73         u32 __percpu *b = echainiv_iv;
  74
  75         for (; size >= 4; size -= 4) {
  76                 *a++ = this_cpu_read(*b);
  77                 b++;
  78         }
  79 }
  80
  81 static void echainiv_write_iv(const u8 *src, unsigned size)
  82 {
  83         const u32 *a = (const u32 *)src;
  84         u32 __percpu *b = echainiv_iv;
  85
  86         for (; size >= 4; size -= 4) {
  87                 this_cpu_write(*b, *a);
  88                 a++;
  89                 b++;
  90         }
  91 }
  92
  93 static void echainiv_encrypt_compat_complete2(struct aead_request *req,
  94                                                  int err)
  95 {
  96         struct echainiv_request_ctx *rctx = aead_request_ctx(req);
  97         struct aead_givcrypt_request *subreq = &rctx->subreq;
  98         struct crypto_aead *geniv;
  99
 100         if (err == -EINPROGRESS)
 101                 return;
 102
 103         if (err)
 104                 goto out;
 105
 106         geniv = crypto_aead_reqtfm(req);
 107         scatterwalk_map_and_copy(subreq->giv, rctx->ivsg, 0,
 108                                  crypto_aead_ivsize(geniv), 1);
 109
 110 out:
 111         kzfree(subreq->giv);
 112 }
 113
 114 static void echainiv_encrypt_compat_complete(
 115         struct crypto_async_request *base, int err)
 116 {
 117         struct aead_request *req = base->data;
 118
 119         echainiv_encrypt_compat_complete2(req, err);
 120         aead_request_complete(req, err);
 121 }
 122
 123 static void echainiv_encrypt_complete2(struct aead_request *req, int err)
 124 {
 125         struct aead_request *subreq = aead_request_ctx(req);
 126         struct crypto_aead *geniv;
 127         unsigned int ivsize;
 128
 129         if (err == -EINPROGRESS)
 130                 return;
 131
 132         if (err)
 133                 goto out;
 134
 135         geniv = crypto_aead_reqtfm(req);
 136         ivsize = crypto_aead_ivsize(geniv);
 137
 138         echainiv_write_iv(subreq->iv, ivsize);
 139
 140         if (req->iv != subreq->iv)
 141                 memcpy(req->iv, subreq->iv, ivsize);
 142
 143 out:
 144         if (req->iv != subreq->iv)
 145                 kzfree(subreq->iv);
 146 }
 147
 148 static void echainiv_encrypt_complete(struct crypto_async_request *base,
 149                                          int err)
 150 {
 151         struct aead_request *req = base->data;
 152
 153         echainiv_encrypt_complete2(req, err);
 154         aead_request_complete(req, err);
 155 }
 156
 157 static int echainiv_encrypt_compat(struct aead_request *req)
 158 {
 159         struct crypto_aead *geniv = crypto_aead_reqtfm(req);
 160         struct echainiv_ctx *ctx = crypto_aead_ctx(geniv);
 161         struct echainiv_request_ctx *rctx = aead_request_ctx(req);
 162         struct aead_givcrypt_request *subreq = &rctx->subreq;
 163         unsigned int ivsize = crypto_aead_ivsize(geniv);
 164         crypto_completion_t compl;
 165         void *data;
 166         u8 *info;
 167         __be64 seq;
 168         int err;
 169
 170         if (req->cryptlen < ivsize)
 171                 return -EINVAL;
 172
 173         compl = req->base.complete;
 174         data = req->base.data;
 175
 176         rctx->ivsg = scatterwalk_ffwd(rctx->ivbuf, req->dst, req->assoclen);
 177         info = PageHighMem(sg_page(rctx->ivsg)) ? NULL : sg_virt(rctx->ivsg);
 178
 179         if (!info) {
 180                 info = kmalloc(ivsize, req->base.flags &
 181                                        CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL:
 182                                                                   GFP_ATOMIC);
 183                 if (!info)
 184                         return -ENOMEM;
 185
 186                 compl = echainiv_encrypt_compat_complete;
 187                 data = req;
 188         }
 189
 190         memcpy(&seq, req->iv + ivsize - sizeof(seq), sizeof(seq));
 191
 192         aead_givcrypt_set_tfm(subreq, ctx->child);
 193         aead_givcrypt_set_callback(subreq, req->base.flags,
 194                                    req->base.complete, req->base.data);
 195         aead_givcrypt_set_crypt(subreq,
 196                                 scatterwalk_ffwd(rctx->src, req->src,
 197                                                  req->assoclen + ivsize),
 198                                 scatterwalk_ffwd(rctx->dst, rctx->ivsg,
 199                                                  ivsize),
 200                                 req->cryptlen - ivsize, req->iv);
 201         aead_givcrypt_set_assoc(subreq, req->src, req->assoclen);
 202         aead_givcrypt_set_giv(subreq, info, be64_to_cpu(seq));
 203
 204         err = crypto_aead_givencrypt(subreq);
 205         if (unlikely(PageHighMem(sg_page(rctx->ivsg))))
 206                 echainiv_encrypt_compat_complete2(req, err);
 207         return err;
 208 }
 209
 210 static int echainiv_encrypt(struct aead_request *req)
 211 {
 212         struct crypto_aead *geniv = crypto_aead_reqtfm(req);
 213         struct echainiv_ctx *ctx = crypto_aead_ctx(geniv);
 214         struct aead_request *subreq = aead_request_ctx(req);
 215         crypto_completion_t compl;
 216         void *data;
 217         u8 *info;
 218         unsigned int ivsize = crypto_aead_ivsize(geniv);
 219         int err;
 220
 221         if (req->cryptlen < ivsize)
 222                 return -EINVAL;
 223
 224         aead_request_set_tfm(subreq, ctx->child);
 225
 226         compl = echainiv_encrypt_complete;
 227         data = req;
 228         info = req->iv;
 229
 230         if (req->src != req->dst) {
 231                 struct scatterlist src[2];
 232                 struct scatterlist dst[2];
 233                 struct blkcipher_desc desc = {
 234                         .tfm = ctx->null,
 235                 };
 236
 237                 err = crypto_blkcipher_encrypt(
 238                         &desc,
 239                         scatterwalk_ffwd(dst, req->dst,
 240                                          req->assoclen + ivsize),
 241                         scatterwalk_ffwd(src, req->src,
 242                                          req->assoclen + ivsize),
 243                         req->cryptlen - ivsize);
 244                 if (err)
 245                         return err;
 246         }
 247
 248         if (unlikely(!IS_ALIGNED((unsigned long)info,
 249                                  crypto_aead_alignmask(geniv) + 1))) {
 250                 info = kmalloc(ivsize, req->base.flags &
 251                                        CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL:
 252                                                                   GFP_ATOMIC);
 253                 if (!info)
 254                         return -ENOMEM;
 255
 256                 memcpy(info, req->iv, ivsize);
 257         }
 258
 259         aead_request_set_callback(subreq, req->base.flags, compl, data);
 260         aead_request_set_crypt(subreq, req->dst, req->dst,
 261                                req->cryptlen - ivsize, info);
 262         aead_request_set_ad(subreq, req->assoclen + ivsize, 0);
 263
 264         crypto_xor(info, ctx->salt, ivsize);
 265         scatterwalk_map_and_copy(info, req->dst, req->assoclen, ivsize, 1);
 266         echainiv_read_iv(info, ivsize);
 267
 268         err = crypto_aead_encrypt(subreq);
 269         echainiv_encrypt_complete2(req, err);
 270         return err;
 271 }
 272
 273 static int echainiv_decrypt_compat(struct aead_request *req)
 274 {
 275         struct crypto_aead *geniv = crypto_aead_reqtfm(req);
 276         struct echainiv_ctx *ctx = crypto_aead_ctx(geniv);
 277         struct echainiv_request_ctx *rctx = aead_request_ctx(req);
 278         struct aead_request *subreq = &rctx->subreq.areq;
 279         crypto_completion_t compl;
 280         void *data;
 281         unsigned int ivsize = crypto_aead_ivsize(geniv);
 282
 283         if (req->cryptlen < ivsize + crypto_aead_authsize(geniv))
 284                 return -EINVAL;
 285
 286         aead_request_set_tfm(subreq, ctx->child);
 287
 288         compl = req->base.complete;
 289         data = req->base.data;
 290
 291         aead_request_set_callback(subreq, req->base.flags, compl, data);
 292         aead_request_set_crypt(subreq,
 293                                scatterwalk_ffwd(rctx->src, req->src,
 294                                                 req->assoclen + ivsize),
 295                                scatterwalk_ffwd(rctx->dst, req->dst,
 296                                                 req->assoclen + ivsize),
 297                                req->cryptlen - ivsize, req->iv);
 298         aead_request_set_assoc(subreq, req->src, req->assoclen);
 299
 300         scatterwalk_map_and_copy(req->iv, req->src, req->assoclen, ivsize, 0);
 301
 302         return crypto_aead_decrypt(subreq);
 303 }
 304
 305 static int echainiv_decrypt(struct aead_request *req)
 306 {
 307         struct crypto_aead *geniv = crypto_aead_reqtfm(req);
 308         struct echainiv_ctx *ctx = crypto_aead_ctx(geniv);
 309         struct aead_request *subreq = aead_request_ctx(req);
 310         crypto_completion_t compl;
 311         void *data;
 312         unsigned int ivsize = crypto_aead_ivsize(geniv);
 313
 314         if (req->cryptlen < ivsize + crypto_aead_authsize(geniv))
 315                 return -EINVAL;
 316
 317         aead_request_set_tfm(subreq, ctx->child);
 318
 319         compl = req->base.complete;
 320         data = req->base.data;
 321
 322         aead_request_set_callback(subreq, req->base.flags, compl, data);
 323         aead_request_set_crypt(subreq, req->src, req->dst,
 324                                req->cryptlen - ivsize, req->iv);
 325         aead_request_set_ad(subreq, req->assoclen + ivsize, 0);
 326
 327         scatterwalk_map_and_copy(req->iv, req->src, req->assoclen, ivsize, 0);
 328         if (req->src != req->dst)
 329                 scatterwalk_map_and_copy(req->iv, req->dst,
 330                                          req->assoclen, ivsize, 1);
 331
 332         return crypto_aead_decrypt(subreq);
 333 }
 334
 335 static int echainiv_encrypt_compat_first(struct aead_request *req)
 336 {
 337         struct crypto_aead *geniv = crypto_aead_reqtfm(req);
 338         struct echainiv_ctx *ctx = crypto_aead_ctx(geniv);
 339         int err = 0;
 340
 341         spin_lock_bh(&ctx->lock);
 342         if (geniv->encrypt != echainiv_encrypt_compat_first)
 343                 goto unlock;
 344
 345         geniv->encrypt = echainiv_encrypt_compat;
 346         err = crypto_rng_get_bytes(crypto_default_rng, ctx->salt,
 347                                    crypto_aead_ivsize(geniv));
 348
 349 unlock:
 350         spin_unlock_bh(&ctx->lock);
 351
 352         if (err)
 353                 return err;
 354
 355         return echainiv_encrypt_compat(req);
 356 }
 357
 358 static int echainiv_encrypt_first(struct aead_request *req)
 359 {
 360         struct crypto_aead *geniv = crypto_aead_reqtfm(req);
 361         struct echainiv_ctx *ctx = crypto_aead_ctx(geniv);
 362         int err = 0;
 363
 364         spin_lock_bh(&ctx->lock);
 365         if (geniv->encrypt != echainiv_encrypt_first)
 366                 goto unlock;
 367
 368         geniv->encrypt = echainiv_encrypt;
 369         err = crypto_rng_get_bytes(crypto_default_rng, ctx->salt,
 370                                    crypto_aead_ivsize(geniv));
 371
 372 unlock:
 373         spin_unlock_bh(&ctx->lock);
 374
 375         if (err)
 376                 return err;
 377
 378         return echainiv_encrypt(req);
 379 }
 380
 381 static int echainiv_compat_init(struct crypto_tfm *tfm)
 382 {
 383         struct crypto_aead *geniv = __crypto_aead_cast(tfm);
 384         struct echainiv_ctx *ctx = crypto_aead_ctx(geniv);
 385         int err;
 386
 387         spin_lock_init(&ctx->lock);
 388
 389         crypto_aead_set_reqsize(geniv, sizeof(struct echainiv_request_ctx));
 390
 391         err = aead_geniv_init(tfm);
 392
 393         ctx->child = geniv->child;
 394         geniv->child = geniv;
 395
 396         return err;
 397 }
 398
 399 static int echainiv_init(struct crypto_tfm *tfm)
 400 {
 401         struct crypto_aead *geniv = __crypto_aead_cast(tfm);
 402         struct echainiv_ctx *ctx = crypto_aead_ctx(geniv);
 403         int err;
 404
 405         spin_lock_init(&ctx->lock);
 406
 407         crypto_aead_set_reqsize(geniv, sizeof(struct aead_request));
 408
 409         ctx->null = crypto_get_default_null_skcipher();
 410         err = PTR_ERR(ctx->null);
 411         if (IS_ERR(ctx->null))
 412                 goto out;
 413
 414         err = aead_geniv_init(tfm);
 415         if (err)
 416                 goto drop_null;
 417
 418         ctx->child = geniv->child;
 419         geniv->child = geniv;
 420
 421 out:
 422         return err;
 423
 424 drop_null:
 425         crypto_put_default_null_skcipher();
 426         goto out;
 427 }
 428
 429 static void echainiv_compat_exit(struct crypto_tfm *tfm)
 430 {
 431         struct echainiv_ctx *ctx = crypto_tfm_ctx(tfm);
 432
 433         crypto_free_aead(ctx->child);
 434 }
 435
 436 static void echainiv_exit(struct crypto_tfm *tfm)
 437 {
 438         struct echainiv_ctx *ctx = crypto_tfm_ctx(tfm);
 439
 440         crypto_free_aead(ctx->child);
 441         crypto_put_default_null_skcipher();
 442 }
 443
 444 static int echainiv_aead_create(struct crypto_template *tmpl,
 445                                 struct rtattr **tb)
 446 {
 447         struct aead_instance *inst;
 448         struct crypto_aead_spawn *spawn;
 449         struct aead_alg *alg;
 450         int err;
 451
 452         inst = aead_geniv_alloc(tmpl, tb, 0, 0);
 453
 454         if (IS_ERR(inst))
 455                 return PTR_ERR(inst);
 456
 457         err = -EINVAL;
 458         if (inst->alg.ivsize < sizeof(u64) ||
 459             inst->alg.ivsize & (sizeof(u32) - 1) ||
 460             inst->alg.ivsize > MAX_IV_SIZE)
 461                 goto free_inst;
 462
 463         spawn = aead_instance_ctx(inst);
 464         alg = crypto_spawn_aead_alg(spawn);
 465
 466         inst->alg.setkey = echainiv_setkey;
 467         inst->alg.setauthsize = echainiv_setauthsize;
 468         inst->alg.encrypt = echainiv_encrypt_first;
 469         inst->alg.decrypt = echainiv_decrypt;
 470
 471         inst->alg.base.cra_init = echainiv_init;
 472         inst->alg.base.cra_exit = echainiv_exit;
 473
 474         inst->alg.base.cra_alignmask |= __alignof__(u32) - 1;
 475         inst->alg.base.cra_ctxsize = sizeof(struct echainiv_ctx);
 476         inst->alg.base.cra_ctxsize += inst->alg.base.cra_aead.ivsize;
 477
 478         if (alg->base.cra_aead.encrypt) {
 479                 inst->alg.encrypt = echainiv_encrypt_compat_first;
 480                 inst->alg.decrypt = echainiv_decrypt_compat;
 481
 482                 inst->alg.base.cra_init = echainiv_compat_init;
 483                 inst->alg.base.cra_exit = echainiv_compat_exit;
 484         }
 485
 486         err = aead_register_instance(tmpl, inst);
 487         if (err)
 488                 goto free_inst;
 489
 490 out:
 491         return err;
 492
 493 free_inst:
 494         aead_geniv_free(inst);
 495         goto out;
 496 }
 497
 498 static int echainiv_create(struct crypto_template *tmpl, struct rtattr **tb)
 499 {
 500         int err;
 501
 502         err = crypto_get_default_rng();
 503         if (err)
 504                 goto out;
 505
 506         err = echainiv_aead_create(tmpl, tb);
 507         if (err)
 508                 goto put_rng;
 509
 510 out:
 511         return err;
 512
 513 put_rng:
 514         crypto_put_default_rng();
 515         goto out;
 516 }
 517
 518 static void echainiv_free(struct crypto_instance *inst)
 519 {
 520         aead_geniv_free(aead_instance(inst));
 521         crypto_put_default_rng();
 522 }
 523
 524 static struct crypto_template echainiv_tmpl = {
 525         .name = "echainiv",
 526         .create = echainiv_create,
 527         .free = echainiv_free,
 528         .module = THIS_MODULE,
 529 };
 530
 531 static int __init echainiv_module_init(void)
 532 {
 533         return crypto_register_template(&echainiv_tmpl);
 534 }
 535
 536 static void __exit echainiv_module_exit(void)
 537 {
 538         crypto_unregister_template(&echainiv_tmpl);
 539 }
 540
 541 module_init(echainiv_module_init);
 542 module_exit(echainiv_module_exit);
 543
 544 MODULE_LICENSE("GPL");
 545 MODULE_DESCRIPTION("Encrypted Chain IV Generator");
 546 MODULE_ALIAS_CRYPTO("echainiv");