drivers/crypto/caam/caamalg.c

   1 /*
   2  * caam - Freescale FSL CAAM support for crypto API
   3  *
   4  * Copyright 2008-2011 Freescale Semiconductor, Inc.
   5  * Copyright 2016 NXP
   6  *
   7  * Based on talitos crypto API driver.
   8  *
   9  * relationship of job descriptors to shared descriptors (SteveC Dec 10 2008):
  10  *
  11  * ---------------                     ---------------
  12  * | JobDesc #1  |-------------------->|  ShareDesc  |
  13  * | *(packet 1) |                     |   (PDB)     |
  14  * ---------------      |------------->|  (hashKey)  |
  15  *       .              |              | (cipherKey) |
  16  *       .              |    |-------->| (operation) |
  17  * ---------------      |    |         ---------------
  18  * | JobDesc #2  |------|    |
  19  * | *(packet 2) |           |
  20  * ---------------           |
  21  *       .                   |
  22  *       .                   |
  23  * ---------------           |
  24  * | JobDesc #3  |------------
  25  * | *(packet 3) |
  26  * ---------------
  27  *
  28  * The SharedDesc never changes for a connection unless rekeyed, but
  29  * each packet will likely be in a different place. So all we need
  30  * to know to process the packet is where the input is, where the
  31  * output goes, and what context we want to process with. Context is
  32  * in the SharedDesc, packet references in the JobDesc.
  33  *
  34  * So, a job desc looks like:
  35  *
  36  * ---------------------
  37  * | Header            |
  38  * | ShareDesc Pointer |
  39  * | SEQ_OUT_PTR       |
  40  * | (output buffer)   |
  41  * | (output length)   |
  42  * | SEQ_IN_PTR        |
  43  * | (input buffer)    |
  44  * | (input length)    |
  45  * ---------------------
  46  */
  47
  48 #include "compat.h"
  49
  50 #include "regs.h"
  51 #include "intern.h"
  52 #include "desc_constr.h"
  53 #include "jr.h"
  54 #include "error.h"
  55 #include "sg_sw_sec4.h"
  56 #include "key_gen.h"
  57 #include "caamalg_desc.h"
  58
  59 /*
  60  * crypto alg
  61  */
  62 #define CAAM_CRA_PRIORITY               3000
  63 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */
  64 #define CAAM_MAX_KEY_SIZE               (AES_MAX_KEY_SIZE + \
  65                                          CTR_RFC3686_NONCE_SIZE + \
  66                                          SHA512_DIGEST_SIZE * 2)
  67
  68 #define AEAD_DESC_JOB_IO_LEN            (DESC_JOB_IO_LEN + CAAM_CMD_SZ * 2)
  69 #define GCM_DESC_JOB_IO_LEN             (AEAD_DESC_JOB_IO_LEN + \
  70                                          CAAM_CMD_SZ * 4)
  71 #define AUTHENC_DESC_JOB_IO_LEN         (AEAD_DESC_JOB_IO_LEN + \
  72                                          CAAM_CMD_SZ * 5)
  73
  74 #define DESC_MAX_USED_BYTES             (CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN)
  75 #define DESC_MAX_USED_LEN               (DESC_MAX_USED_BYTES / CAAM_CMD_SZ)
  76
  77 #ifdef DEBUG
  78 /* for print_hex_dumps with line references */
  79 #define debug(format, arg...) printk(format, arg)
  80 #else
  81 #define debug(format, arg...)
  82 #endif
  83
  84 #ifdef DEBUG
  85 #include <linux/highmem.h>
  86
  87 static void dbg_dump_sg(const char *level, const char *prefix_str,
  88                         int prefix_type, int rowsize, int groupsize,
  89                         struct scatterlist *sg, size_t tlen, bool ascii)
  90 {
  91         struct scatterlist *it;
  92         void *it_page;
  93         size_t len;
  94         void *buf;
  95
  96         for (it = sg; it != NULL && tlen > 0 ; it = sg_next(sg)) {
  97                 /*
  98                  * make sure the scatterlist's page
  99                  * has a valid virtual memory mapping
 100                  */
 101                 it_page = kmap_atomic(sg_page(it));
 102                 if (unlikely(!it_page)) {
 103                         printk(KERN_ERR "dbg_dump_sg: kmap failed\n");
 104                         return;
 105                 }
 106
 107                 buf = it_page + it->offset;
 108                 len = min_t(size_t, tlen, it->length);
 109                 print_hex_dump(level, prefix_str, prefix_type, rowsize,
 110                                groupsize, buf, len, ascii);
 111                 tlen -= len;
 112
 113                 kunmap_atomic(it_page);
 114         }
 115 }
 116 #endif
 117
 118 static struct list_head alg_list;
 119
 120 struct caam_alg_entry {
 121         int class1_alg_type;
 122         int class2_alg_type;
 123         bool rfc3686;
 124         bool geniv;
 125 };
 126
 127 struct caam_aead_alg {
 128         struct aead_alg aead;
 129         struct caam_alg_entry caam;
 130         bool registered;
 131 };
 132
 133 /*
 134  * per-session context
 135  */
 136 struct caam_ctx {
 137         struct device *jrdev;
 138         u32 sh_desc_enc[DESC_MAX_USED_LEN];
 139         u32 sh_desc_dec[DESC_MAX_USED_LEN];
 140         u32 sh_desc_givenc[DESC_MAX_USED_LEN];
 141         dma_addr_t sh_desc_enc_dma;
 142         dma_addr_t sh_desc_dec_dma;
 143         dma_addr_t sh_desc_givenc_dma;
 144         u8 key[CAAM_MAX_KEY_SIZE];
 145         dma_addr_t key_dma;
 146         struct alginfo adata;
 147         struct alginfo cdata;
 148         unsigned int authsize;
 149 };
 150
 151 static int aead_null_set_sh_desc(struct crypto_aead *aead)
 152 {
 153         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 154         struct device *jrdev = ctx->jrdev;
 155         u32 *desc;
 156         int rem_bytes = CAAM_DESC_BYTES_MAX - AEAD_DESC_JOB_IO_LEN -
 157                         ctx->adata.keylen_pad;
 158
 159         /*
 160          * Job Descriptor and Shared Descriptors
 161          * must all fit into the 64-word Descriptor h/w Buffer
 162          */
 163         if (rem_bytes >= DESC_AEAD_NULL_ENC_LEN) {
 164                 ctx->adata.key_inline = true;
 165                 ctx->adata.key_virt = ctx->key;
 166         } else {
 167                 ctx->adata.key_inline = false;
 168                 ctx->adata.key_dma = ctx->key_dma;
 169         }
 170
 171         /* aead_encrypt shared descriptor */
 172         desc = ctx->sh_desc_enc;
 173         cnstr_shdsc_aead_null_encap(desc, &ctx->adata, ctx->authsize);
 174         ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
 175                                               desc_bytes(desc),
 176                                               DMA_TO_DEVICE);
 177         if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
 178                 dev_err(jrdev, "unable to map shared descriptor\n");
 179                 return -ENOMEM;
 180         }
 181
 182         /*
 183          * Job Descriptor and Shared Descriptors
 184          * must all fit into the 64-word Descriptor h/w Buffer
 185          */
 186         if (rem_bytes >= DESC_AEAD_NULL_DEC_LEN) {
 187                 ctx->adata.key_inline = true;
 188                 ctx->adata.key_virt = ctx->key;
 189         } else {
 190                 ctx->adata.key_inline = false;
 191                 ctx->adata.key_dma = ctx->key_dma;
 192         }
 193
 194         /* aead_decrypt shared descriptor */
 195         desc = ctx->sh_desc_dec;
 196         cnstr_shdsc_aead_null_decap(desc, &ctx->adata, ctx->authsize);
 197         ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
 198                                               desc_bytes(desc),
 199                                               DMA_TO_DEVICE);
 200         if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
 201                 dev_err(jrdev, "unable to map shared descriptor\n");
 202                 return -ENOMEM;
 203         }
 204
 205         return 0;
 206 }
 207
 208 static int aead_set_sh_desc(struct crypto_aead *aead)
 209 {
 210         struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
 211                                                  struct caam_aead_alg, aead);
 212         unsigned int ivsize = crypto_aead_ivsize(aead);
 213         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 214         struct device *jrdev = ctx->jrdev;
 215         u32 ctx1_iv_off = 0;
 216         u32 *desc, *nonce = NULL;
 217         u32 inl_mask;
 218         unsigned int data_len[2];
 219         const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) ==
 220                                OP_ALG_AAI_CTR_MOD128);
 221         const bool is_rfc3686 = alg->caam.rfc3686;
 222
 223         if (!ctx->authsize)
 224                 return 0;
 225
 226         /* NULL encryption / decryption */
 227         if (!ctx->cdata.keylen)
 228                 return aead_null_set_sh_desc(aead);
 229
 230         /*
 231          * AES-CTR needs to load IV in CONTEXT1 reg
 232          * at an offset of 128bits (16bytes)
 233          * CONTEXT1[255:128] = IV
 234          */
 235         if (ctr_mode)
 236                 ctx1_iv_off = 16;
 237
 238         /*
 239          * RFC3686 specific:
 240          *      CONTEXT1[255:128] = {NONCE, IV, COUNTER}
 241          */
 242         if (is_rfc3686) {
 243                 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
 244                 nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad +
 245                                 ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE);
 246         }
 247
 248         data_len[0] = ctx->adata.keylen_pad;
 249         data_len[1] = ctx->cdata.keylen;
 250
 251         if (alg->caam.geniv)
 252                 goto skip_enc;
 253
 254         /*
 255          * Job Descriptor and Shared Descriptors
 256          * must all fit into the 64-word Descriptor h/w Buffer
 257          */
 258         if (desc_inline_query(DESC_AEAD_ENC_LEN +
 259                               (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
 260                               AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask,
 261                               ARRAY_SIZE(data_len)) < 0)
 262                 return -EINVAL;
 263
 264         if (inl_mask & 1)
 265                 ctx->adata.key_virt = ctx->key;
 266         else
 267                 ctx->adata.key_dma = ctx->key_dma;
 268
 269         if (inl_mask & 2)
 270                 ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
 271         else
 272                 ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
 273
 274         ctx->adata.key_inline = !!(inl_mask & 1);
 275         ctx->cdata.key_inline = !!(inl_mask & 2);
 276
 277         /* aead_encrypt shared descriptor */
 278         desc = ctx->sh_desc_enc;
 279         cnstr_shdsc_aead_encap(desc, &ctx->cdata, &ctx->adata, ctx->authsize,
 280                                is_rfc3686, nonce, ctx1_iv_off);
 281         ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
 282                                               desc_bytes(desc),
 283                                               DMA_TO_DEVICE);
 284         if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
 285                 dev_err(jrdev, "unable to map shared descriptor\n");
 286                 return -ENOMEM;
 287         }
 288
 289 skip_enc:
 290         /*
 291          * Job Descriptor and Shared Descriptors
 292          * must all fit into the 64-word Descriptor h/w Buffer
 293          */
 294         if (desc_inline_query(DESC_AEAD_DEC_LEN +
 295                               (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
 296                               AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask,
 297                               ARRAY_SIZE(data_len)) < 0)
 298                 return -EINVAL;
 299
 300         if (inl_mask & 1)
 301                 ctx->adata.key_virt = ctx->key;
 302         else
 303                 ctx->adata.key_dma = ctx->key_dma;
 304
 305         if (inl_mask & 2)
 306                 ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
 307         else
 308                 ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
 309
 310         ctx->adata.key_inline = !!(inl_mask & 1);
 311         ctx->cdata.key_inline = !!(inl_mask & 2);
 312
 313         /* aead_decrypt shared descriptor */
 314         desc = ctx->sh_desc_dec;
 315         cnstr_shdsc_aead_decap(desc, &ctx->cdata, &ctx->adata, ivsize,
 316                                ctx->authsize, alg->caam.geniv, is_rfc3686,
 317                                nonce, ctx1_iv_off);
 318         ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
 319                                               desc_bytes(desc),
 320                                               DMA_TO_DEVICE);
 321         if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
 322                 dev_err(jrdev, "unable to map shared descriptor\n");
 323                 return -ENOMEM;
 324         }
 325
 326         if (!alg->caam.geniv)
 327                 goto skip_givenc;
 328
 329         /*
 330          * Job Descriptor and Shared Descriptors
 331          * must all fit into the 64-word Descriptor h/w Buffer
 332          */
 333         if (desc_inline_query(DESC_AEAD_GIVENC_LEN +
 334                               (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
 335                               AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask,
 336                               ARRAY_SIZE(data_len)) < 0)
 337                 return -EINVAL;
 338
 339         if (inl_mask & 1)
 340                 ctx->adata.key_virt = ctx->key;
 341         else
 342                 ctx->adata.key_dma = ctx->key_dma;
 343
 344         if (inl_mask & 2)
 345                 ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
 346         else
 347                 ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
 348
 349         ctx->adata.key_inline = !!(inl_mask & 1);
 350         ctx->cdata.key_inline = !!(inl_mask & 2);
 351
 352         /* aead_givencrypt shared descriptor */
 353         desc = ctx->sh_desc_enc;
 354         cnstr_shdsc_aead_givencap(desc, &ctx->cdata, &ctx->adata, ivsize,
 355                                   ctx->authsize, is_rfc3686, nonce,
 356                                   ctx1_iv_off);
 357         ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
 358                                               desc_bytes(desc),
 359                                               DMA_TO_DEVICE);
 360         if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
 361                 dev_err(jrdev, "unable to map shared descriptor\n");
 362                 return -ENOMEM;
 363         }
 364
 365 skip_givenc:
 366         return 0;
 367 }
 368
 369 static int aead_setauthsize(struct crypto_aead *authenc,
 370                                     unsigned int authsize)
 371 {
 372         struct caam_ctx *ctx = crypto_aead_ctx(authenc);
 373
 374         ctx->authsize = authsize;
 375         aead_set_sh_desc(authenc);
 376
 377         return 0;
 378 }
 379
 380 static int gcm_set_sh_desc(struct crypto_aead *aead)
 381 {
 382         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 383         struct device *jrdev = ctx->jrdev;
 384         u32 *desc;
 385         int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN -
 386                         ctx->cdata.keylen;
 387
 388         if (!ctx->cdata.keylen || !ctx->authsize)
 389                 return 0;
 390
 391         /*
 392          * AES GCM encrypt shared descriptor
 393          * Job Descriptor and Shared Descriptor
 394          * must fit into the 64-word Descriptor h/w Buffer
 395          */
 396         if (rem_bytes >= DESC_GCM_ENC_LEN) {
 397                 ctx->cdata.key_inline = true;
 398                 ctx->cdata.key_virt = ctx->key;
 399         } else {
 400                 ctx->cdata.key_inline = false;
 401                 ctx->cdata.key_dma = ctx->key_dma;
 402         }
 403
 404         desc = ctx->sh_desc_enc;
 405         cnstr_shdsc_gcm_encap(desc, &ctx->cdata, ctx->authsize);
 406         ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
 407                                               desc_bytes(desc),
 408                                               DMA_TO_DEVICE);
 409         if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
 410                 dev_err(jrdev, "unable to map shared descriptor\n");
 411                 return -ENOMEM;
 412         }
 413
 414         /*
 415          * Job Descriptor and Shared Descriptors
 416          * must all fit into the 64-word Descriptor h/w Buffer
 417          */
 418         if (rem_bytes >= DESC_GCM_DEC_LEN) {
 419                 ctx->cdata.key_inline = true;
 420                 ctx->cdata.key_virt = ctx->key;
 421         } else {
 422                 ctx->cdata.key_inline = false;
 423                 ctx->cdata.key_dma = ctx->key_dma;
 424         }
 425
 426         desc = ctx->sh_desc_dec;
 427         cnstr_shdsc_gcm_decap(desc, &ctx->cdata, ctx->authsize);
 428         ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
 429                                               desc_bytes(desc),
 430                                               DMA_TO_DEVICE);
 431         if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
 432                 dev_err(jrdev, "unable to map shared descriptor\n");
 433                 return -ENOMEM;
 434         }
 435
 436         return 0;
 437 }
 438
 439 static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
 440 {
 441         struct caam_ctx *ctx = crypto_aead_ctx(authenc);
 442
 443         ctx->authsize = authsize;
 444         gcm_set_sh_desc(authenc);
 445
 446         return 0;
 447 }
 448
 449 static int rfc4106_set_sh_desc(struct crypto_aead *aead)
 450 {
 451         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 452         struct device *jrdev = ctx->jrdev;
 453         u32 *desc;
 454         int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN -
 455                         ctx->cdata.keylen;
 456
 457         if (!ctx->cdata.keylen || !ctx->authsize)
 458                 return 0;
 459
 460         /*
 461          * RFC4106 encrypt shared descriptor
 462          * Job Descriptor and Shared Descriptor
 463          * must fit into the 64-word Descriptor h/w Buffer
 464          */
 465         if (rem_bytes >= DESC_RFC4106_ENC_LEN) {
 466                 ctx->cdata.key_inline = true;
 467                 ctx->cdata.key_virt = ctx->key;
 468         } else {
 469                 ctx->cdata.key_inline = false;
 470                 ctx->cdata.key_dma = ctx->key_dma;
 471         }
 472
 473         desc = ctx->sh_desc_enc;
 474         cnstr_shdsc_rfc4106_encap(desc, &ctx->cdata, ctx->authsize);
 475         ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
 476                                               desc_bytes(desc),
 477                                               DMA_TO_DEVICE);
 478         if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
 479                 dev_err(jrdev, "unable to map shared descriptor\n");
 480                 return -ENOMEM;
 481         }
 482
 483         /*
 484          * Job Descriptor and Shared Descriptors
 485          * must all fit into the 64-word Descriptor h/w Buffer
 486          */
 487         if (rem_bytes >= DESC_RFC4106_DEC_LEN) {
 488                 ctx->cdata.key_inline = true;
 489                 ctx->cdata.key_virt = ctx->key;
 490         } else {
 491                 ctx->cdata.key_inline = false;
 492                 ctx->cdata.key_dma = ctx->key_dma;
 493         }
 494
 495         desc = ctx->sh_desc_dec;
 496         cnstr_shdsc_rfc4106_decap(desc, &ctx->cdata, ctx->authsize);
 497         ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
 498                                               desc_bytes(desc),
 499                                               DMA_TO_DEVICE);
 500         if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
 501                 dev_err(jrdev, "unable to map shared descriptor\n");
 502                 return -ENOMEM;
 503         }
 504
 505         return 0;
 506 }
 507
 508 static int rfc4106_setauthsize(struct crypto_aead *authenc,
 509                                unsigned int authsize)
 510 {
 511         struct caam_ctx *ctx = crypto_aead_ctx(authenc);
 512
 513         ctx->authsize = authsize;
 514         rfc4106_set_sh_desc(authenc);
 515
 516         return 0;
 517 }
 518
 519 static int rfc4543_set_sh_desc(struct crypto_aead *aead)
 520 {
 521         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 522         struct device *jrdev = ctx->jrdev;
 523         u32 *desc;
 524         int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN -
 525                         ctx->cdata.keylen;
 526
 527         if (!ctx->cdata.keylen || !ctx->authsize)
 528                 return 0;
 529
 530         /*
 531          * RFC4543 encrypt shared descriptor
 532          * Job Descriptor and Shared Descriptor
 533          * must fit into the 64-word Descriptor h/w Buffer
 534          */
 535         if (rem_bytes >= DESC_RFC4543_ENC_LEN) {
 536                 ctx->cdata.key_inline = true;
 537                 ctx->cdata.key_virt = ctx->key;
 538         } else {
 539                 ctx->cdata.key_inline = false;
 540                 ctx->cdata.key_dma = ctx->key_dma;
 541         }
 542
 543         desc = ctx->sh_desc_enc;
 544         cnstr_shdsc_rfc4543_encap(desc, &ctx->cdata, ctx->authsize);
 545         ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
 546                                               desc_bytes(desc),
 547                                               DMA_TO_DEVICE);
 548         if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
 549                 dev_err(jrdev, "unable to map shared descriptor\n");
 550                 return -ENOMEM;
 551         }
 552
 553         /*
 554          * Job Descriptor and Shared Descriptors
 555          * must all fit into the 64-word Descriptor h/w Buffer
 556          */
 557         if (rem_bytes >= DESC_RFC4543_DEC_LEN) {
 558                 ctx->cdata.key_inline = true;
 559                 ctx->cdata.key_virt = ctx->key;
 560         } else {
 561                 ctx->cdata.key_inline = false;
 562                 ctx->cdata.key_dma = ctx->key_dma;
 563         }
 564
 565         desc = ctx->sh_desc_dec;
 566         cnstr_shdsc_rfc4543_decap(desc, &ctx->cdata, ctx->authsize);
 567         ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
 568                                               desc_bytes(desc),
 569                                               DMA_TO_DEVICE);
 570         if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
 571                 dev_err(jrdev, "unable to map shared descriptor\n");
 572                 return -ENOMEM;
 573         }
 574
 575         return 0;
 576 }
 577
 578 static int rfc4543_setauthsize(struct crypto_aead *authenc,
 579                                unsigned int authsize)
 580 {
 581         struct caam_ctx *ctx = crypto_aead_ctx(authenc);
 582
 583         ctx->authsize = authsize;
 584         rfc4543_set_sh_desc(authenc);
 585
 586         return 0;
 587 }
 588
 589 static int aead_setkey(struct crypto_aead *aead,
 590                                const u8 *key, unsigned int keylen)
 591 {
 592         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 593         struct device *jrdev = ctx->jrdev;
 594         struct crypto_authenc_keys keys;
 595         int ret = 0;
 596
 597         if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
 598                 goto badkey;
 599
 600 #ifdef DEBUG
 601         printk(KERN_ERR "keylen %d enckeylen %d authkeylen %d\n",
 602                keys.authkeylen + keys.enckeylen, keys.enckeylen,
 603                keys.authkeylen);
 604         print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
 605                        DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 606 #endif
 607
 608         ret = gen_split_key(ctx->jrdev, ctx->key, &ctx->adata, keys.authkey,
 609                             keys.authkeylen, CAAM_MAX_KEY_SIZE -
 610                             keys.enckeylen);
 611         if (ret) {
 612                 goto badkey;
 613         }
 614
 615         /* postpend encryption key to auth split key */
 616         memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen);
 617
 618         ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->adata.keylen_pad +
 619                                       keys.enckeylen, DMA_TO_DEVICE);
 620         if (dma_mapping_error(jrdev, ctx->key_dma)) {
 621                 dev_err(jrdev, "unable to map key i/o memory\n");
 622                 return -ENOMEM;
 623         }
 624 #ifdef DEBUG
 625         print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ",
 626                        DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
 627                        ctx->adata.keylen_pad + keys.enckeylen, 1);
 628 #endif
 629
 630         ctx->cdata.keylen = keys.enckeylen;
 631
 632         ret = aead_set_sh_desc(aead);
 633         if (ret) {
 634                 dma_unmap_single(jrdev, ctx->key_dma, ctx->adata.keylen_pad +
 635                                  keys.enckeylen, DMA_TO_DEVICE);
 636         }
 637
 638         return ret;
 639 badkey:
 640         crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
 641         return -EINVAL;
 642 }
 643
 644 static int gcm_setkey(struct crypto_aead *aead,
 645                       const u8 *key, unsigned int keylen)
 646 {
 647         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 648         struct device *jrdev = ctx->jrdev;
 649         int ret = 0;
 650
 651 #ifdef DEBUG
 652         print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
 653                        DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 654 #endif
 655
 656         memcpy(ctx->key, key, keylen);
 657         ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen,
 658                                       DMA_TO_DEVICE);
 659         if (dma_mapping_error(jrdev, ctx->key_dma)) {
 660                 dev_err(jrdev, "unable to map key i/o memory\n");
 661                 return -ENOMEM;
 662         }
 663         ctx->cdata.keylen = keylen;
 664
 665         ret = gcm_set_sh_desc(aead);
 666         if (ret) {
 667                 dma_unmap_single(jrdev, ctx->key_dma, ctx->cdata.keylen,
 668                                  DMA_TO_DEVICE);
 669         }
 670
 671         return ret;
 672 }
 673
 674 static int rfc4106_setkey(struct crypto_aead *aead,
 675                           const u8 *key, unsigned int keylen)
 676 {
 677         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 678         struct device *jrdev = ctx->jrdev;
 679         int ret = 0;
 680
 681         if (keylen < 4)
 682                 return -EINVAL;
 683
 684 #ifdef DEBUG
 685         print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
 686                        DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 687 #endif
 688
 689         memcpy(ctx->key, key, keylen);
 690
 691         /*
 692          * The last four bytes of the key material are used as the salt value
 693          * in the nonce. Update the AES key length.
 694          */
 695         ctx->cdata.keylen = keylen - 4;
 696
 697         ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->cdata.keylen,
 698                                       DMA_TO_DEVICE);
 699         if (dma_mapping_error(jrdev, ctx->key_dma)) {
 700                 dev_err(jrdev, "unable to map key i/o memory\n");
 701                 return -ENOMEM;
 702         }
 703
 704         ret = rfc4106_set_sh_desc(aead);
 705         if (ret) {
 706                 dma_unmap_single(jrdev, ctx->key_dma, ctx->cdata.keylen,
 707                                  DMA_TO_DEVICE);
 708         }
 709
 710         return ret;
 711 }
 712
 713 static int rfc4543_setkey(struct crypto_aead *aead,
 714                           const u8 *key, unsigned int keylen)
 715 {
 716         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 717         struct device *jrdev = ctx->jrdev;
 718         int ret = 0;
 719
 720         if (keylen < 4)
 721                 return -EINVAL;
 722
 723 #ifdef DEBUG
 724         print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
 725                        DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 726 #endif
 727
 728         memcpy(ctx->key, key, keylen);
 729
 730         /*
 731          * The last four bytes of the key material are used as the salt value
 732          * in the nonce. Update the AES key length.
 733          */
 734         ctx->cdata.keylen = keylen - 4;
 735
 736         ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->cdata.keylen,
 737                                       DMA_TO_DEVICE);
 738         if (dma_mapping_error(jrdev, ctx->key_dma)) {
 739                 dev_err(jrdev, "unable to map key i/o memory\n");
 740                 return -ENOMEM;
 741         }
 742
 743         ret = rfc4543_set_sh_desc(aead);
 744         if (ret) {
 745                 dma_unmap_single(jrdev, ctx->key_dma, ctx->cdata.keylen,
 746                                  DMA_TO_DEVICE);
 747         }
 748
 749         return ret;
 750 }
 751
 752 static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
 753                              const u8 *key, unsigned int keylen)
 754 {
 755         struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
 756         struct crypto_tfm *tfm = crypto_ablkcipher_tfm(ablkcipher);
 757         const char *alg_name = crypto_tfm_alg_name(tfm);
 758         struct device *jrdev = ctx->jrdev;
 759         unsigned int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
 760         u32 *desc;
 761         u32 ctx1_iv_off = 0;
 762         const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) ==
 763                                OP_ALG_AAI_CTR_MOD128);
 764         const bool is_rfc3686 = (ctr_mode &&
 765                                  (strstr(alg_name, "rfc3686") != NULL));
 766
 767         memcpy(ctx->key, key, keylen);
 768 #ifdef DEBUG
 769         print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
 770                        DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 771 #endif
 772         /*
 773          * AES-CTR needs to load IV in CONTEXT1 reg
 774          * at an offset of 128bits (16bytes)
 775          * CONTEXT1[255:128] = IV
 776          */
 777         if (ctr_mode)
 778                 ctx1_iv_off = 16;
 779
 780         /*
 781          * RFC3686 specific:
 782          *      | CONTEXT1[255:128] = {NONCE, IV, COUNTER}
 783          *      | *key = {KEY, NONCE}
 784          */
 785         if (is_rfc3686) {
 786                 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
 787                 keylen -= CTR_RFC3686_NONCE_SIZE;
 788         }
 789
 790         ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen,
 791                                       DMA_TO_DEVICE);
 792         if (dma_mapping_error(jrdev, ctx->key_dma)) {
 793                 dev_err(jrdev, "unable to map key i/o memory\n");
 794                 return -ENOMEM;
 795         }
 796         ctx->cdata.keylen = keylen;
 797         ctx->cdata.key_virt = ctx->key;
 798         ctx->cdata.key_inline = true;
 799
 800         /* ablkcipher_encrypt shared descriptor */
 801         desc = ctx->sh_desc_enc;
 802         cnstr_shdsc_ablkcipher_encap(desc, &ctx->cdata, ivsize, is_rfc3686,
 803                                      ctx1_iv_off);
 804         ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
 805                                               desc_bytes(desc),
 806                                               DMA_TO_DEVICE);
 807         if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
 808                 dev_err(jrdev, "unable to map shared descriptor\n");
 809                 return -ENOMEM;
 810         }
 811
 812         /* ablkcipher_decrypt shared descriptor */
 813         desc = ctx->sh_desc_dec;
 814         cnstr_shdsc_ablkcipher_decap(desc, &ctx->cdata, ivsize, is_rfc3686,
 815                                      ctx1_iv_off);
 816         ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
 817                                               desc_bytes(desc),
 818                                               DMA_TO_DEVICE);
 819         if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
 820                 dev_err(jrdev, "unable to map shared descriptor\n");
 821                 return -ENOMEM;
 822         }
 823
 824         /* ablkcipher_givencrypt shared descriptor */
 825         desc = ctx->sh_desc_givenc;
 826         cnstr_shdsc_ablkcipher_givencap(desc, &ctx->cdata, ivsize, is_rfc3686,
 827                                         ctx1_iv_off);
 828         ctx->sh_desc_givenc_dma = dma_map_single(jrdev, desc,
 829                                                  desc_bytes(desc),
 830                                                  DMA_TO_DEVICE);
 831         if (dma_mapping_error(jrdev, ctx->sh_desc_givenc_dma)) {
 832                 dev_err(jrdev, "unable to map shared descriptor\n");
 833                 return -ENOMEM;
 834         }
 835
 836         return 0;
 837 }
 838
 839 static int xts_ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
 840                                  const u8 *key, unsigned int keylen)
 841 {
 842         struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
 843         struct device *jrdev = ctx->jrdev;
 844         u32 *desc;
 845
 846         if (keylen != 2 * AES_MIN_KEY_SIZE  && keylen != 2 * AES_MAX_KEY_SIZE) {
 847                 crypto_ablkcipher_set_flags(ablkcipher,
 848                                             CRYPTO_TFM_RES_BAD_KEY_LEN);
 849                 dev_err(jrdev, "key size mismatch\n");
 850                 return -EINVAL;
 851         }
 852
 853         memcpy(ctx->key, key, keylen);
 854         ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen, DMA_TO_DEVICE);
 855         if (dma_mapping_error(jrdev, ctx->key_dma)) {
 856                 dev_err(jrdev, "unable to map key i/o memory\n");
 857                 return -ENOMEM;
 858         }
 859         ctx->cdata.keylen = keylen;
 860         ctx->cdata.key_virt = ctx->key;
 861         ctx->cdata.key_inline = true;
 862
 863         /* xts_ablkcipher_encrypt shared descriptor */
 864         desc = ctx->sh_desc_enc;
 865         cnstr_shdsc_xts_ablkcipher_encap(desc, &ctx->cdata);
 866         ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, desc_bytes(desc),
 867                                               DMA_TO_DEVICE);
 868         if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
 869                 dev_err(jrdev, "unable to map shared descriptor\n");
 870                 return -ENOMEM;
 871         }
 872
 873         /* xts_ablkcipher_decrypt shared descriptor */
 874         desc = ctx->sh_desc_dec;
 875         cnstr_shdsc_xts_ablkcipher_decap(desc, &ctx->cdata);
 876         ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, desc_bytes(desc),
 877                                               DMA_TO_DEVICE);
 878         if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
 879                 dma_unmap_single(jrdev, ctx->sh_desc_enc_dma,
 880                                  desc_bytes(ctx->sh_desc_enc), DMA_TO_DEVICE);
 881                 dev_err(jrdev, "unable to map shared descriptor\n");
 882                 return -ENOMEM;
 883         }
 884
 885         return 0;
 886 }
 887
 888 /*
 889  * aead_edesc - s/w-extended aead descriptor
 890  * @src_nents: number of segments in input scatterlist
 891  * @dst_nents: number of segments in output scatterlist
 892  * @sec4_sg_bytes: length of dma mapped sec4_sg space
 893  * @sec4_sg_dma: bus physical mapped address of h/w link table
 894  * @sec4_sg: pointer to h/w link table
 895  * @hw_desc: the h/w job descriptor followed by any referenced link tables
 896  */
 897 struct aead_edesc {
 898         int src_nents;
 899         int dst_nents;
 900         int sec4_sg_bytes;
 901         dma_addr_t sec4_sg_dma;
 902         struct sec4_sg_entry *sec4_sg;
 903         u32 hw_desc[];
 904 };
 905
 906 /*
 907  * ablkcipher_edesc - s/w-extended ablkcipher descriptor
 908  * @src_nents: number of segments in input scatterlist
 909  * @dst_nents: number of segments in output scatterlist
 910  * @iv_dma: dma address of iv for checking continuity and link table
 911  * @sec4_sg_bytes: length of dma mapped sec4_sg space
 912  * @sec4_sg_dma: bus physical mapped address of h/w link table
 913  * @sec4_sg: pointer to h/w link table
 914  * @hw_desc: the h/w job descriptor followed by any referenced link tables
 915  */
 916 struct ablkcipher_edesc {
 917         int src_nents;
 918         int dst_nents;
 919         dma_addr_t iv_dma;
 920         int sec4_sg_bytes;
 921         dma_addr_t sec4_sg_dma;
 922         struct sec4_sg_entry *sec4_sg;
 923         u32 hw_desc[0];
 924 };
 925
 926 static void caam_unmap(struct device *dev, struct scatterlist *src,
 927                        struct scatterlist *dst, int src_nents,
 928                        int dst_nents,
 929                        dma_addr_t iv_dma, int ivsize, dma_addr_t sec4_sg_dma,
 930                        int sec4_sg_bytes)
 931 {
 932         if (dst != src) {
 933                 dma_unmap_sg(dev, src, src_nents ? : 1, DMA_TO_DEVICE);
 934                 dma_unmap_sg(dev, dst, dst_nents ? : 1, DMA_FROM_DEVICE);
 935         } else {
 936                 dma_unmap_sg(dev, src, src_nents ? : 1, DMA_BIDIRECTIONAL);
 937         }
 938
 939         if (iv_dma)
 940                 dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE);
 941         if (sec4_sg_bytes)
 942                 dma_unmap_single(dev, sec4_sg_dma, sec4_sg_bytes,
 943                                  DMA_TO_DEVICE);
 944 }
 945
 946 static void aead_unmap(struct device *dev,
 947                        struct aead_edesc *edesc,
 948                        struct aead_request *req)
 949 {
 950         caam_unmap(dev, req->src, req->dst,
 951                    edesc->src_nents, edesc->dst_nents, 0, 0,
 952                    edesc->sec4_sg_dma, edesc->sec4_sg_bytes);
 953 }
 954
 955 static void ablkcipher_unmap(struct device *dev,
 956                              struct ablkcipher_edesc *edesc,
 957                              struct ablkcipher_request *req)
 958 {
 959         struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
 960         int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
 961
 962         caam_unmap(dev, req->src, req->dst,
 963                    edesc->src_nents, edesc->dst_nents,
 964                    edesc->iv_dma, ivsize,
 965                    edesc->sec4_sg_dma, edesc->sec4_sg_bytes);
 966 }
 967
 968 static void aead_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
 969                                    void *context)
 970 {
 971         struct aead_request *req = context;
 972         struct aead_edesc *edesc;
 973
 974 #ifdef DEBUG
 975         dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
 976 #endif
 977
 978         edesc = container_of(desc, struct aead_edesc, hw_desc[0]);
 979
 980         if (err)
 981                 caam_jr_strstatus(jrdev, err);
 982
 983         aead_unmap(jrdev, edesc, req);
 984
 985         kfree(edesc);
 986
 987         aead_request_complete(req, err);
 988 }
 989
 990 static void aead_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
 991                                    void *context)
 992 {
 993         struct aead_request *req = context;
 994         struct aead_edesc *edesc;
 995
 996 #ifdef DEBUG
 997         dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
 998 #endif
 999
1000         edesc = container_of(desc, struct aead_edesc, hw_desc[0]);
1001
1002         if (err)
1003                 caam_jr_strstatus(jrdev, err);
1004
1005         aead_unmap(jrdev, edesc, req);
1006
1007         /*
1008          * verify hw auth check passed else return -EBADMSG
1009          */
1010         if ((err & JRSTA_CCBERR_ERRID_MASK) == JRSTA_CCBERR_ERRID_ICVCHK)
1011                 err = -EBADMSG;
1012
1013         kfree(edesc);
1014
1015         aead_request_complete(req, err);
1016 }
1017
1018 static void ablkcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
1019                                    void *context)
1020 {
1021         struct ablkcipher_request *req = context;
1022         struct ablkcipher_edesc *edesc;
1023 #ifdef DEBUG
1024         struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
1025         int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
1026
1027         dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
1028 #endif
1029
1030         edesc = container_of(desc, struct ablkcipher_edesc, hw_desc[0]);
1031
1032         if (err)
1033                 caam_jr_strstatus(jrdev, err);
1034
1035 #ifdef DEBUG
1036         print_hex_dump(KERN_ERR, "dstiv  @"__stringify(__LINE__)": ",
1037                        DUMP_PREFIX_ADDRESS, 16, 4, req->info,
1038                        edesc->src_nents > 1 ? 100 : ivsize, 1);
1039         dbg_dump_sg(KERN_ERR, "dst    @"__stringify(__LINE__)": ",
1040                     DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
1041                     edesc->dst_nents > 1 ? 100 : req->nbytes, 1);
1042 #endif
1043
1044         ablkcipher_unmap(jrdev, edesc, req);
1045         kfree(edesc);
1046
1047         ablkcipher_request_complete(req, err);
1048 }
1049
1050 static void ablkcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
1051                                     void *context)
1052 {
1053         struct ablkcipher_request *req = context;
1054         struct ablkcipher_edesc *edesc;
1055 #ifdef DEBUG
1056         struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
1057         int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
1058
1059         dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
1060 #endif
1061
1062         edesc = container_of(desc, struct ablkcipher_edesc, hw_desc[0]);
1063         if (err)
1064                 caam_jr_strstatus(jrdev, err);
1065
1066 #ifdef DEBUG
1067         print_hex_dump(KERN_ERR, "dstiv  @"__stringify(__LINE__)": ",
1068                        DUMP_PREFIX_ADDRESS, 16, 4, req->info,
1069                        ivsize, 1);
1070         dbg_dump_sg(KERN_ERR, "dst    @"__stringify(__LINE__)": ",
1071                     DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
1072                     edesc->dst_nents > 1 ? 100 : req->nbytes, 1);
1073 #endif
1074
1075         ablkcipher_unmap(jrdev, edesc, req);
1076         kfree(edesc);
1077
1078         ablkcipher_request_complete(req, err);
1079 }
1080
1081 /*
1082  * Fill in aead job descriptor
1083  */
1084 static void init_aead_job(struct aead_request *req,
1085                           struct aead_edesc *edesc,
1086                           bool all_contig, bool encrypt)
1087 {
1088         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1089         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1090         int authsize = ctx->authsize;
1091         u32 *desc = edesc->hw_desc;
1092         u32 out_options, in_options;
1093         dma_addr_t dst_dma, src_dma;
1094         int len, sec4_sg_index = 0;
1095         dma_addr_t ptr;
1096         u32 *sh_desc;
1097
1098         sh_desc = encrypt ? ctx->sh_desc_enc : ctx->sh_desc_dec;
1099         ptr = encrypt ? ctx->sh_desc_enc_dma : ctx->sh_desc_dec_dma;
1100
1101         len = desc_len(sh_desc);
1102         init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
1103
1104         if (all_contig) {
1105                 src_dma = sg_dma_address(req->src);
1106                 in_options = 0;
1107         } else {
1108                 src_dma = edesc->sec4_sg_dma;
1109                 sec4_sg_index += edesc->src_nents;
1110                 in_options = LDST_SGF;
1111         }
1112
1113         append_seq_in_ptr(desc, src_dma, req->assoclen + req->cryptlen,
1114                           in_options);
1115
1116         dst_dma = src_dma;
1117         out_options = in_options;
1118
1119         if (unlikely(req->src != req->dst)) {
1120                 if (!edesc->dst_nents) {
1121                         dst_dma = sg_dma_address(req->dst);
1122                 } else {
1123                         dst_dma = edesc->sec4_sg_dma +
1124                                   sec4_sg_index *
1125                                   sizeof(struct sec4_sg_entry);
1126                         out_options = LDST_SGF;
1127                 }
1128         }
1129
1130         if (encrypt)
1131                 append_seq_out_ptr(desc, dst_dma,
1132                                    req->assoclen + req->cryptlen + authsize,
1133                                    out_options);
1134         else
1135                 append_seq_out_ptr(desc, dst_dma,
1136                                    req->assoclen + req->cryptlen - authsize,
1137                                    out_options);
1138
1139         /* REG3 = assoclen */
1140         append_math_add_imm_u32(desc, REG3, ZERO, IMM, req->assoclen);
1141 }
1142
1143 static void init_gcm_job(struct aead_request *req,
1144                          struct aead_edesc *edesc,
1145                          bool all_contig, bool encrypt)
1146 {
1147         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1148         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1149         unsigned int ivsize = crypto_aead_ivsize(aead);
1150         u32 *desc = edesc->hw_desc;
1151         bool generic_gcm = (ivsize == 12);
1152         unsigned int last;
1153
1154         init_aead_job(req, edesc, all_contig, encrypt);
1155
1156         /* BUG This should not be specific to generic GCM. */
1157         last = 0;
1158         if (encrypt && generic_gcm && !(req->assoclen + req->cryptlen))
1159                 last = FIFOLD_TYPE_LAST1;
1160
1161         /* Read GCM IV */
1162         append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE |
1163                          FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | 12 | last);
1164         /* Append Salt */
1165         if (!generic_gcm)
1166                 append_data(desc, ctx->key + ctx->cdata.keylen, 4);
1167         /* Append IV */
1168         append_data(desc, req->iv, ivsize);
1169         /* End of blank commands */
1170 }
1171
1172 static void init_authenc_job(struct aead_request *req,
1173                              struct aead_edesc *edesc,
1174                              bool all_contig, bool encrypt)
1175 {
1176         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1177         struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
1178                                                  struct caam_aead_alg, aead);
1179         unsigned int ivsize = crypto_aead_ivsize(aead);
1180         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1181         const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) ==
1182                                OP_ALG_AAI_CTR_MOD128);
1183         const bool is_rfc3686 = alg->caam.rfc3686;
1184         u32 *desc = edesc->hw_desc;
1185         u32 ivoffset = 0;
1186
1187         /*
1188          * AES-CTR needs to load IV in CONTEXT1 reg
1189          * at an offset of 128bits (16bytes)
1190          * CONTEXT1[255:128] = IV
1191          */
1192         if (ctr_mode)
1193                 ivoffset = 16;
1194
1195         /*
1196          * RFC3686 specific:
1197          *      CONTEXT1[255:128] = {NONCE, IV, COUNTER}
1198          */
1199         if (is_rfc3686)
1200                 ivoffset = 16 + CTR_RFC3686_NONCE_SIZE;
1201
1202         init_aead_job(req, edesc, all_contig, encrypt);
1203
1204         if (ivsize && ((is_rfc3686 && encrypt) || !alg->caam.geniv))
1205                 append_load_as_imm(desc, req->iv, ivsize,
1206                                    LDST_CLASS_1_CCB |
1207                                    LDST_SRCDST_BYTE_CONTEXT |
1208                                    (ivoffset << LDST_OFFSET_SHIFT));
1209 }
1210
1211 /*
1212  * Fill in ablkcipher job descriptor
1213  */
1214 static void init_ablkcipher_job(u32 *sh_desc, dma_addr_t ptr,
1215                                 struct ablkcipher_edesc *edesc,
1216                                 struct ablkcipher_request *req,
1217                                 bool iv_contig)
1218 {
1219         struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
1220         int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
1221         u32 *desc = edesc->hw_desc;
1222         u32 out_options = 0, in_options;
1223         dma_addr_t dst_dma, src_dma;
1224         int len, sec4_sg_index = 0;
1225
1226 #ifdef DEBUG
1227         print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ",
1228                        DUMP_PREFIX_ADDRESS, 16, 4, req->info,
1229                        ivsize, 1);
1230         printk(KERN_ERR "asked=%d, nbytes%d\n", (int)edesc->src_nents ? 100 : req->nbytes, req->nbytes);
1231         dbg_dump_sg(KERN_ERR, "src    @"__stringify(__LINE__)": ",
1232                     DUMP_PREFIX_ADDRESS, 16, 4, req->src,
1233                     edesc->src_nents ? 100 : req->nbytes, 1);
1234 #endif
1235
1236         len = desc_len(sh_desc);
1237         init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
1238
1239         if (iv_contig) {
1240                 src_dma = edesc->iv_dma;
1241                 in_options = 0;
1242         } else {
1243                 src_dma = edesc->sec4_sg_dma;
1244                 sec4_sg_index += edesc->src_nents + 1;
1245                 in_options = LDST_SGF;
1246         }
1247         append_seq_in_ptr(desc, src_dma, req->nbytes + ivsize, in_options);
1248
1249         if (likely(req->src == req->dst)) {
1250                 if (!edesc->src_nents && iv_contig) {
1251                         dst_dma = sg_dma_address(req->src);
1252                 } else {
1253                         dst_dma = edesc->sec4_sg_dma +
1254                                 sizeof(struct sec4_sg_entry);
1255                         out_options = LDST_SGF;
1256                 }
1257         } else {
1258                 if (!edesc->dst_nents) {
1259                         dst_dma = sg_dma_address(req->dst);
1260                 } else {
1261                         dst_dma = edesc->sec4_sg_dma +
1262                                 sec4_sg_index * sizeof(struct sec4_sg_entry);
1263                         out_options = LDST_SGF;
1264                 }
1265         }
1266         append_seq_out_ptr(desc, dst_dma, req->nbytes, out_options);
1267 }
1268
1269 /*
1270  * Fill in ablkcipher givencrypt job descriptor
1271  */
1272 static void init_ablkcipher_giv_job(u32 *sh_desc, dma_addr_t ptr,
1273                                     struct ablkcipher_edesc *edesc,
1274                                     struct ablkcipher_request *req,
1275                                     bool iv_contig)
1276 {
1277         struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
1278         int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
1279         u32 *desc = edesc->hw_desc;
1280         u32 out_options, in_options;
1281         dma_addr_t dst_dma, src_dma;
1282         int len, sec4_sg_index = 0;
1283
1284 #ifdef DEBUG
1285         print_hex_dump(KERN_ERR, "presciv@" __stringify(__LINE__) ": ",
1286                        DUMP_PREFIX_ADDRESS, 16, 4, req->info,
1287                        ivsize, 1);
1288         dbg_dump_sg(KERN_ERR, "src    @" __stringify(__LINE__) ": ",
1289                     DUMP_PREFIX_ADDRESS, 16, 4, req->src,
1290                     edesc->src_nents ? 100 : req->nbytes, 1);
1291 #endif
1292
1293         len = desc_len(sh_desc);
1294         init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
1295
1296         if (!edesc->src_nents) {
1297                 src_dma = sg_dma_address(req->src);
1298                 in_options = 0;
1299         } else {
1300                 src_dma = edesc->sec4_sg_dma;
1301                 sec4_sg_index += edesc->src_nents;
1302                 in_options = LDST_SGF;
1303         }
1304         append_seq_in_ptr(desc, src_dma, req->nbytes, in_options);
1305
1306         if (iv_contig) {
1307                 dst_dma = edesc->iv_dma;
1308                 out_options = 0;
1309         } else {
1310                 dst_dma = edesc->sec4_sg_dma +
1311                           sec4_sg_index * sizeof(struct sec4_sg_entry);
1312                 out_options = LDST_SGF;
1313         }
1314         append_seq_out_ptr(desc, dst_dma, req->nbytes + ivsize, out_options);
1315 }
1316
1317 /*
1318  * allocate and map the aead extended descriptor
1319  */
1320 static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
1321                                            int desc_bytes, bool *all_contig_ptr,
1322                                            bool encrypt)
1323 {
1324         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1325         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1326         struct device *jrdev = ctx->jrdev;
1327         gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
1328                        CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
1329         int src_nents, dst_nents = 0;
1330         struct aead_edesc *edesc;
1331         int sgc;
1332         bool all_contig = true;
1333         int sec4_sg_index, sec4_sg_len = 0, sec4_sg_bytes;
1334         unsigned int authsize = ctx->authsize;
1335
1336         if (unlikely(req->dst != req->src)) {
1337                 src_nents = sg_count(req->src, req->assoclen + req->cryptlen);
1338                 dst_nents = sg_count(req->dst,
1339                                      req->assoclen + req->cryptlen +
1340                                         (encrypt ? authsize : (-authsize)));
1341         } else {
1342                 src_nents = sg_count(req->src,
1343                                      req->assoclen + req->cryptlen +
1344                                         (encrypt ? authsize : 0));
1345         }
1346
1347         /* Check if data are contiguous. */
1348         all_contig = !src_nents;
1349         if (!all_contig)
1350                 sec4_sg_len = src_nents;
1351
1352         sec4_sg_len += dst_nents;
1353
1354         sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry);
1355
1356         /* allocate space for base edesc and hw desc commands, link tables */
1357         edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes,
1358                         GFP_DMA | flags);
1359         if (!edesc) {
1360                 dev_err(jrdev, "could not allocate extended descriptor\n");
1361                 return ERR_PTR(-ENOMEM);
1362         }
1363
1364         if (likely(req->src == req->dst)) {
1365                 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1,
1366                                  DMA_BIDIRECTIONAL);
1367                 if (unlikely(!sgc)) {
1368                         dev_err(jrdev, "unable to map source\n");
1369                         kfree(edesc);
1370                         return ERR_PTR(-ENOMEM);
1371                 }
1372         } else {
1373                 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1,
1374                                  DMA_TO_DEVICE);
1375                 if (unlikely(!sgc)) {
1376                         dev_err(jrdev, "unable to map source\n");
1377                         kfree(edesc);
1378                         return ERR_PTR(-ENOMEM);
1379                 }
1380
1381                 sgc = dma_map_sg(jrdev, req->dst, dst_nents ? : 1,
1382                                  DMA_FROM_DEVICE);
1383                 if (unlikely(!sgc)) {
1384                         dev_err(jrdev, "unable to map destination\n");
1385                         dma_unmap_sg(jrdev, req->src, src_nents ? : 1,
1386                                      DMA_TO_DEVICE);
1387                         kfree(edesc);
1388                         return ERR_PTR(-ENOMEM);
1389                 }
1390         }
1391
1392         edesc->src_nents = src_nents;
1393         edesc->dst_nents = dst_nents;
1394         edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) +
1395                          desc_bytes;
1396         *all_contig_ptr = all_contig;
1397
1398         sec4_sg_index = 0;
1399         if (!all_contig) {
1400                 sg_to_sec4_sg_last(req->src, src_nents,
1401                               edesc->sec4_sg + sec4_sg_index, 0);
1402                 sec4_sg_index += src_nents;
1403         }
1404         if (dst_nents) {
1405                 sg_to_sec4_sg_last(req->dst, dst_nents,
1406                                    edesc->sec4_sg + sec4_sg_index, 0);
1407         }
1408
1409         if (!sec4_sg_bytes)
1410                 return edesc;
1411
1412         edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
1413                                             sec4_sg_bytes, DMA_TO_DEVICE);
1414         if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
1415                 dev_err(jrdev, "unable to map S/G table\n");
1416                 aead_unmap(jrdev, edesc, req);
1417                 kfree(edesc);
1418                 return ERR_PTR(-ENOMEM);
1419         }
1420
1421         edesc->sec4_sg_bytes = sec4_sg_bytes;
1422
1423         return edesc;
1424 }
1425
1426 static int gcm_encrypt(struct aead_request *req)
1427 {
1428         struct aead_edesc *edesc;
1429         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1430         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1431         struct device *jrdev = ctx->jrdev;
1432         bool all_contig;
1433         u32 *desc;
1434         int ret = 0;
1435
1436         /* allocate extended descriptor */
1437         edesc = aead_edesc_alloc(req, GCM_DESC_JOB_IO_LEN, &all_contig, true);
1438         if (IS_ERR(edesc))
1439                 return PTR_ERR(edesc);
1440
1441         /* Create and submit job descriptor */
1442         init_gcm_job(req, edesc, all_contig, true);
1443 #ifdef DEBUG
1444         print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
1445                        DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
1446                        desc_bytes(edesc->hw_desc), 1);
1447 #endif
1448
1449         desc = edesc->hw_desc;
1450         ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req);
1451         if (!ret) {
1452                 ret = -EINPROGRESS;
1453         } else {
1454                 aead_unmap(jrdev, edesc, req);
1455                 kfree(edesc);
1456         }
1457
1458         return ret;
1459 }
1460
1461 static int ipsec_gcm_encrypt(struct aead_request *req)
1462 {
1463         if (req->assoclen < 8)
1464                 return -EINVAL;
1465
1466         return gcm_encrypt(req);
1467 }
1468
1469 static int aead_encrypt(struct aead_request *req)
1470 {
1471         struct aead_edesc *edesc;
1472         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1473         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1474         struct device *jrdev = ctx->jrdev;
1475         bool all_contig;
1476         u32 *desc;
1477         int ret = 0;
1478
1479         /* allocate extended descriptor */
1480         edesc = aead_edesc_alloc(req, AUTHENC_DESC_JOB_IO_LEN,
1481                                  &all_contig, true);
1482         if (IS_ERR(edesc))
1483                 return PTR_ERR(edesc);
1484
1485         /* Create and submit job descriptor */
1486         init_authenc_job(req, edesc, all_contig, true);
1487 #ifdef DEBUG
1488         print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
1489                        DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
1490                        desc_bytes(edesc->hw_desc), 1);
1491 #endif
1492
1493         desc = edesc->hw_desc;
1494         ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req);
1495         if (!ret) {
1496                 ret = -EINPROGRESS;
1497         } else {
1498                 aead_unmap(jrdev, edesc, req);
1499                 kfree(edesc);
1500         }
1501
1502         return ret;
1503 }
1504
1505 static int gcm_decrypt(struct aead_request *req)
1506 {
1507         struct aead_edesc *edesc;
1508         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1509         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1510         struct device *jrdev = ctx->jrdev;
1511         bool all_contig;
1512         u32 *desc;
1513         int ret = 0;
1514
1515         /* allocate extended descriptor */
1516         edesc = aead_edesc_alloc(req, GCM_DESC_JOB_IO_LEN, &all_contig, false);
1517         if (IS_ERR(edesc))
1518                 return PTR_ERR(edesc);
1519
1520         /* Create and submit job descriptor*/
1521         init_gcm_job(req, edesc, all_contig, false);
1522 #ifdef DEBUG
1523         print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
1524                        DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
1525                        desc_bytes(edesc->hw_desc), 1);
1526 #endif
1527
1528         desc = edesc->hw_desc;
1529         ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req);
1530         if (!ret) {
1531                 ret = -EINPROGRESS;
1532         } else {
1533                 aead_unmap(jrdev, edesc, req);
1534                 kfree(edesc);
1535         }
1536
1537         return ret;
1538 }
1539
1540 static int ipsec_gcm_decrypt(struct aead_request *req)
1541 {
1542         if (req->assoclen < 8)
1543                 return -EINVAL;
1544
1545         return gcm_decrypt(req);
1546 }
1547
1548 static int aead_decrypt(struct aead_request *req)
1549 {
1550         struct aead_edesc *edesc;
1551         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1552         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1553         struct device *jrdev = ctx->jrdev;
1554         bool all_contig;
1555         u32 *desc;
1556         int ret = 0;
1557
1558 #ifdef DEBUG
1559         dbg_dump_sg(KERN_ERR, "dec src@"__stringify(__LINE__)": ",
1560                     DUMP_PREFIX_ADDRESS, 16, 4, req->src,
1561                     req->assoclen + req->cryptlen, 1);
1562 #endif
1563
1564         /* allocate extended descriptor */
1565         edesc = aead_edesc_alloc(req, AUTHENC_DESC_JOB_IO_LEN,
1566                                  &all_contig, false);
1567         if (IS_ERR(edesc))
1568                 return PTR_ERR(edesc);
1569
1570         /* Create and submit job descriptor*/
1571         init_authenc_job(req, edesc, all_contig, false);
1572 #ifdef DEBUG
1573         print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
1574                        DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
1575                        desc_bytes(edesc->hw_desc), 1);
1576 #endif
1577
1578         desc = edesc->hw_desc;
1579         ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req);
1580         if (!ret) {
1581                 ret = -EINPROGRESS;
1582         } else {
1583                 aead_unmap(jrdev, edesc, req);
1584                 kfree(edesc);
1585         }
1586
1587         return ret;
1588 }
1589
1590 /*
1591  * allocate and map the ablkcipher extended descriptor for ablkcipher
1592  */
1593 static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request
1594                                                        *req, int desc_bytes,
1595                                                        bool *iv_contig_out)
1596 {
1597         struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
1598         struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
1599         struct device *jrdev = ctx->jrdev;
1600         gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
1601                                           CRYPTO_TFM_REQ_MAY_SLEEP)) ?
1602                        GFP_KERNEL : GFP_ATOMIC;
1603         int src_nents, dst_nents = 0, sec4_sg_bytes;
1604         struct ablkcipher_edesc *edesc;
1605         dma_addr_t iv_dma = 0;
1606         bool iv_contig = false;
1607         int sgc;
1608         int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
1609         int sec4_sg_index;
1610
1611         src_nents = sg_count(req->src, req->nbytes);
1612
1613         if (req->dst != req->src)
1614                 dst_nents = sg_count(req->dst, req->nbytes);
1615
1616         if (likely(req->src == req->dst)) {
1617                 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1,
1618                                  DMA_BIDIRECTIONAL);
1619                 if (unlikely(!sgc)) {
1620                         dev_err(jrdev, "unable to map source\n");
1621                         return ERR_PTR(-ENOMEM);
1622                 }
1623         } else {
1624                 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1,
1625                                  DMA_TO_DEVICE);
1626                 if (unlikely(!sgc)) {
1627                         dev_err(jrdev, "unable to map source\n");
1628                         return ERR_PTR(-ENOMEM);
1629                 }
1630
1631                 sgc = dma_map_sg(jrdev, req->dst, dst_nents ? : 1,
1632                                  DMA_FROM_DEVICE);
1633                 if (unlikely(!sgc)) {
1634                         dev_err(jrdev, "unable to map destination\n");
1635                         dma_unmap_sg(jrdev, req->src, src_nents ? : 1,
1636                                      DMA_TO_DEVICE);
1637                         return ERR_PTR(-ENOMEM);
1638                 }
1639         }
1640
1641         iv_dma = dma_map_single(jrdev, req->info, ivsize, DMA_TO_DEVICE);
1642         if (dma_mapping_error(jrdev, iv_dma)) {
1643                 dev_err(jrdev, "unable to map IV\n");
1644                 caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0,
1645                            0, 0, 0);
1646                 return ERR_PTR(-ENOMEM);
1647         }
1648
1649         /*
1650          * Check if iv can be contiguous with source and destination.
1651          * If so, include it. If not, create scatterlist.
1652          */
1653         if (!src_nents && iv_dma + ivsize == sg_dma_address(req->src))
1654                 iv_contig = true;
1655         else
1656                 src_nents = src_nents ? : 1;
1657         sec4_sg_bytes = ((iv_contig ? 0 : 1) + src_nents + dst_nents) *
1658                         sizeof(struct sec4_sg_entry);
1659
1660         /* allocate space for base edesc and hw desc commands, link tables */
1661         edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes,
1662                         GFP_DMA | flags);
1663         if (!edesc) {
1664                 dev_err(jrdev, "could not allocate extended descriptor\n");
1665                 caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents,
1666                            iv_dma, ivsize, 0, 0);
1667                 return ERR_PTR(-ENOMEM);
1668         }
1669
1670         edesc->src_nents = src_nents;
1671         edesc->dst_nents = dst_nents;
1672         edesc->sec4_sg_bytes = sec4_sg_bytes;
1673         edesc->sec4_sg = (void *)edesc + sizeof(struct ablkcipher_edesc) +
1674                          desc_bytes;
1675
1676         sec4_sg_index = 0;
1677         if (!iv_contig) {
1678                 dma_to_sec4_sg_one(edesc->sec4_sg, iv_dma, ivsize, 0);
1679                 sg_to_sec4_sg_last(req->src, src_nents,
1680                                    edesc->sec4_sg + 1, 0);
1681                 sec4_sg_index += 1 + src_nents;
1682         }
1683
1684         if (dst_nents) {
1685                 sg_to_sec4_sg_last(req->dst, dst_nents,
1686                         edesc->sec4_sg + sec4_sg_index, 0);
1687         }
1688
1689         edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
1690                                             sec4_sg_bytes, DMA_TO_DEVICE);
1691         if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
1692                 dev_err(jrdev, "unable to map S/G table\n");
1693                 caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents,
1694                            iv_dma, ivsize, 0, 0);
1695                 kfree(edesc);
1696                 return ERR_PTR(-ENOMEM);
1697         }
1698
1699         edesc->iv_dma = iv_dma;
1700
1701 #ifdef DEBUG
1702         print_hex_dump(KERN_ERR, "ablkcipher sec4_sg@"__stringify(__LINE__)": ",
1703                        DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg,
1704                        sec4_sg_bytes, 1);
1705 #endif
1706
1707         *iv_contig_out = iv_contig;
1708         return edesc;
1709 }
1710
1711 static int ablkcipher_encrypt(struct ablkcipher_request *req)
1712 {
1713         struct ablkcipher_edesc *edesc;
1714         struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
1715         struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
1716         struct device *jrdev = ctx->jrdev;
1717         bool iv_contig;
1718         u32 *desc;
1719         int ret = 0;
1720
1721         /* allocate extended descriptor */
1722         edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN *
1723                                        CAAM_CMD_SZ, &iv_contig);
1724         if (IS_ERR(edesc))
1725                 return PTR_ERR(edesc);
1726
1727         /* Create and submit job descriptor*/
1728         init_ablkcipher_job(ctx->sh_desc_enc,
1729                 ctx->sh_desc_enc_dma, edesc, req, iv_contig);
1730 #ifdef DEBUG
1731         print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"__stringify(__LINE__)": ",
1732                        DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
1733                        desc_bytes(edesc->hw_desc), 1);
1734 #endif
1735         desc = edesc->hw_desc;
1736         ret = caam_jr_enqueue(jrdev, desc, ablkcipher_encrypt_done, req);
1737
1738         if (!ret) {
1739                 ret = -EINPROGRESS;
1740         } else {
1741                 ablkcipher_unmap(jrdev, edesc, req);
1742                 kfree(edesc);
1743         }
1744
1745         return ret;
1746 }
1747
1748 static int ablkcipher_decrypt(struct ablkcipher_request *req)
1749 {
1750         struct ablkcipher_edesc *edesc;
1751         struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
1752         struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
1753         struct device *jrdev = ctx->jrdev;
1754         bool iv_contig;
1755         u32 *desc;
1756         int ret = 0;
1757
1758         /* allocate extended descriptor */
1759         edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN *
1760                                        CAAM_CMD_SZ, &iv_contig);
1761         if (IS_ERR(edesc))
1762                 return PTR_ERR(edesc);
1763
1764         /* Create and submit job descriptor*/
1765         init_ablkcipher_job(ctx->sh_desc_dec,
1766                 ctx->sh_desc_dec_dma, edesc, req, iv_contig);
1767         desc = edesc->hw_desc;
1768 #ifdef DEBUG
1769         print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"__stringify(__LINE__)": ",
1770                        DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
1771                        desc_bytes(edesc->hw_desc), 1);
1772 #endif
1773
1774         ret = caam_jr_enqueue(jrdev, desc, ablkcipher_decrypt_done, req);
1775         if (!ret) {
1776                 ret = -EINPROGRESS;
1777         } else {
1778                 ablkcipher_unmap(jrdev, edesc, req);
1779                 kfree(edesc);
1780         }
1781
1782         return ret;
1783 }
1784
1785 /*
1786  * allocate and map the ablkcipher extended descriptor
1787  * for ablkcipher givencrypt
1788  */
1789 static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc(
1790                                 struct skcipher_givcrypt_request *greq,
1791                                 int desc_bytes,
1792                                 bool *iv_contig_out)
1793 {
1794         struct ablkcipher_request *req = &greq->creq;
1795         struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
1796         struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
1797         struct device *jrdev = ctx->jrdev;
1798         gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
1799                                           CRYPTO_TFM_REQ_MAY_SLEEP)) ?
1800                        GFP_KERNEL : GFP_ATOMIC;
1801         int src_nents, dst_nents = 0, sec4_sg_bytes;
1802         struct ablkcipher_edesc *edesc;
1803         dma_addr_t iv_dma = 0;
1804         bool iv_contig = false;
1805         int sgc;
1806         int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
1807         int sec4_sg_index;
1808
1809         src_nents = sg_count(req->src, req->nbytes);
1810
1811         if (unlikely(req->dst != req->src))
1812                 dst_nents = sg_count(req->dst, req->nbytes);
1813
1814         if (likely(req->src == req->dst)) {
1815                 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1,
1816                                  DMA_BIDIRECTIONAL);
1817                 if (unlikely(!sgc)) {
1818                         dev_err(jrdev, "unable to map source\n");
1819                         return ERR_PTR(-ENOMEM);
1820                 }
1821         } else {
1822                 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1,
1823                                  DMA_TO_DEVICE);
1824                 if (unlikely(!sgc)) {
1825                         dev_err(jrdev, "unable to map source\n");
1826                         return ERR_PTR(-ENOMEM);
1827                 }
1828
1829                 sgc = dma_map_sg(jrdev, req->dst, dst_nents ? : 1,
1830                                  DMA_FROM_DEVICE);
1831                 if (unlikely(!sgc)) {
1832                         dev_err(jrdev, "unable to map destination\n");
1833                         dma_unmap_sg(jrdev, req->src, src_nents ? : 1,
1834                                      DMA_TO_DEVICE);
1835                         return ERR_PTR(-ENOMEM);
1836                 }
1837         }
1838
1839         /*
1840          * Check if iv can be contiguous with source and destination.
1841          * If so, include it. If not, create scatterlist.
1842          */
1843         iv_dma = dma_map_single(jrdev, greq->giv, ivsize, DMA_TO_DEVICE);
1844         if (dma_mapping_error(jrdev, iv_dma)) {
1845                 dev_err(jrdev, "unable to map IV\n");
1846                 caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0,
1847                            0, 0, 0);
1848                 return ERR_PTR(-ENOMEM);
1849         }
1850
1851         if (!dst_nents && iv_dma + ivsize == sg_dma_address(req->dst))
1852                 iv_contig = true;
1853         else
1854                 dst_nents = dst_nents ? : 1;
1855         sec4_sg_bytes = ((iv_contig ? 0 : 1) + src_nents + dst_nents) *
1856                         sizeof(struct sec4_sg_entry);
1857
1858         /* allocate space for base edesc and hw desc commands, link tables */
1859         edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes,
1860                         GFP_DMA | flags);
1861         if (!edesc) {
1862                 dev_err(jrdev, "could not allocate extended descriptor\n");
1863                 caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents,
1864                            iv_dma, ivsize, 0, 0);
1865                 return ERR_PTR(-ENOMEM);
1866         }
1867
1868         edesc->src_nents = src_nents;
1869         edesc->dst_nents = dst_nents;
1870         edesc->sec4_sg_bytes = sec4_sg_bytes;
1871         edesc->sec4_sg = (void *)edesc + sizeof(struct ablkcipher_edesc) +
1872                          desc_bytes;
1873
1874         sec4_sg_index = 0;
1875         if (src_nents) {
1876                 sg_to_sec4_sg_last(req->src, src_nents, edesc->sec4_sg, 0);
1877                 sec4_sg_index += src_nents;
1878         }
1879
1880         if (!iv_contig) {
1881                 dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index,
1882                                    iv_dma, ivsize, 0);
1883                 sec4_sg_index += 1;
1884                 sg_to_sec4_sg_last(req->dst, dst_nents,
1885                                    edesc->sec4_sg + sec4_sg_index, 0);
1886         }
1887
1888         edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
1889                                             sec4_sg_bytes, DMA_TO_DEVICE);
1890         if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
1891                 dev_err(jrdev, "unable to map S/G table\n");
1892                 caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents,
1893                            iv_dma, ivsize, 0, 0);
1894                 kfree(edesc);
1895                 return ERR_PTR(-ENOMEM);
1896         }
1897         edesc->iv_dma = iv_dma;
1898
1899 #ifdef DEBUG
1900         print_hex_dump(KERN_ERR,
1901                        "ablkcipher sec4_sg@" __stringify(__LINE__) ": ",
1902                        DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg,
1903                        sec4_sg_bytes, 1);
1904 #endif
1905
1906         *iv_contig_out = iv_contig;
1907         return edesc;
1908 }
1909
1910 static int ablkcipher_givencrypt(struct skcipher_givcrypt_request *creq)
1911 {
1912         struct ablkcipher_request *req = &creq->creq;
1913         struct ablkcipher_edesc *edesc;
1914         struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
1915         struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
1916         struct device *jrdev = ctx->jrdev;
1917         bool iv_contig;
1918         u32 *desc;
1919         int ret = 0;
1920
1921         /* allocate extended descriptor */
1922         edesc = ablkcipher_giv_edesc_alloc(creq, DESC_JOB_IO_LEN *
1923                                        CAAM_CMD_SZ, &iv_contig);
1924         if (IS_ERR(edesc))
1925                 return PTR_ERR(edesc);
1926
1927         /* Create and submit job descriptor*/
1928         init_ablkcipher_giv_job(ctx->sh_desc_givenc, ctx->sh_desc_givenc_dma,
1929                                 edesc, req, iv_contig);
1930 #ifdef DEBUG
1931         print_hex_dump(KERN_ERR,
1932                        "ablkcipher jobdesc@" __stringify(__LINE__) ": ",
1933                        DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
1934                        desc_bytes(edesc->hw_desc), 1);
1935 #endif
1936         desc = edesc->hw_desc;
1937         ret = caam_jr_enqueue(jrdev, desc, ablkcipher_encrypt_done, req);
1938
1939         if (!ret) {
1940                 ret = -EINPROGRESS;
1941         } else {
1942                 ablkcipher_unmap(jrdev, edesc, req);
1943                 kfree(edesc);
1944         }
1945
1946         return ret;
1947 }
1948
1949 #define template_aead           template_u.aead
1950 #define template_ablkcipher     template_u.ablkcipher
1951 struct caam_alg_template {
1952         char name[CRYPTO_MAX_ALG_NAME];
1953         char driver_name[CRYPTO_MAX_ALG_NAME];
1954         unsigned int blocksize;
1955         u32 type;
1956         union {
1957                 struct ablkcipher_alg ablkcipher;
1958         } template_u;
1959         u32 class1_alg_type;
1960         u32 class2_alg_type;
1961 };
1962
1963 static struct caam_alg_template driver_algs[] = {
1964         /* ablkcipher descriptor */
1965         {
1966                 .name = "cbc(aes)",
1967                 .driver_name = "cbc-aes-caam",
1968                 .blocksize = AES_BLOCK_SIZE,
1969                 .type = CRYPTO_ALG_TYPE_GIVCIPHER,
1970                 .template_ablkcipher = {
1971                         .setkey = ablkcipher_setkey,
1972                         .encrypt = ablkcipher_encrypt,
1973                         .decrypt = ablkcipher_decrypt,
1974                         .givencrypt = ablkcipher_givencrypt,
1975                         .geniv = "<built-in>",
1976                         .min_keysize = AES_MIN_KEY_SIZE,
1977                         .max_keysize = AES_MAX_KEY_SIZE,
1978                         .ivsize = AES_BLOCK_SIZE,
1979                         },
1980                 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1981         },
1982         {
1983                 .name = "cbc(des3_ede)",
1984                 .driver_name = "cbc-3des-caam",
1985                 .blocksize = DES3_EDE_BLOCK_SIZE,
1986                 .type = CRYPTO_ALG_TYPE_GIVCIPHER,
1987                 .template_ablkcipher = {
1988                         .setkey = ablkcipher_setkey,
1989                         .encrypt = ablkcipher_encrypt,
1990                         .decrypt = ablkcipher_decrypt,
1991                         .givencrypt = ablkcipher_givencrypt,
1992                         .geniv = "<built-in>",
1993                         .min_keysize = DES3_EDE_KEY_SIZE,
1994                         .max_keysize = DES3_EDE_KEY_SIZE,
1995                         .ivsize = DES3_EDE_BLOCK_SIZE,
1996                         },
1997                 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1998         },
1999         {
2000                 .name = "cbc(des)",
2001                 .driver_name = "cbc-des-caam",
2002                 .blocksize = DES_BLOCK_SIZE,
2003                 .type = CRYPTO_ALG_TYPE_GIVCIPHER,
2004                 .template_ablkcipher = {
2005                         .setkey = ablkcipher_setkey,
2006                         .encrypt = ablkcipher_encrypt,
2007                         .decrypt = ablkcipher_decrypt,
2008                         .givencrypt = ablkcipher_givencrypt,
2009                         .geniv = "<built-in>",
2010                         .min_keysize = DES_KEY_SIZE,
2011                         .max_keysize = DES_KEY_SIZE,
2012                         .ivsize = DES_BLOCK_SIZE,
2013                         },
2014                 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2015         },
2016         {
2017                 .name = "ctr(aes)",
2018                 .driver_name = "ctr-aes-caam",
2019                 .blocksize = 1,
2020                 .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2021                 .template_ablkcipher = {
2022                         .setkey = ablkcipher_setkey,
2023                         .encrypt = ablkcipher_encrypt,
2024                         .decrypt = ablkcipher_decrypt,
2025                         .geniv = "chainiv",
2026                         .min_keysize = AES_MIN_KEY_SIZE,
2027                         .max_keysize = AES_MAX_KEY_SIZE,
2028                         .ivsize = AES_BLOCK_SIZE,
2029                         },
2030                 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128,
2031         },
2032         {
2033                 .name = "rfc3686(ctr(aes))",
2034                 .driver_name = "rfc3686-ctr-aes-caam",
2035                 .blocksize = 1,
2036                 .type = CRYPTO_ALG_TYPE_GIVCIPHER,
2037                 .template_ablkcipher = {
2038                         .setkey = ablkcipher_setkey,
2039                         .encrypt = ablkcipher_encrypt,
2040                         .decrypt = ablkcipher_decrypt,
2041                         .givencrypt = ablkcipher_givencrypt,
2042                         .geniv = "<built-in>",
2043                         .min_keysize = AES_MIN_KEY_SIZE +
2044                                        CTR_RFC3686_NONCE_SIZE,
2045                         .max_keysize = AES_MAX_KEY_SIZE +
2046                                        CTR_RFC3686_NONCE_SIZE,
2047                         .ivsize = CTR_RFC3686_IV_SIZE,
2048                         },
2049                 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128,
2050         },
2051         {
2052                 .name = "xts(aes)",
2053                 .driver_name = "xts-aes-caam",
2054                 .blocksize = AES_BLOCK_SIZE,
2055                 .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2056                 .template_ablkcipher = {
2057                         .setkey = xts_ablkcipher_setkey,
2058                         .encrypt = ablkcipher_encrypt,
2059                         .decrypt = ablkcipher_decrypt,
2060                         .geniv = "eseqiv",
2061                         .min_keysize = 2 * AES_MIN_KEY_SIZE,
2062                         .max_keysize = 2 * AES_MAX_KEY_SIZE,
2063                         .ivsize = AES_BLOCK_SIZE,
2064                         },
2065                 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS,
2066         },
2067 };
2068
2069 static struct caam_aead_alg driver_aeads[] = {
2070         {
2071                 .aead = {
2072                         .base = {
2073                                 .cra_name = "rfc4106(gcm(aes))",
2074                                 .cra_driver_name = "rfc4106-gcm-aes-caam",
2075                                 .cra_blocksize = 1,
2076                         },
2077                         .setkey = rfc4106_setkey,
2078                         .setauthsize = rfc4106_setauthsize,
2079                         .encrypt = ipsec_gcm_encrypt,
2080                         .decrypt = ipsec_gcm_decrypt,
2081                         .ivsize = 8,
2082                         .maxauthsize = AES_BLOCK_SIZE,
2083                 },
2084                 .caam = {
2085                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
2086                 },
2087         },
2088         {
2089                 .aead = {
2090                         .base = {
2091                                 .cra_name = "rfc4543(gcm(aes))",
2092                                 .cra_driver_name = "rfc4543-gcm-aes-caam",
2093                                 .cra_blocksize = 1,
2094                         },
2095                         .setkey = rfc4543_setkey,
2096                         .setauthsize = rfc4543_setauthsize,
2097                         .encrypt = ipsec_gcm_encrypt,
2098                         .decrypt = ipsec_gcm_decrypt,
2099                         .ivsize = 8,
2100                         .maxauthsize = AES_BLOCK_SIZE,
2101                 },
2102                 .caam = {
2103                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
2104                 },
2105         },
2106         /* Galois Counter Mode */
2107         {
2108                 .aead = {
2109                         .base = {
2110                                 .cra_name = "gcm(aes)",
2111                                 .cra_driver_name = "gcm-aes-caam",
2112                                 .cra_blocksize = 1,
2113                         },
2114                         .setkey = gcm_setkey,
2115                         .setauthsize = gcm_setauthsize,
2116                         .encrypt = gcm_encrypt,
2117                         .decrypt = gcm_decrypt,
2118                         .ivsize = 12,
2119                         .maxauthsize = AES_BLOCK_SIZE,
2120                 },
2121                 .caam = {
2122                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
2123                 },
2124         },
2125         /* single-pass ipsec_esp descriptor */
2126         {
2127                 .aead = {
2128                         .base = {
2129                                 .cra_name = "authenc(hmac(md5),"
2130                                             "ecb(cipher_null))",
2131                                 .cra_driver_name = "authenc-hmac-md5-"
2132                                                    "ecb-cipher_null-caam",
2133                                 .cra_blocksize = NULL_BLOCK_SIZE,
2134                         },
2135                         .setkey = aead_setkey,
2136                         .setauthsize = aead_setauthsize,
2137                         .encrypt = aead_encrypt,
2138                         .decrypt = aead_decrypt,
2139                         .ivsize = NULL_IV_SIZE,
2140                         .maxauthsize = MD5_DIGEST_SIZE,
2141                 },
2142                 .caam = {
2143                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2144                                            OP_ALG_AAI_HMAC_PRECOMP,
2145                 },
2146         },
2147         {
2148                 .aead = {
2149                         .base = {
2150                                 .cra_name = "authenc(hmac(sha1),"
2151                                             "ecb(cipher_null))",
2152                                 .cra_driver_name = "authenc-hmac-sha1-"
2153                                                    "ecb-cipher_null-caam",
2154                                 .cra_blocksize = NULL_BLOCK_SIZE,
2155                         },
2156                         .setkey = aead_setkey,
2157                         .setauthsize = aead_setauthsize,
2158                         .encrypt = aead_encrypt,
2159                         .decrypt = aead_decrypt,
2160                         .ivsize = NULL_IV_SIZE,
2161                         .maxauthsize = SHA1_DIGEST_SIZE,
2162                 },
2163                 .caam = {
2164                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2165                                            OP_ALG_AAI_HMAC_PRECOMP,
2166                 },
2167         },
2168         {
2169                 .aead = {
2170                         .base = {
2171                                 .cra_name = "authenc(hmac(sha224),"
2172                                             "ecb(cipher_null))",
2173                                 .cra_driver_name = "authenc-hmac-sha224-"
2174                                                    "ecb-cipher_null-caam",
2175                                 .cra_blocksize = NULL_BLOCK_SIZE,
2176                         },
2177                         .setkey = aead_setkey,
2178                         .setauthsize = aead_setauthsize,
2179                         .encrypt = aead_encrypt,
2180                         .decrypt = aead_decrypt,
2181                         .ivsize = NULL_IV_SIZE,
2182                         .maxauthsize = SHA224_DIGEST_SIZE,
2183                 },
2184                 .caam = {
2185                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2186                                            OP_ALG_AAI_HMAC_PRECOMP,
2187                 },
2188         },
2189         {
2190                 .aead = {
2191                         .base = {
2192                                 .cra_name = "authenc(hmac(sha256),"
2193                                             "ecb(cipher_null))",
2194                                 .cra_driver_name = "authenc-hmac-sha256-"
2195                                                    "ecb-cipher_null-caam",
2196                                 .cra_blocksize = NULL_BLOCK_SIZE,
2197                         },
2198                         .setkey = aead_setkey,
2199                         .setauthsize = aead_setauthsize,
2200                         .encrypt = aead_encrypt,
2201                         .decrypt = aead_decrypt,
2202                         .ivsize = NULL_IV_SIZE,
2203                         .maxauthsize = SHA256_DIGEST_SIZE,
2204                 },
2205                 .caam = {
2206                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2207                                            OP_ALG_AAI_HMAC_PRECOMP,
2208                 },
2209         },
2210         {
2211                 .aead = {
2212                         .base = {
2213                                 .cra_name = "authenc(hmac(sha384),"
2214                                             "ecb(cipher_null))",
2215                                 .cra_driver_name = "authenc-hmac-sha384-"
2216                                                    "ecb-cipher_null-caam",
2217                                 .cra_blocksize = NULL_BLOCK_SIZE,
2218                         },
2219                         .setkey = aead_setkey,
2220                         .setauthsize = aead_setauthsize,
2221                         .encrypt = aead_encrypt,
2222                         .decrypt = aead_decrypt,
2223                         .ivsize = NULL_IV_SIZE,
2224                         .maxauthsize = SHA384_DIGEST_SIZE,
2225                 },
2226                 .caam = {
2227                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2228                                            OP_ALG_AAI_HMAC_PRECOMP,
2229                 },
2230         },
2231         {
2232                 .aead = {
2233                         .base = {
2234                                 .cra_name = "authenc(hmac(sha512),"
2235                                             "ecb(cipher_null))",
2236                                 .cra_driver_name = "authenc-hmac-sha512-"
2237                                                    "ecb-cipher_null-caam",
2238                                 .cra_blocksize = NULL_BLOCK_SIZE,
2239                         },
2240                         .setkey = aead_setkey,
2241                         .setauthsize = aead_setauthsize,
2242                         .encrypt = aead_encrypt,
2243                         .decrypt = aead_decrypt,
2244                         .ivsize = NULL_IV_SIZE,
2245                         .maxauthsize = SHA512_DIGEST_SIZE,
2246                 },
2247                 .caam = {
2248                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2249                                            OP_ALG_AAI_HMAC_PRECOMP,
2250                 },
2251         },
2252         {
2253                 .aead = {
2254                         .base = {
2255                                 .cra_name = "authenc(hmac(md5),cbc(aes))",
2256                                 .cra_driver_name = "authenc-hmac-md5-"
2257                                                    "cbc-aes-caam",
2258                                 .cra_blocksize = AES_BLOCK_SIZE,
2259                         },
2260                         .setkey = aead_setkey,
2261                         .setauthsize = aead_setauthsize,
2262                         .encrypt = aead_encrypt,
2263                         .decrypt = aead_decrypt,
2264                         .ivsize = AES_BLOCK_SIZE,
2265                         .maxauthsize = MD5_DIGEST_SIZE,
2266                 },
2267                 .caam = {
2268                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2269                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2270                                            OP_ALG_AAI_HMAC_PRECOMP,
2271                 },
2272         },
2273         {
2274                 .aead = {
2275                         .base = {
2276                                 .cra_name = "echainiv(authenc(hmac(md5),"
2277                                             "cbc(aes)))",
2278                                 .cra_driver_name = "echainiv-authenc-hmac-md5-"
2279                                                    "cbc-aes-caam",
2280                                 .cra_blocksize = AES_BLOCK_SIZE,
2281                         },
2282                         .setkey = aead_setkey,
2283                         .setauthsize = aead_setauthsize,
2284                         .encrypt = aead_encrypt,
2285                         .decrypt = aead_decrypt,
2286                         .ivsize = AES_BLOCK_SIZE,
2287                         .maxauthsize = MD5_DIGEST_SIZE,
2288                 },
2289                 .caam = {
2290                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2291                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2292                                            OP_ALG_AAI_HMAC_PRECOMP,
2293                         .geniv = true,
2294                 },
2295         },
2296         {
2297                 .aead = {
2298                         .base = {
2299                                 .cra_name = "authenc(hmac(sha1),cbc(aes))",
2300                                 .cra_driver_name = "authenc-hmac-sha1-"
2301                                                    "cbc-aes-caam",
2302                                 .cra_blocksize = AES_BLOCK_SIZE,
2303                         },
2304                         .setkey = aead_setkey,
2305                         .setauthsize = aead_setauthsize,
2306                         .encrypt = aead_encrypt,
2307                         .decrypt = aead_decrypt,
2308                         .ivsize = AES_BLOCK_SIZE,
2309                         .maxauthsize = SHA1_DIGEST_SIZE,
2310                 },
2311                 .caam = {
2312                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2313                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2314                                            OP_ALG_AAI_HMAC_PRECOMP,
2315                 },
2316         },
2317         {
2318                 .aead = {
2319                         .base = {
2320                                 .cra_name = "echainiv(authenc(hmac(sha1),"
2321                                             "cbc(aes)))",
2322                                 .cra_driver_name = "echainiv-authenc-"
2323                                                    "hmac-sha1-cbc-aes-caam",
2324                                 .cra_blocksize = AES_BLOCK_SIZE,
2325                         },
2326                         .setkey = aead_setkey,
2327                         .setauthsize = aead_setauthsize,
2328                         .encrypt = aead_encrypt,
2329                         .decrypt = aead_decrypt,
2330                         .ivsize = AES_BLOCK_SIZE,
2331                         .maxauthsize = SHA1_DIGEST_SIZE,
2332                 },
2333                 .caam = {
2334                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2335                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2336                                            OP_ALG_AAI_HMAC_PRECOMP,
2337                         .geniv = true,
2338                 },
2339         },
2340         {
2341                 .aead = {
2342                         .base = {
2343                                 .cra_name = "authenc(hmac(sha224),cbc(aes))",
2344                                 .cra_driver_name = "authenc-hmac-sha224-"
2345                                                    "cbc-aes-caam",
2346                                 .cra_blocksize = AES_BLOCK_SIZE,
2347                         },
2348                         .setkey = aead_setkey,
2349                         .setauthsize = aead_setauthsize,
2350                         .encrypt = aead_encrypt,
2351                         .decrypt = aead_decrypt,
2352                         .ivsize = AES_BLOCK_SIZE,
2353                         .maxauthsize = SHA224_DIGEST_SIZE,
2354                 },
2355                 .caam = {
2356                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2357                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2358                                            OP_ALG_AAI_HMAC_PRECOMP,
2359                 },
2360         },
2361         {
2362                 .aead = {
2363                         .base = {
2364                                 .cra_name = "echainiv(authenc(hmac(sha224),"
2365                                             "cbc(aes)))",
2366                                 .cra_driver_name = "echainiv-authenc-"
2367                                                    "hmac-sha224-cbc-aes-caam",
2368                                 .cra_blocksize = AES_BLOCK_SIZE,
2369                         },
2370                         .setkey = aead_setkey,
2371                         .setauthsize = aead_setauthsize,
2372                         .encrypt = aead_encrypt,
2373                         .decrypt = aead_decrypt,
2374                         .ivsize = AES_BLOCK_SIZE,
2375                         .maxauthsize = SHA224_DIGEST_SIZE,
2376                 },
2377                 .caam = {
2378                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2379                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2380                                            OP_ALG_AAI_HMAC_PRECOMP,
2381                         .geniv = true,
2382                 },
2383         },
2384         {
2385                 .aead = {
2386                         .base = {
2387                                 .cra_name = "authenc(hmac(sha256),cbc(aes))",
2388                                 .cra_driver_name = "authenc-hmac-sha256-"
2389                                                    "cbc-aes-caam",
2390                                 .cra_blocksize = AES_BLOCK_SIZE,
2391                         },
2392                         .setkey = aead_setkey,
2393                         .setauthsize = aead_setauthsize,
2394                         .encrypt = aead_encrypt,
2395                         .decrypt = aead_decrypt,
2396                         .ivsize = AES_BLOCK_SIZE,
2397                         .maxauthsize = SHA256_DIGEST_SIZE,
2398                 },
2399                 .caam = {
2400                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2401                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2402                                            OP_ALG_AAI_HMAC_PRECOMP,
2403                 },
2404         },
2405         {
2406                 .aead = {
2407                         .base = {
2408                                 .cra_name = "echainiv(authenc(hmac(sha256),"
2409                                             "cbc(aes)))",
2410                                 .cra_driver_name = "echainiv-authenc-"
2411                                                    "hmac-sha256-cbc-aes-caam",
2412                                 .cra_blocksize = AES_BLOCK_SIZE,
2413                         },
2414                         .setkey = aead_setkey,
2415                         .setauthsize = aead_setauthsize,
2416                         .encrypt = aead_encrypt,
2417                         .decrypt = aead_decrypt,
2418                         .ivsize = AES_BLOCK_SIZE,
2419                         .maxauthsize = SHA256_DIGEST_SIZE,
2420                 },
2421                 .caam = {
2422                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2423                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2424                                            OP_ALG_AAI_HMAC_PRECOMP,
2425                         .geniv = true,
2426                 },
2427         },
2428         {
2429                 .aead = {
2430                         .base = {
2431                                 .cra_name = "authenc(hmac(sha384),cbc(aes))",
2432                                 .cra_driver_name = "authenc-hmac-sha384-"
2433                                                    "cbc-aes-caam",
2434                                 .cra_blocksize = AES_BLOCK_SIZE,
2435                         },
2436                         .setkey = aead_setkey,
2437                         .setauthsize = aead_setauthsize,
2438                         .encrypt = aead_encrypt,
2439                         .decrypt = aead_decrypt,
2440                         .ivsize = AES_BLOCK_SIZE,
2441                         .maxauthsize = SHA384_DIGEST_SIZE,
2442                 },
2443                 .caam = {
2444                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2445                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2446                                            OP_ALG_AAI_HMAC_PRECOMP,
2447                 },
2448         },
2449         {
2450                 .aead = {
2451                         .base = {
2452                                 .cra_name = "echainiv(authenc(hmac(sha384),"
2453                                             "cbc(aes)))",
2454                                 .cra_driver_name = "echainiv-authenc-"
2455                                                    "hmac-sha384-cbc-aes-caam",
2456                                 .cra_blocksize = AES_BLOCK_SIZE,
2457                         },
2458                         .setkey = aead_setkey,
2459                         .setauthsize = aead_setauthsize,
2460                         .encrypt = aead_encrypt,
2461                         .decrypt = aead_decrypt,
2462                         .ivsize = AES_BLOCK_SIZE,
2463                         .maxauthsize = SHA384_DIGEST_SIZE,
2464                 },
2465                 .caam = {
2466                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2467                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2468                                            OP_ALG_AAI_HMAC_PRECOMP,
2469                         .geniv = true,
2470                 },
2471         },
2472         {
2473                 .aead = {
2474                         .base = {
2475                                 .cra_name = "authenc(hmac(sha512),cbc(aes))",
2476                                 .cra_driver_name = "authenc-hmac-sha512-"
2477                                                    "cbc-aes-caam",
2478                                 .cra_blocksize = AES_BLOCK_SIZE,
2479                         },
2480                         .setkey = aead_setkey,
2481                         .setauthsize = aead_setauthsize,
2482                         .encrypt = aead_encrypt,
2483                         .decrypt = aead_decrypt,
2484                         .ivsize = AES_BLOCK_SIZE,
2485                         .maxauthsize = SHA512_DIGEST_SIZE,
2486                 },
2487                 .caam = {
2488                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2489                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2490                                            OP_ALG_AAI_HMAC_PRECOMP,
2491                 },
2492         },
2493         {
2494                 .aead = {
2495                         .base = {
2496                                 .cra_name = "echainiv(authenc(hmac(sha512),"
2497                                             "cbc(aes)))",
2498                                 .cra_driver_name = "echainiv-authenc-"
2499                                                    "hmac-sha512-cbc-aes-caam",
2500                                 .cra_blocksize = AES_BLOCK_SIZE,
2501                         },
2502                         .setkey = aead_setkey,
2503                         .setauthsize = aead_setauthsize,
2504                         .encrypt = aead_encrypt,
2505                         .decrypt = aead_decrypt,
2506                         .ivsize = AES_BLOCK_SIZE,
2507                         .maxauthsize = SHA512_DIGEST_SIZE,
2508                 },
2509                 .caam = {
2510                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2511                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2512                                            OP_ALG_AAI_HMAC_PRECOMP,
2513                         .geniv = true,
2514                 },
2515         },
2516         {
2517                 .aead = {
2518                         .base = {
2519                                 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
2520                                 .cra_driver_name = "authenc-hmac-md5-"
2521                                                    "cbc-des3_ede-caam",
2522                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2523                         },
2524                         .setkey = aead_setkey,
2525                         .setauthsize = aead_setauthsize,
2526                         .encrypt = aead_encrypt,
2527                         .decrypt = aead_decrypt,
2528                         .ivsize = DES3_EDE_BLOCK_SIZE,
2529                         .maxauthsize = MD5_DIGEST_SIZE,
2530                 },
2531                 .caam = {
2532                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2533                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2534                                            OP_ALG_AAI_HMAC_PRECOMP,
2535                 }
2536         },
2537         {
2538                 .aead = {
2539                         .base = {
2540                                 .cra_name = "echainiv(authenc(hmac(md5),"
2541                                             "cbc(des3_ede)))",
2542                                 .cra_driver_name = "echainiv-authenc-hmac-md5-"
2543                                                    "cbc-des3_ede-caam",
2544                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2545                         },
2546                         .setkey = aead_setkey,
2547                         .setauthsize = aead_setauthsize,
2548                         .encrypt = aead_encrypt,
2549                         .decrypt = aead_decrypt,
2550                         .ivsize = DES3_EDE_BLOCK_SIZE,
2551                         .maxauthsize = MD5_DIGEST_SIZE,
2552                 },
2553                 .caam = {
2554                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2555                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2556                                            OP_ALG_AAI_HMAC_PRECOMP,
2557                         .geniv = true,
2558                 }
2559         },
2560         {
2561                 .aead = {
2562                         .base = {
2563                                 .cra_name = "authenc(hmac(sha1),"
2564                                             "cbc(des3_ede))",
2565                                 .cra_driver_name = "authenc-hmac-sha1-"
2566                                                    "cbc-des3_ede-caam",
2567                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2568                         },
2569                         .setkey = aead_setkey,
2570                         .setauthsize = aead_setauthsize,
2571                         .encrypt = aead_encrypt,
2572                         .decrypt = aead_decrypt,
2573                         .ivsize = DES3_EDE_BLOCK_SIZE,
2574                         .maxauthsize = SHA1_DIGEST_SIZE,
2575                 },
2576                 .caam = {
2577                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2578                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2579                                            OP_ALG_AAI_HMAC_PRECOMP,
2580                 },
2581         },
2582         {
2583                 .aead = {
2584                         .base = {
2585                                 .cra_name = "echainiv(authenc(hmac(sha1),"
2586                                             "cbc(des3_ede)))",
2587                                 .cra_driver_name = "echainiv-authenc-"
2588                                                    "hmac-sha1-"
2589                                                    "cbc-des3_ede-caam",
2590                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2591                         },
2592                         .setkey = aead_setkey,
2593                         .setauthsize = aead_setauthsize,
2594                         .encrypt = aead_encrypt,
2595                         .decrypt = aead_decrypt,
2596                         .ivsize = DES3_EDE_BLOCK_SIZE,
2597                         .maxauthsize = SHA1_DIGEST_SIZE,
2598                 },
2599                 .caam = {
2600                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2601                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2602                                            OP_ALG_AAI_HMAC_PRECOMP,
2603                         .geniv = true,
2604                 },
2605         },
2606         {
2607                 .aead = {
2608                         .base = {
2609                                 .cra_name = "authenc(hmac(sha224),"
2610                                             "cbc(des3_ede))",
2611                                 .cra_driver_name = "authenc-hmac-sha224-"
2612                                                    "cbc-des3_ede-caam",
2613                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2614                         },
2615                         .setkey = aead_setkey,
2616                         .setauthsize = aead_setauthsize,
2617                         .encrypt = aead_encrypt,
2618                         .decrypt = aead_decrypt,
2619                         .ivsize = DES3_EDE_BLOCK_SIZE,
2620                         .maxauthsize = SHA224_DIGEST_SIZE,
2621                 },
2622                 .caam = {
2623                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2624                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2625                                            OP_ALG_AAI_HMAC_PRECOMP,
2626                 },
2627         },
2628         {
2629                 .aead = {
2630                         .base = {
2631                                 .cra_name = "echainiv(authenc(hmac(sha224),"
2632                                             "cbc(des3_ede)))",
2633                                 .cra_driver_name = "echainiv-authenc-"
2634                                                    "hmac-sha224-"
2635                                                    "cbc-des3_ede-caam",
2636                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2637                         },
2638                         .setkey = aead_setkey,
2639                         .setauthsize = aead_setauthsize,
2640                         .encrypt = aead_encrypt,
2641                         .decrypt = aead_decrypt,
2642                         .ivsize = DES3_EDE_BLOCK_SIZE,
2643                         .maxauthsize = SHA224_DIGEST_SIZE,
2644                 },
2645                 .caam = {
2646                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2647                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2648                                            OP_ALG_AAI_HMAC_PRECOMP,
2649                         .geniv = true,
2650                 },
2651         },
2652         {
2653                 .aead = {
2654                         .base = {
2655                                 .cra_name = "authenc(hmac(sha256),"
2656                                             "cbc(des3_ede))",
2657                                 .cra_driver_name = "authenc-hmac-sha256-"
2658                                                    "cbc-des3_ede-caam",
2659                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2660                         },
2661                         .setkey = aead_setkey,
2662                         .setauthsize = aead_setauthsize,
2663                         .encrypt = aead_encrypt,
2664                         .decrypt = aead_decrypt,
2665                         .ivsize = DES3_EDE_BLOCK_SIZE,
2666                         .maxauthsize = SHA256_DIGEST_SIZE,
2667                 },
2668                 .caam = {
2669                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2670                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2671                                            OP_ALG_AAI_HMAC_PRECOMP,
2672                 },
2673         },
2674         {
2675                 .aead = {
2676                         .base = {
2677                                 .cra_name = "echainiv(authenc(hmac(sha256),"
2678                                             "cbc(des3_ede)))",
2679                                 .cra_driver_name = "echainiv-authenc-"
2680                                                    "hmac-sha256-"
2681                                                    "cbc-des3_ede-caam",
2682                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2683                         },
2684                         .setkey = aead_setkey,
2685                         .setauthsize = aead_setauthsize,
2686                         .encrypt = aead_encrypt,
2687                         .decrypt = aead_decrypt,
2688                         .ivsize = DES3_EDE_BLOCK_SIZE,
2689                         .maxauthsize = SHA256_DIGEST_SIZE,
2690                 },
2691                 .caam = {
2692                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2693                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2694                                            OP_ALG_AAI_HMAC_PRECOMP,
2695                         .geniv = true,
2696                 },
2697         },
2698         {
2699                 .aead = {
2700                         .base = {
2701                                 .cra_name = "authenc(hmac(sha384),"
2702                                             "cbc(des3_ede))",
2703                                 .cra_driver_name = "authenc-hmac-sha384-"
2704                                                    "cbc-des3_ede-caam",
2705                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2706                         },
2707                         .setkey = aead_setkey,
2708                         .setauthsize = aead_setauthsize,
2709                         .encrypt = aead_encrypt,
2710                         .decrypt = aead_decrypt,
2711                         .ivsize = DES3_EDE_BLOCK_SIZE,
2712                         .maxauthsize = SHA384_DIGEST_SIZE,
2713                 },
2714                 .caam = {
2715                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2716                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2717                                            OP_ALG_AAI_HMAC_PRECOMP,
2718                 },
2719         },
2720         {
2721                 .aead = {
2722                         .base = {
2723                                 .cra_name = "echainiv(authenc(hmac(sha384),"
2724                                             "cbc(des3_ede)))",
2725                                 .cra_driver_name = "echainiv-authenc-"
2726                                                    "hmac-sha384-"
2727                                                    "cbc-des3_ede-caam",
2728                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2729                         },
2730                         .setkey = aead_setkey,
2731                         .setauthsize = aead_setauthsize,
2732                         .encrypt = aead_encrypt,
2733                         .decrypt = aead_decrypt,
2734                         .ivsize = DES3_EDE_BLOCK_SIZE,
2735                         .maxauthsize = SHA384_DIGEST_SIZE,
2736                 },
2737                 .caam = {
2738                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2739                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2740                                            OP_ALG_AAI_HMAC_PRECOMP,
2741                         .geniv = true,
2742                 },
2743         },
2744         {
2745                 .aead = {
2746                         .base = {
2747                                 .cra_name = "authenc(hmac(sha512),"
2748                                             "cbc(des3_ede))",
2749                                 .cra_driver_name = "authenc-hmac-sha512-"
2750                                                    "cbc-des3_ede-caam",
2751                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2752                         },
2753                         .setkey = aead_setkey,
2754                         .setauthsize = aead_setauthsize,
2755                         .encrypt = aead_encrypt,
2756                         .decrypt = aead_decrypt,
2757                         .ivsize = DES3_EDE_BLOCK_SIZE,
2758                         .maxauthsize = SHA512_DIGEST_SIZE,
2759                 },
2760                 .caam = {
2761                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2762                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2763                                            OP_ALG_AAI_HMAC_PRECOMP,
2764                 },
2765         },
2766         {
2767                 .aead = {
2768                         .base = {
2769                                 .cra_name = "echainiv(authenc(hmac(sha512),"
2770                                             "cbc(des3_ede)))",
2771                                 .cra_driver_name = "echainiv-authenc-"
2772                                                    "hmac-sha512-"
2773                                                    "cbc-des3_ede-caam",
2774                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2775                         },
2776                         .setkey = aead_setkey,
2777                         .setauthsize = aead_setauthsize,
2778                         .encrypt = aead_encrypt,
2779                         .decrypt = aead_decrypt,
2780                         .ivsize = DES3_EDE_BLOCK_SIZE,
2781                         .maxauthsize = SHA512_DIGEST_SIZE,
2782                 },
2783                 .caam = {
2784                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2785                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2786                                            OP_ALG_AAI_HMAC_PRECOMP,
2787                         .geniv = true,
2788                 },
2789         },
2790         {
2791                 .aead = {
2792                         .base = {
2793                                 .cra_name = "authenc(hmac(md5),cbc(des))",
2794                                 .cra_driver_name = "authenc-hmac-md5-"
2795                                                    "cbc-des-caam",
2796                                 .cra_blocksize = DES_BLOCK_SIZE,
2797                         },
2798                         .setkey = aead_setkey,
2799                         .setauthsize = aead_setauthsize,
2800                         .encrypt = aead_encrypt,
2801                         .decrypt = aead_decrypt,
2802                         .ivsize = DES_BLOCK_SIZE,
2803                         .maxauthsize = MD5_DIGEST_SIZE,
2804                 },
2805                 .caam = {
2806                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2807                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2808                                            OP_ALG_AAI_HMAC_PRECOMP,
2809                 },
2810         },
2811         {
2812                 .aead = {
2813                         .base = {
2814                                 .cra_name = "echainiv(authenc(hmac(md5),"
2815                                             "cbc(des)))",
2816                                 .cra_driver_name = "echainiv-authenc-hmac-md5-"
2817                                                    "cbc-des-caam",
2818                                 .cra_blocksize = DES_BLOCK_SIZE,
2819                         },
2820                         .setkey = aead_setkey,
2821                         .setauthsize = aead_setauthsize,
2822                         .encrypt = aead_encrypt,
2823                         .decrypt = aead_decrypt,
2824                         .ivsize = DES_BLOCK_SIZE,
2825                         .maxauthsize = MD5_DIGEST_SIZE,
2826                 },
2827                 .caam = {
2828                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2829                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2830                                            OP_ALG_AAI_HMAC_PRECOMP,
2831                         .geniv = true,
2832                 },
2833         },
2834         {
2835                 .aead = {
2836                         .base = {
2837                                 .cra_name = "authenc(hmac(sha1),cbc(des))",
2838                                 .cra_driver_name = "authenc-hmac-sha1-"
2839                                                    "cbc-des-caam",
2840                                 .cra_blocksize = DES_BLOCK_SIZE,
2841                         },
2842                         .setkey = aead_setkey,
2843                         .setauthsize = aead_setauthsize,
2844                         .encrypt = aead_encrypt,
2845                         .decrypt = aead_decrypt,
2846                         .ivsize = DES_BLOCK_SIZE,
2847                         .maxauthsize = SHA1_DIGEST_SIZE,
2848                 },
2849                 .caam = {
2850                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2851                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2852                                            OP_ALG_AAI_HMAC_PRECOMP,
2853                 },
2854         },
2855         {
2856                 .aead = {
2857                         .base = {
2858                                 .cra_name = "echainiv(authenc(hmac(sha1),"
2859                                             "cbc(des)))",
2860                                 .cra_driver_name = "echainiv-authenc-"
2861                                                    "hmac-sha1-cbc-des-caam",
2862                                 .cra_blocksize = DES_BLOCK_SIZE,
2863                         },
2864                         .setkey = aead_setkey,
2865                         .setauthsize = aead_setauthsize,
2866                         .encrypt = aead_encrypt,
2867                         .decrypt = aead_decrypt,
2868                         .ivsize = DES_BLOCK_SIZE,
2869                         .maxauthsize = SHA1_DIGEST_SIZE,
2870                 },
2871                 .caam = {
2872                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2873                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2874                                            OP_ALG_AAI_HMAC_PRECOMP,
2875                         .geniv = true,
2876                 },
2877         },
2878         {
2879                 .aead = {
2880                         .base = {
2881                                 .cra_name = "authenc(hmac(sha224),cbc(des))",
2882                                 .cra_driver_name = "authenc-hmac-sha224-"
2883                                                    "cbc-des-caam",
2884                                 .cra_blocksize = DES_BLOCK_SIZE,
2885                         },
2886                         .setkey = aead_setkey,
2887                         .setauthsize = aead_setauthsize,
2888                         .encrypt = aead_encrypt,
2889                         .decrypt = aead_decrypt,
2890                         .ivsize = DES_BLOCK_SIZE,
2891                         .maxauthsize = SHA224_DIGEST_SIZE,
2892                 },
2893                 .caam = {
2894                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2895                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2896                                            OP_ALG_AAI_HMAC_PRECOMP,
2897                 },
2898         },
2899         {
2900                 .aead = {
2901                         .base = {
2902                                 .cra_name = "echainiv(authenc(hmac(sha224),"
2903                                             "cbc(des)))",
2904                                 .cra_driver_name = "echainiv-authenc-"
2905                                                    "hmac-sha224-cbc-des-caam",
2906                                 .cra_blocksize = DES_BLOCK_SIZE,
2907                         },
2908                         .setkey = aead_setkey,
2909                         .setauthsize = aead_setauthsize,
2910                         .encrypt = aead_encrypt,
2911                         .decrypt = aead_decrypt,
2912                         .ivsize = DES_BLOCK_SIZE,
2913                         .maxauthsize = SHA224_DIGEST_SIZE,
2914                 },
2915                 .caam = {
2916                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2917                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2918                                            OP_ALG_AAI_HMAC_PRECOMP,
2919                         .geniv = true,
2920                 },
2921         },
2922         {
2923                 .aead = {
2924                         .base = {
2925                                 .cra_name = "authenc(hmac(sha256),cbc(des))",
2926                                 .cra_driver_name = "authenc-hmac-sha256-"
2927                                                    "cbc-des-caam",
2928                                 .cra_blocksize = DES_BLOCK_SIZE,
2929                         },
2930                         .setkey = aead_setkey,
2931                         .setauthsize = aead_setauthsize,
2932                         .encrypt = aead_encrypt,
2933                         .decrypt = aead_decrypt,
2934                         .ivsize = DES_BLOCK_SIZE,
2935                         .maxauthsize = SHA256_DIGEST_SIZE,
2936                 },
2937                 .caam = {
2938                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2939                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2940                                            OP_ALG_AAI_HMAC_PRECOMP,
2941                 },
2942         },
2943         {
2944                 .aead = {
2945                         .base = {
2946                                 .cra_name = "echainiv(authenc(hmac(sha256),"
2947                                             "cbc(des)))",
2948                                 .cra_driver_name = "echainiv-authenc-"
2949                                                    "hmac-sha256-cbc-des-caam",
2950                                 .cra_blocksize = DES_BLOCK_SIZE,
2951                         },
2952                         .setkey = aead_setkey,
2953                         .setauthsize = aead_setauthsize,
2954                         .encrypt = aead_encrypt,
2955                         .decrypt = aead_decrypt,
2956                         .ivsize = DES_BLOCK_SIZE,
2957                         .maxauthsize = SHA256_DIGEST_SIZE,
2958                 },
2959                 .caam = {
2960                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2961                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2962                                            OP_ALG_AAI_HMAC_PRECOMP,
2963                         .geniv = true,
2964                 },
2965         },
2966         {
2967                 .aead = {
2968                         .base = {
2969                                 .cra_name = "authenc(hmac(sha384),cbc(des))",
2970                                 .cra_driver_name = "authenc-hmac-sha384-"
2971                                                    "cbc-des-caam",
2972                                 .cra_blocksize = DES_BLOCK_SIZE,
2973                         },
2974                         .setkey = aead_setkey,
2975                         .setauthsize = aead_setauthsize,
2976                         .encrypt = aead_encrypt,
2977                         .decrypt = aead_decrypt,
2978                         .ivsize = DES_BLOCK_SIZE,
2979                         .maxauthsize = SHA384_DIGEST_SIZE,
2980                 },
2981                 .caam = {
2982                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2983                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2984                                            OP_ALG_AAI_HMAC_PRECOMP,
2985                 },
2986         },
2987         {
2988                 .aead = {
2989                         .base = {
2990                                 .cra_name = "echainiv(authenc(hmac(sha384),"
2991                                             "cbc(des)))",
2992                                 .cra_driver_name = "echainiv-authenc-"
2993                                                    "hmac-sha384-cbc-des-caam",
2994                                 .cra_blocksize = DES_BLOCK_SIZE,
2995                         },
2996                         .setkey = aead_setkey,
2997                         .setauthsize = aead_setauthsize,
2998                         .encrypt = aead_encrypt,
2999                         .decrypt = aead_decrypt,
3000                         .ivsize = DES_BLOCK_SIZE,
3001                         .maxauthsize = SHA384_DIGEST_SIZE,
3002                 },
3003                 .caam = {
3004                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3005                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3006                                            OP_ALG_AAI_HMAC_PRECOMP,
3007                         .geniv = true,
3008                 },
3009         },
3010         {
3011                 .aead = {
3012                         .base = {
3013                                 .cra_name = "authenc(hmac(sha512),cbc(des))",
3014                                 .cra_driver_name = "authenc-hmac-sha512-"
3015                                                    "cbc-des-caam",
3016                                 .cra_blocksize = DES_BLOCK_SIZE,
3017                         },
3018                         .setkey = aead_setkey,
3019                         .setauthsize = aead_setauthsize,
3020                         .encrypt = aead_encrypt,
3021                         .decrypt = aead_decrypt,
3022                         .ivsize = DES_BLOCK_SIZE,
3023                         .maxauthsize = SHA512_DIGEST_SIZE,
3024                 },
3025                 .caam = {
3026                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3027                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3028                                            OP_ALG_AAI_HMAC_PRECOMP,
3029                 },
3030         },
3031         {
3032                 .aead = {
3033                         .base = {
3034                                 .cra_name = "echainiv(authenc(hmac(sha512),"
3035                                             "cbc(des)))",
3036                                 .cra_driver_name = "echainiv-authenc-"
3037                                                    "hmac-sha512-cbc-des-caam",
3038                                 .cra_blocksize = DES_BLOCK_SIZE,
3039                         },
3040                         .setkey = aead_setkey,
3041                         .setauthsize = aead_setauthsize,
3042                         .encrypt = aead_encrypt,
3043                         .decrypt = aead_decrypt,
3044                         .ivsize = DES_BLOCK_SIZE,
3045                         .maxauthsize = SHA512_DIGEST_SIZE,
3046                 },
3047                 .caam = {
3048                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3049                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3050                                            OP_ALG_AAI_HMAC_PRECOMP,
3051                         .geniv = true,
3052                 },
3053         },
3054         {
3055                 .aead = {
3056                         .base = {
3057                                 .cra_name = "authenc(hmac(md5),"
3058                                             "rfc3686(ctr(aes)))",
3059                                 .cra_driver_name = "authenc-hmac-md5-"
3060                                                    "rfc3686-ctr-aes-caam",
3061                                 .cra_blocksize = 1,
3062                         },
3063                         .setkey = aead_setkey,
3064                         .setauthsize = aead_setauthsize,
3065                         .encrypt = aead_encrypt,
3066                         .decrypt = aead_decrypt,
3067                         .ivsize = CTR_RFC3686_IV_SIZE,
3068                         .maxauthsize = MD5_DIGEST_SIZE,
3069                 },
3070                 .caam = {
3071                         .class1_alg_type = OP_ALG_ALGSEL_AES |
3072                                            OP_ALG_AAI_CTR_MOD128,
3073                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
3074                                            OP_ALG_AAI_HMAC_PRECOMP,
3075                         .rfc3686 = true,
3076                 },
3077         },
3078         {
3079                 .aead = {
3080                         .base = {
3081                                 .cra_name = "seqiv(authenc("
3082                                             "hmac(md5),rfc3686(ctr(aes))))",
3083                                 .cra_driver_name = "seqiv-authenc-hmac-md5-"
3084                                                    "rfc3686-ctr-aes-caam",
3085                                 .cra_blocksize = 1,
3086                         },
3087                         .setkey = aead_setkey,
3088                         .setauthsize = aead_setauthsize,
3089                         .encrypt = aead_encrypt,
3090                         .decrypt = aead_decrypt,
3091                         .ivsize = CTR_RFC3686_IV_SIZE,
3092                         .maxauthsize = MD5_DIGEST_SIZE,
3093                 },
3094                 .caam = {
3095                         .class1_alg_type = OP_ALG_ALGSEL_AES |
3096                                            OP_ALG_AAI_CTR_MOD128,
3097                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
3098                                            OP_ALG_AAI_HMAC_PRECOMP,
3099                         .rfc3686 = true,
3100                         .geniv = true,
3101                 },
3102         },
3103         {
3104                 .aead = {
3105                         .base = {
3106                                 .cra_name = "authenc(hmac(sha1),"
3107                                             "rfc3686(ctr(aes)))",
3108                                 .cra_driver_name = "authenc-hmac-sha1-"
3109                                                    "rfc3686-ctr-aes-caam",
3110                                 .cra_blocksize = 1,
3111                         },
3112                         .setkey = aead_setkey,
3113                         .setauthsize = aead_setauthsize,
3114                         .encrypt = aead_encrypt,
3115                         .decrypt = aead_decrypt,
3116                         .ivsize = CTR_RFC3686_IV_SIZE,
3117                         .maxauthsize = SHA1_DIGEST_SIZE,
3118                 },
3119                 .caam = {
3120                         .class1_alg_type = OP_ALG_ALGSEL_AES |
3121                                            OP_ALG_AAI_CTR_MOD128,
3122                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
3123                                            OP_ALG_AAI_HMAC_PRECOMP,
3124                         .rfc3686 = true,
3125                 },
3126         },
3127         {
3128                 .aead = {
3129                         .base = {
3130                                 .cra_name = "seqiv(authenc("
3131                                             "hmac(sha1),rfc3686(ctr(aes))))",
3132                                 .cra_driver_name = "seqiv-authenc-hmac-sha1-"
3133                                                    "rfc3686-ctr-aes-caam",
3134                                 .cra_blocksize = 1,
3135                         },
3136                         .setkey = aead_setkey,
3137                         .setauthsize = aead_setauthsize,
3138                         .encrypt = aead_encrypt,
3139                         .decrypt = aead_decrypt,
3140                         .ivsize = CTR_RFC3686_IV_SIZE,
3141                         .maxauthsize = SHA1_DIGEST_SIZE,
3142                 },
3143                 .caam = {
3144                         .class1_alg_type = OP_ALG_ALGSEL_AES |
3145                                            OP_ALG_AAI_CTR_MOD128,
3146                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
3147                                            OP_ALG_AAI_HMAC_PRECOMP,
3148                         .rfc3686 = true,
3149                         .geniv = true,
3150                 },
3151         },
3152         {
3153                 .aead = {
3154                         .base = {
3155                                 .cra_name = "authenc(hmac(sha224),"
3156                                             "rfc3686(ctr(aes)))",
3157                                 .cra_driver_name = "authenc-hmac-sha224-"
3158                                                    "rfc3686-ctr-aes-caam",
3159                                 .cra_blocksize = 1,
3160                         },
3161                         .setkey = aead_setkey,
3162                         .setauthsize = aead_setauthsize,
3163                         .encrypt = aead_encrypt,
3164                         .decrypt = aead_decrypt,
3165                         .ivsize = CTR_RFC3686_IV_SIZE,
3166                         .maxauthsize = SHA224_DIGEST_SIZE,
3167                 },
3168                 .caam = {
3169                         .class1_alg_type = OP_ALG_ALGSEL_AES |
3170                                            OP_ALG_AAI_CTR_MOD128,
3171                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3172                                            OP_ALG_AAI_HMAC_PRECOMP,
3173                         .rfc3686 = true,
3174                 },
3175         },
3176         {
3177                 .aead = {
3178                         .base = {
3179                                 .cra_name = "seqiv(authenc("
3180                                             "hmac(sha224),rfc3686(ctr(aes))))",
3181                                 .cra_driver_name = "seqiv-authenc-hmac-sha224-"
3182                                                    "rfc3686-ctr-aes-caam",
3183                                 .cra_blocksize = 1,
3184                         },
3185                         .setkey = aead_setkey,
3186                         .setauthsize = aead_setauthsize,
3187                         .encrypt = aead_encrypt,
3188                         .decrypt = aead_decrypt,
3189                         .ivsize = CTR_RFC3686_IV_SIZE,
3190                         .maxauthsize = SHA224_DIGEST_SIZE,
3191                 },
3192                 .caam = {
3193                         .class1_alg_type = OP_ALG_ALGSEL_AES |
3194                                            OP_ALG_AAI_CTR_MOD128,
3195                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3196                                            OP_ALG_AAI_HMAC_PRECOMP,
3197                         .rfc3686 = true,
3198                         .geniv = true,
3199                 },
3200         },
3201         {
3202                 .aead = {
3203                         .base = {
3204                                 .cra_name = "authenc(hmac(sha256),"
3205                                             "rfc3686(ctr(aes)))",
3206                                 .cra_driver_name = "authenc-hmac-sha256-"
3207                                                    "rfc3686-ctr-aes-caam",
3208                                 .cra_blocksize = 1,
3209                         },
3210                         .setkey = aead_setkey,
3211                         .setauthsize = aead_setauthsize,
3212                         .encrypt = aead_encrypt,
3213                         .decrypt = aead_decrypt,
3214                         .ivsize = CTR_RFC3686_IV_SIZE,
3215                         .maxauthsize = SHA256_DIGEST_SIZE,
3216                 },
3217                 .caam = {
3218                         .class1_alg_type = OP_ALG_ALGSEL_AES |
3219                                            OP_ALG_AAI_CTR_MOD128,
3220                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3221                                            OP_ALG_AAI_HMAC_PRECOMP,
3222                         .rfc3686 = true,
3223                 },
3224         },
3225         {
3226                 .aead = {
3227                         .base = {
3228                                 .cra_name = "seqiv(authenc(hmac(sha256),"
3229                                             "rfc3686(ctr(aes))))",
3230                                 .cra_driver_name = "seqiv-authenc-hmac-sha256-"
3231                                                    "rfc3686-ctr-aes-caam",
3232                                 .cra_blocksize = 1,
3233                         },
3234                         .setkey = aead_setkey,
3235                         .setauthsize = aead_setauthsize,
3236                         .encrypt = aead_encrypt,
3237                         .decrypt = aead_decrypt,
3238                         .ivsize = CTR_RFC3686_IV_SIZE,
3239                         .maxauthsize = SHA256_DIGEST_SIZE,
3240                 },
3241                 .caam = {
3242                         .class1_alg_type = OP_ALG_ALGSEL_AES |
3243                                            OP_ALG_AAI_CTR_MOD128,
3244                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3245                                            OP_ALG_AAI_HMAC_PRECOMP,
3246                         .rfc3686 = true,
3247                         .geniv = true,
3248                 },
3249         },
3250         {
3251                 .aead = {
3252                         .base = {
3253                                 .cra_name = "authenc(hmac(sha384),"
3254                                             "rfc3686(ctr(aes)))",
3255                                 .cra_driver_name = "authenc-hmac-sha384-"
3256                                                    "rfc3686-ctr-aes-caam",
3257                                 .cra_blocksize = 1,
3258                         },
3259                         .setkey = aead_setkey,
3260                         .setauthsize = aead_setauthsize,
3261                         .encrypt = aead_encrypt,
3262                         .decrypt = aead_decrypt,
3263                         .ivsize = CTR_RFC3686_IV_SIZE,
3264                         .maxauthsize = SHA384_DIGEST_SIZE,
3265                 },
3266                 .caam = {
3267                         .class1_alg_type = OP_ALG_ALGSEL_AES |
3268                                            OP_ALG_AAI_CTR_MOD128,
3269                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3270                                            OP_ALG_AAI_HMAC_PRECOMP,
3271                         .rfc3686 = true,
3272                 },
3273         },
3274         {
3275                 .aead = {
3276                         .base = {
3277                                 .cra_name = "seqiv(authenc(hmac(sha384),"
3278                                             "rfc3686(ctr(aes))))",
3279                                 .cra_driver_name = "seqiv-authenc-hmac-sha384-"
3280                                                    "rfc3686-ctr-aes-caam",
3281                                 .cra_blocksize = 1,
3282                         },
3283                         .setkey = aead_setkey,
3284                         .setauthsize = aead_setauthsize,
3285                         .encrypt = aead_encrypt,
3286                         .decrypt = aead_decrypt,
3287                         .ivsize = CTR_RFC3686_IV_SIZE,
3288                         .maxauthsize = SHA384_DIGEST_SIZE,
3289                 },
3290                 .caam = {
3291                         .class1_alg_type = OP_ALG_ALGSEL_AES |
3292                                            OP_ALG_AAI_CTR_MOD128,
3293                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3294                                            OP_ALG_AAI_HMAC_PRECOMP,
3295                         .rfc3686 = true,
3296                         .geniv = true,
3297                 },
3298         },
3299         {
3300                 .aead = {
3301                         .base = {
3302                                 .cra_name = "authenc(hmac(sha512),"
3303                                             "rfc3686(ctr(aes)))",
3304                                 .cra_driver_name = "authenc-hmac-sha512-"
3305                                                    "rfc3686-ctr-aes-caam",
3306                                 .cra_blocksize = 1,
3307                         },
3308                         .setkey = aead_setkey,
3309                         .setauthsize = aead_setauthsize,
3310                         .encrypt = aead_encrypt,
3311                         .decrypt = aead_decrypt,
3312                         .ivsize = CTR_RFC3686_IV_SIZE,
3313                         .maxauthsize = SHA512_DIGEST_SIZE,
3314                 },
3315                 .caam = {
3316                         .class1_alg_type = OP_ALG_ALGSEL_AES |
3317                                            OP_ALG_AAI_CTR_MOD128,
3318                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3319                                            OP_ALG_AAI_HMAC_PRECOMP,
3320                         .rfc3686 = true,
3321                 },
3322         },
3323         {
3324                 .aead = {
3325                         .base = {
3326                                 .cra_name = "seqiv(authenc(hmac(sha512),"
3327                                             "rfc3686(ctr(aes))))",
3328                                 .cra_driver_name = "seqiv-authenc-hmac-sha512-"
3329                                                    "rfc3686-ctr-aes-caam",
3330                                 .cra_blocksize = 1,
3331                         },
3332                         .setkey = aead_setkey,
3333                         .setauthsize = aead_setauthsize,
3334                         .encrypt = aead_encrypt,
3335                         .decrypt = aead_decrypt,
3336                         .ivsize = CTR_RFC3686_IV_SIZE,
3337                         .maxauthsize = SHA512_DIGEST_SIZE,
3338                 },
3339                 .caam = {
3340                         .class1_alg_type = OP_ALG_ALGSEL_AES |
3341                                            OP_ALG_AAI_CTR_MOD128,
3342                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3343                                            OP_ALG_AAI_HMAC_PRECOMP,
3344                         .rfc3686 = true,
3345                         .geniv = true,
3346                 },
3347         },
3348 };
3349
3350 struct caam_crypto_alg {
3351         struct crypto_alg crypto_alg;
3352         struct list_head entry;
3353         struct caam_alg_entry caam;
3354 };
3355
3356 static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam)
3357 {
3358         ctx->jrdev = caam_jr_alloc();
3359         if (IS_ERR(ctx->jrdev)) {
3360                 pr_err("Job Ring Device allocation for transform failed\n");
3361                 return PTR_ERR(ctx->jrdev);
3362         }
3363
3364         /* copy descriptor header template value */
3365         ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type;
3366         ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type;
3367
3368         return 0;
3369 }
3370
3371 static int caam_cra_init(struct crypto_tfm *tfm)
3372 {
3373         struct crypto_alg *alg = tfm->__crt_alg;
3374         struct caam_crypto_alg *caam_alg =
3375                  container_of(alg, struct caam_crypto_alg, crypto_alg);
3376         struct caam_ctx *ctx = crypto_tfm_ctx(tfm);
3377
3378         return caam_init_common(ctx, &caam_alg->caam);
3379 }
3380
3381 static int caam_aead_init(struct crypto_aead *tfm)
3382 {
3383         struct aead_alg *alg = crypto_aead_alg(tfm);
3384         struct caam_aead_alg *caam_alg =
3385                  container_of(alg, struct caam_aead_alg, aead);
3386         struct caam_ctx *ctx = crypto_aead_ctx(tfm);
3387
3388         return caam_init_common(ctx, &caam_alg->caam);
3389 }
3390
3391 static void caam_exit_common(struct caam_ctx *ctx)
3392 {
3393         if (ctx->sh_desc_enc_dma &&
3394             !dma_mapping_error(ctx->jrdev, ctx->sh_desc_enc_dma))
3395                 dma_unmap_single(ctx->jrdev, ctx->sh_desc_enc_dma,
3396                                  desc_bytes(ctx->sh_desc_enc), DMA_TO_DEVICE);
3397         if (ctx->sh_desc_dec_dma &&
3398             !dma_mapping_error(ctx->jrdev, ctx->sh_desc_dec_dma))
3399                 dma_unmap_single(ctx->jrdev, ctx->sh_desc_dec_dma,
3400                                  desc_bytes(ctx->sh_desc_dec), DMA_TO_DEVICE);
3401         if (ctx->sh_desc_givenc_dma &&
3402             !dma_mapping_error(ctx->jrdev, ctx->sh_desc_givenc_dma))
3403                 dma_unmap_single(ctx->jrdev, ctx->sh_desc_givenc_dma,
3404                                  desc_bytes(ctx->sh_desc_givenc),
3405                                  DMA_TO_DEVICE);
3406         if (ctx->key_dma &&
3407             !dma_mapping_error(ctx->jrdev, ctx->key_dma))
3408                 dma_unmap_single(ctx->jrdev, ctx->key_dma,
3409                                  ctx->cdata.keylen + ctx->adata.keylen_pad,
3410                                  DMA_TO_DEVICE);
3411
3412         caam_jr_free(ctx->jrdev);
3413 }
3414
3415 static void caam_cra_exit(struct crypto_tfm *tfm)
3416 {
3417         caam_exit_common(crypto_tfm_ctx(tfm));
3418 }
3419
3420 static void caam_aead_exit(struct crypto_aead *tfm)
3421 {
3422         caam_exit_common(crypto_aead_ctx(tfm));
3423 }
3424
3425 static void __exit caam_algapi_exit(void)
3426 {
3427
3428         struct caam_crypto_alg *t_alg, *n;
3429         int i;
3430
3431         for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
3432                 struct caam_aead_alg *t_alg = driver_aeads + i;
3433
3434                 if (t_alg->registered)
3435                         crypto_unregister_aead(&t_alg->aead);
3436         }
3437
3438         if (!alg_list.next)
3439                 return;
3440
3441         list_for_each_entry_safe(t_alg, n, &alg_list, entry) {
3442                 crypto_unregister_alg(&t_alg->crypto_alg);
3443                 list_del(&t_alg->entry);
3444                 kfree(t_alg);
3445         }
3446 }
3447
3448 static struct caam_crypto_alg *caam_alg_alloc(struct caam_alg_template
3449                                               *template)
3450 {
3451         struct caam_crypto_alg *t_alg;
3452         struct crypto_alg *alg;
3453
3454         t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL);
3455         if (!t_alg) {
3456                 pr_err("failed to allocate t_alg\n");
3457                 return ERR_PTR(-ENOMEM);
3458         }
3459
3460         alg = &t_alg->crypto_alg;
3461
3462         snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", template->name);
3463         snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
3464                  template->driver_name);
3465         alg->cra_module = THIS_MODULE;
3466         alg->cra_init = caam_cra_init;
3467         alg->cra_exit = caam_cra_exit;
3468         alg->cra_priority = CAAM_CRA_PRIORITY;
3469         alg->cra_blocksize = template->blocksize;
3470         alg->cra_alignmask = 0;
3471         alg->cra_ctxsize = sizeof(struct caam_ctx);
3472         alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY |
3473                          template->type;
3474         switch (template->type) {
3475         case CRYPTO_ALG_TYPE_GIVCIPHER:
3476                 alg->cra_type = &crypto_givcipher_type;
3477                 alg->cra_ablkcipher = template->template_ablkcipher;
3478                 break;
3479         case CRYPTO_ALG_TYPE_ABLKCIPHER:
3480                 alg->cra_type = &crypto_ablkcipher_type;
3481                 alg->cra_ablkcipher = template->template_ablkcipher;
3482                 break;
3483         }
3484
3485         t_alg->caam.class1_alg_type = template->class1_alg_type;
3486         t_alg->caam.class2_alg_type = template->class2_alg_type;
3487
3488         return t_alg;
3489 }
3490
3491 static void caam_aead_alg_init(struct caam_aead_alg *t_alg)
3492 {
3493         struct aead_alg *alg = &t_alg->aead;
3494
3495         alg->base.cra_module = THIS_MODULE;
3496         alg->base.cra_priority = CAAM_CRA_PRIORITY;
3497         alg->base.cra_ctxsize = sizeof(struct caam_ctx);
3498         alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY;
3499
3500         alg->init = caam_aead_init;
3501         alg->exit = caam_aead_exit;
3502 }
3503
3504 static int __init caam_algapi_init(void)
3505 {
3506         struct device_node *dev_node;
3507         struct platform_device *pdev;
3508         struct device *ctrldev;
3509         struct caam_drv_private *priv;
3510         int i = 0, err = 0;
3511         u32 cha_vid, cha_inst, des_inst, aes_inst, md_inst;
3512         unsigned int md_limit = SHA512_DIGEST_SIZE;
3513         bool registered = false;
3514
3515         dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0");
3516         if (!dev_node) {
3517                 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0");
3518                 if (!dev_node)
3519                         return -ENODEV;
3520         }
3521
3522         pdev = of_find_device_by_node(dev_node);
3523         if (!pdev) {
3524                 of_node_put(dev_node);
3525                 return -ENODEV;
3526         }
3527
3528         ctrldev = &pdev->dev;
3529         priv = dev_get_drvdata(ctrldev);
3530         of_node_put(dev_node);
3531
3532         /*
3533          * If priv is NULL, it's probably because the caam driver wasn't
3534          * properly initialized (e.g. RNG4 init failed). Thus, bail out here.
3535          */
3536         if (!priv)
3537                 return -ENODEV;
3538
3539
3540         INIT_LIST_HEAD(&alg_list);
3541
3542         /*
3543          * Register crypto algorithms the device supports.
3544          * First, detect presence and attributes of DES, AES, and MD blocks.
3545          */
3546         cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls);
3547         cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls);
3548         des_inst = (cha_inst & CHA_ID_LS_DES_MASK) >> CHA_ID_LS_DES_SHIFT;
3549         aes_inst = (cha_inst & CHA_ID_LS_AES_MASK) >> CHA_ID_LS_AES_SHIFT;
3550         md_inst = (cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT;
3551
3552         /* If MD is present, limit digest size based on LP256 */
3553         if (md_inst && ((cha_vid & CHA_ID_LS_MD_MASK) == CHA_ID_LS_MD_LP256))
3554                 md_limit = SHA256_DIGEST_SIZE;
3555
3556         for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
3557                 struct caam_crypto_alg *t_alg;
3558                 struct caam_alg_template *alg = driver_algs + i;
3559                 u32 alg_sel = alg->class1_alg_type & OP_ALG_ALGSEL_MASK;
3560
3561                 /* Skip DES algorithms if not supported by device */
3562                 if (!des_inst &&
3563                     ((alg_sel == OP_ALG_ALGSEL_3DES) ||
3564                      (alg_sel == OP_ALG_ALGSEL_DES)))
3565                                 continue;
3566
3567                 /* Skip AES algorithms if not supported by device */
3568                 if (!aes_inst && (alg_sel == OP_ALG_ALGSEL_AES))
3569                                 continue;
3570
3571                 /*
3572                  * Check support for AES modes not available
3573                  * on LP devices.
3574                  */
3575                 if ((cha_vid & CHA_ID_LS_AES_MASK) == CHA_ID_LS_AES_LP)
3576                         if ((alg->class1_alg_type & OP_ALG_AAI_MASK) ==
3577                              OP_ALG_AAI_XTS)
3578                                 continue;
3579
3580                 t_alg = caam_alg_alloc(alg);
3581                 if (IS_ERR(t_alg)) {
3582                         err = PTR_ERR(t_alg);
3583                         pr_warn("%s alg allocation failed\n", alg->driver_name);
3584                         continue;
3585                 }
3586
3587                 err = crypto_register_alg(&t_alg->crypto_alg);
3588                 if (err) {
3589                         pr_warn("%s alg registration failed\n",
3590                                 t_alg->crypto_alg.cra_driver_name);
3591                         kfree(t_alg);
3592                         continue;
3593                 }
3594
3595                 list_add_tail(&t_alg->entry, &alg_list);
3596                 registered = true;
3597         }
3598
3599         for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
3600                 struct caam_aead_alg *t_alg = driver_aeads + i;
3601                 u32 c1_alg_sel = t_alg->caam.class1_alg_type &
3602                                  OP_ALG_ALGSEL_MASK;
3603                 u32 c2_alg_sel = t_alg->caam.class2_alg_type &
3604                                  OP_ALG_ALGSEL_MASK;
3605                 u32 alg_aai = t_alg->caam.class1_alg_type & OP_ALG_AAI_MASK;
3606
3607                 /* Skip DES algorithms if not supported by device */
3608                 if (!des_inst &&
3609                     ((c1_alg_sel == OP_ALG_ALGSEL_3DES) ||
3610                      (c1_alg_sel == OP_ALG_ALGSEL_DES)))
3611                                 continue;
3612
3613                 /* Skip AES algorithms if not supported by device */
3614                 if (!aes_inst && (c1_alg_sel == OP_ALG_ALGSEL_AES))
3615                                 continue;
3616
3617                 /*
3618                  * Check support for AES algorithms not available
3619                  * on LP devices.
3620                  */
3621                 if ((cha_vid & CHA_ID_LS_AES_MASK) == CHA_ID_LS_AES_LP)
3622                         if (alg_aai == OP_ALG_AAI_GCM)
3623                                 continue;
3624
3625                 /*
3626                  * Skip algorithms requiring message digests
3627                  * if MD or MD size is not supported by device.
3628                  */
3629                 if (c2_alg_sel &&
3630                     (!md_inst || (t_alg->aead.maxauthsize > md_limit)))
3631                                 continue;
3632
3633                 caam_aead_alg_init(t_alg);
3634
3635                 err = crypto_register_aead(&t_alg->aead);
3636                 if (err) {
3637                         pr_warn("%s alg registration failed\n",
3638                                 t_alg->aead.base.cra_driver_name);
3639                         continue;
3640                 }
3641
3642                 t_alg->registered = true;
3643                 registered = true;
3644         }
3645
3646         if (registered)
3647                 pr_info("caam algorithms registered in /proc/crypto\n");
3648
3649         return err;
3650 }
3651
3652 module_init(caam_algapi_init);
3653 module_exit(caam_algapi_exit);
3654
3655 MODULE_LICENSE("GPL");
3656 MODULE_DESCRIPTION("FSL CAAM support for crypto API");
3657 MODULE_AUTHOR("Freescale Semiconductor - NMG/STC");