drivers/crypto/chelsio/chcr_algo.c

   1 /*
   2  * This file is part of the Chelsio T6 Crypto driver for Linux.
   3  *
   4  * Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved.
   5  *
   6  * This software is available to you under a choice of one of two
   7  * licenses.  You may choose to be licensed under the terms of the GNU
   8  * General Public License (GPL) Version 2, available from the file
   9  * COPYING in the main directory of this source tree, or the
  10  * OpenIB.org BSD license below:
  11  *
  12  *     Redistribution and use in source and binary forms, with or
  13  *     without modification, are permitted provided that the following
  14  *     conditions are met:
  15  *
  16  *      - Redistributions of source code must retain the above
  17  *        copyright notice, this list of conditions and the following
  18  *        disclaimer.
  19  *
  20  *      - Redistributions in binary form must reproduce the above
  21  *        copyright notice, this list of conditions and the following
  22  *        disclaimer in the documentation and/or other materials
  23  *        provided with the distribution.
  24  *
  25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  26  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  27  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  28  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  29  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  30  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  31  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  32  * SOFTWARE.
  33  *
  34  * Written and Maintained by:
  35  *      Manoj Malviya (manojmalviya@chelsio.com)
  36  *      Atul Gupta (atul.gupta@chelsio.com)
  37  *      Jitendra Lulla (jlulla@chelsio.com)
  38  *      Yeshaswi M R Gowda (yeshaswi@chelsio.com)
  39  *      Harsh Jain (harsh@chelsio.com)
  40  */
  41
  42 #define pr_fmt(fmt) "chcr:" fmt
  43
  44 #include <linux/kernel.h>
  45 #include <linux/module.h>
  46 #include <linux/crypto.h>
  47 #include <linux/cryptohash.h>
  48 #include <linux/skbuff.h>
  49 #include <linux/rtnetlink.h>
  50 #include <linux/highmem.h>
  51 #include <linux/scatterlist.h>
  52
  53 #include <crypto/aes.h>
  54 #include <crypto/algapi.h>
  55 #include <crypto/hash.h>
  56 #include <crypto/sha.h>
  57 #include <crypto/authenc.h>
  58 #include <crypto/internal/aead.h>
  59 #include <crypto/null.h>
  60 #include <crypto/internal/skcipher.h>
  61 #include <crypto/aead.h>
  62 #include <crypto/scatterwalk.h>
  63 #include <crypto/internal/hash.h>
  64
  65 #include "t4fw_api.h"
  66 #include "t4_msg.h"
  67 #include "chcr_core.h"
  68 #include "chcr_algo.h"
  69 #include "chcr_crypto.h"
  70
  71 static inline  struct chcr_aead_ctx *AEAD_CTX(struct chcr_context *ctx)
  72 {
  73         return ctx->crypto_ctx->aeadctx;
  74 }
  75
  76 static inline struct ablk_ctx *ABLK_CTX(struct chcr_context *ctx)
  77 {
  78         return ctx->crypto_ctx->ablkctx;
  79 }
  80
  81 static inline struct hmac_ctx *HMAC_CTX(struct chcr_context *ctx)
  82 {
  83         return ctx->crypto_ctx->hmacctx;
  84 }
  85
  86 static inline struct chcr_gcm_ctx *GCM_CTX(struct chcr_aead_ctx *gctx)
  87 {
  88         return gctx->ctx->gcm;
  89 }
  90
  91 static inline struct chcr_authenc_ctx *AUTHENC_CTX(struct chcr_aead_ctx *gctx)
  92 {
  93         return gctx->ctx->authenc;
  94 }
  95
  96 static inline struct uld_ctx *ULD_CTX(struct chcr_context *ctx)
  97 {
  98         return ctx->dev->u_ctx;
  99 }
 100
 101 static inline int is_ofld_imm(const struct sk_buff *skb)
 102 {
 103         return (skb->len <= CRYPTO_MAX_IMM_TX_PKT_LEN);
 104 }
 105
 106 /*
 107  *      sgl_len - calculates the size of an SGL of the given capacity
 108  *      @n: the number of SGL entries
 109  *      Calculates the number of flits needed for a scatter/gather list that
 110  *      can hold the given number of entries.
 111  */
 112 static inline unsigned int sgl_len(unsigned int n)
 113 {
 114         n--;
 115         return (3 * n) / 2 + (n & 1) + 2;
 116 }
 117
 118 static void chcr_verify_tag(struct aead_request *req, u8 *input, int *err)
 119 {
 120         u8 temp[SHA512_DIGEST_SIZE];
 121         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 122         int authsize = crypto_aead_authsize(tfm);
 123         struct cpl_fw6_pld *fw6_pld;
 124         int cmp = 0;
 125
 126         fw6_pld = (struct cpl_fw6_pld *)input;
 127         if ((get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) ||
 128             (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_GCM)) {
 129                 cmp = memcmp(&fw6_pld->data[2], (fw6_pld + 1), authsize);
 130         } else {
 131
 132                 sg_pcopy_to_buffer(req->src, sg_nents(req->src), temp,
 133                                 authsize, req->assoclen +
 134                                 req->cryptlen - authsize);
 135                 cmp = memcmp(temp, (fw6_pld + 1), authsize);
 136         }
 137         if (cmp)
 138                 *err = -EBADMSG;
 139         else
 140                 *err = 0;
 141 }
 142
 143 /*
 144  *      chcr_handle_resp - Unmap the DMA buffers associated with the request
 145  *      @req: crypto request
 146  */
 147 int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input,
 148                          int err)
 149 {
 150         struct crypto_tfm *tfm = req->tfm;
 151         struct chcr_context *ctx = crypto_tfm_ctx(tfm);
 152         struct uld_ctx *u_ctx = ULD_CTX(ctx);
 153         struct chcr_req_ctx ctx_req;
 154         struct cpl_fw6_pld *fw6_pld;
 155         unsigned int digestsize, updated_digestsize;
 156
 157         switch (tfm->__crt_alg->cra_flags & CRYPTO_ALG_TYPE_MASK) {
 158         case CRYPTO_ALG_TYPE_AEAD:
 159                 ctx_req.req.aead_req = (struct aead_request *)req;
 160                 ctx_req.ctx.reqctx = aead_request_ctx(ctx_req.req.aead_req);
 161                 dma_unmap_sg(&u_ctx->lldi.pdev->dev, ctx_req.ctx.reqctx->dst,
 162                              ctx_req.ctx.reqctx->dst_nents, DMA_FROM_DEVICE);
 163                 if (ctx_req.ctx.reqctx->skb) {
 164                         kfree_skb(ctx_req.ctx.reqctx->skb);
 165                         ctx_req.ctx.reqctx->skb = NULL;
 166                 }
 167                 if (ctx_req.ctx.reqctx->verify == VERIFY_SW) {
 168                         chcr_verify_tag(ctx_req.req.aead_req, input,
 169                                         &err);
 170                         ctx_req.ctx.reqctx->verify = VERIFY_HW;
 171                 }
 172                 break;
 173
 174         case CRYPTO_ALG_TYPE_ABLKCIPHER:
 175                 ctx_req.req.ablk_req = (struct ablkcipher_request *)req;
 176                 ctx_req.ctx.ablk_ctx =
 177                         ablkcipher_request_ctx(ctx_req.req.ablk_req);
 178                 if (!err) {
 179                         fw6_pld = (struct cpl_fw6_pld *)input;
 180                         memcpy(ctx_req.req.ablk_req->info, &fw6_pld->data[2],
 181                                AES_BLOCK_SIZE);
 182                 }
 183                 dma_unmap_sg(&u_ctx->lldi.pdev->dev, ctx_req.req.ablk_req->dst,
 184                              ctx_req.ctx.ablk_ctx->dst_nents, DMA_FROM_DEVICE);
 185                 if (ctx_req.ctx.ablk_ctx->skb) {
 186                         kfree_skb(ctx_req.ctx.ablk_ctx->skb);
 187                         ctx_req.ctx.ablk_ctx->skb = NULL;
 188                 }
 189                 break;
 190
 191         case CRYPTO_ALG_TYPE_AHASH:
 192                 ctx_req.req.ahash_req = (struct ahash_request *)req;
 193                 ctx_req.ctx.ahash_ctx =
 194                         ahash_request_ctx(ctx_req.req.ahash_req);
 195                 digestsize =
 196                         crypto_ahash_digestsize(crypto_ahash_reqtfm(
 197                                                         ctx_req.req.ahash_req));
 198                 updated_digestsize = digestsize;
 199                 if (digestsize == SHA224_DIGEST_SIZE)
 200                         updated_digestsize = SHA256_DIGEST_SIZE;
 201                 else if (digestsize == SHA384_DIGEST_SIZE)
 202                         updated_digestsize = SHA512_DIGEST_SIZE;
 203                 if (ctx_req.ctx.ahash_ctx->skb) {
 204                         kfree_skb(ctx_req.ctx.ahash_ctx->skb);
 205                         ctx_req.ctx.ahash_ctx->skb = NULL;
 206                 }
 207                 if (ctx_req.ctx.ahash_ctx->result == 1) {
 208                         ctx_req.ctx.ahash_ctx->result = 0;
 209                         memcpy(ctx_req.req.ahash_req->result, input +
 210                                sizeof(struct cpl_fw6_pld),
 211                                digestsize);
 212                 } else {
 213                         memcpy(ctx_req.ctx.ahash_ctx->partial_hash, input +
 214                                sizeof(struct cpl_fw6_pld),
 215                                updated_digestsize);
 216                 }
 217                 break;
 218         }
 219         return err;
 220 }
 221
 222 /*
 223  *      calc_tx_flits_ofld - calculate # of flits for an offload packet
 224  *      @skb: the packet
 225  *      Returns the number of flits needed for the given offload packet.
 226  *      These packets are already fully constructed and no additional headers
 227  *      will be added.
 228  */
 229 static inline unsigned int calc_tx_flits_ofld(const struct sk_buff *skb)
 230 {
 231         unsigned int flits, cnt;
 232
 233         if (is_ofld_imm(skb))
 234                 return DIV_ROUND_UP(skb->len, 8);
 235
 236         flits = skb_transport_offset(skb) / 8;   /* headers */
 237         cnt = skb_shinfo(skb)->nr_frags;
 238         if (skb_tail_pointer(skb) != skb_transport_header(skb))
 239                 cnt++;
 240         return flits + sgl_len(cnt);
 241 }
 242
 243 static inline void get_aes_decrypt_key(unsigned char *dec_key,
 244                                        const unsigned char *key,
 245                                        unsigned int keylength)
 246 {
 247         u32 temp;
 248         u32 w_ring[MAX_NK];
 249         int i, j, k;
 250         u8  nr, nk;
 251
 252         switch (keylength) {
 253         case AES_KEYLENGTH_128BIT:
 254                 nk = KEYLENGTH_4BYTES;
 255                 nr = NUMBER_OF_ROUNDS_10;
 256                 break;
 257         case AES_KEYLENGTH_192BIT:
 258                 nk = KEYLENGTH_6BYTES;
 259                 nr = NUMBER_OF_ROUNDS_12;
 260                 break;
 261         case AES_KEYLENGTH_256BIT:
 262                 nk = KEYLENGTH_8BYTES;
 263                 nr = NUMBER_OF_ROUNDS_14;
 264                 break;
 265         default:
 266                 return;
 267         }
 268         for (i = 0; i < nk; i++)
 269                 w_ring[i] = be32_to_cpu(*(u32 *)&key[4 * i]);
 270
 271         i = 0;
 272         temp = w_ring[nk - 1];
 273         while (i + nk < (nr + 1) * 4) {
 274                 if (!(i % nk)) {
 275                         /* RotWord(temp) */
 276                         temp = (temp << 8) | (temp >> 24);
 277                         temp = aes_ks_subword(temp);
 278                         temp ^= round_constant[i / nk];
 279                 } else if (nk == 8 && (i % 4 == 0)) {
 280                         temp = aes_ks_subword(temp);
 281                 }
 282                 w_ring[i % nk] ^= temp;
 283                 temp = w_ring[i % nk];
 284                 i++;
 285         }
 286         i--;
 287         for (k = 0, j = i % nk; k < nk; k++) {
 288                 *((u32 *)dec_key + k) = htonl(w_ring[j]);
 289                 j--;
 290                 if (j < 0)
 291                         j += nk;
 292         }
 293 }
 294
 295 static struct crypto_shash *chcr_alloc_shash(unsigned int ds)
 296 {
 297         struct crypto_shash *base_hash = NULL;
 298
 299         switch (ds) {
 300         case SHA1_DIGEST_SIZE:
 301                 base_hash = crypto_alloc_shash("sha1", 0, 0);
 302                 break;
 303         case SHA224_DIGEST_SIZE:
 304                 base_hash = crypto_alloc_shash("sha224", 0, 0);
 305                 break;
 306         case SHA256_DIGEST_SIZE:
 307                 base_hash = crypto_alloc_shash("sha256", 0, 0);
 308                 break;
 309         case SHA384_DIGEST_SIZE:
 310                 base_hash = crypto_alloc_shash("sha384", 0, 0);
 311                 break;
 312         case SHA512_DIGEST_SIZE:
 313                 base_hash = crypto_alloc_shash("sha512", 0, 0);
 314                 break;
 315         }
 316
 317         return base_hash;
 318 }
 319
 320 static int chcr_compute_partial_hash(struct shash_desc *desc,
 321                                      char *iopad, char *result_hash,
 322                                      int digest_size)
 323 {
 324         struct sha1_state sha1_st;
 325         struct sha256_state sha256_st;
 326         struct sha512_state sha512_st;
 327         int error;
 328
 329         if (digest_size == SHA1_DIGEST_SIZE) {
 330                 error = crypto_shash_init(desc) ?:
 331                         crypto_shash_update(desc, iopad, SHA1_BLOCK_SIZE) ?:
 332                         crypto_shash_export(desc, (void *)&sha1_st);
 333                 memcpy(result_hash, sha1_st.state, SHA1_DIGEST_SIZE);
 334         } else if (digest_size == SHA224_DIGEST_SIZE) {
 335                 error = crypto_shash_init(desc) ?:
 336                         crypto_shash_update(desc, iopad, SHA256_BLOCK_SIZE) ?:
 337                         crypto_shash_export(desc, (void *)&sha256_st);
 338                 memcpy(result_hash, sha256_st.state, SHA256_DIGEST_SIZE);
 339
 340         } else if (digest_size == SHA256_DIGEST_SIZE) {
 341                 error = crypto_shash_init(desc) ?:
 342                         crypto_shash_update(desc, iopad, SHA256_BLOCK_SIZE) ?:
 343                         crypto_shash_export(desc, (void *)&sha256_st);
 344                 memcpy(result_hash, sha256_st.state, SHA256_DIGEST_SIZE);
 345
 346         } else if (digest_size == SHA384_DIGEST_SIZE) {
 347                 error = crypto_shash_init(desc) ?:
 348                         crypto_shash_update(desc, iopad, SHA512_BLOCK_SIZE) ?:
 349                         crypto_shash_export(desc, (void *)&sha512_st);
 350                 memcpy(result_hash, sha512_st.state, SHA512_DIGEST_SIZE);
 351
 352         } else if (digest_size == SHA512_DIGEST_SIZE) {
 353                 error = crypto_shash_init(desc) ?:
 354                         crypto_shash_update(desc, iopad, SHA512_BLOCK_SIZE) ?:
 355                         crypto_shash_export(desc, (void *)&sha512_st);
 356                 memcpy(result_hash, sha512_st.state, SHA512_DIGEST_SIZE);
 357         } else {
 358                 error = -EINVAL;
 359                 pr_err("Unknown digest size %d\n", digest_size);
 360         }
 361         return error;
 362 }
 363
 364 static void chcr_change_order(char *buf, int ds)
 365 {
 366         int i;
 367
 368         if (ds == SHA512_DIGEST_SIZE) {
 369                 for (i = 0; i < (ds / sizeof(u64)); i++)
 370                         *((__be64 *)buf + i) =
 371                                 cpu_to_be64(*((u64 *)buf + i));
 372         } else {
 373                 for (i = 0; i < (ds / sizeof(u32)); i++)
 374                         *((__be32 *)buf + i) =
 375                                 cpu_to_be32(*((u32 *)buf + i));
 376         }
 377 }
 378
 379 static inline int is_hmac(struct crypto_tfm *tfm)
 380 {
 381         struct crypto_alg *alg = tfm->__crt_alg;
 382         struct chcr_alg_template *chcr_crypto_alg =
 383                 container_of(__crypto_ahash_alg(alg), struct chcr_alg_template,
 384                              alg.hash);
 385         if (chcr_crypto_alg->type == CRYPTO_ALG_TYPE_HMAC)
 386                 return 1;
 387         return 0;
 388 }
 389
 390 static void write_phys_cpl(struct cpl_rx_phys_dsgl *phys_cpl,
 391                            struct scatterlist *sg,
 392                            struct phys_sge_parm *sg_param)
 393 {
 394         struct phys_sge_pairs *to;
 395         int out_buf_size = sg_param->obsize;
 396         unsigned int nents = sg_param->nents, i, j = 0;
 397
 398         phys_cpl->op_to_tid = htonl(CPL_RX_PHYS_DSGL_OPCODE_V(CPL_RX_PHYS_DSGL)
 399                                     | CPL_RX_PHYS_DSGL_ISRDMA_V(0));
 400         phys_cpl->pcirlxorder_to_noofsgentr =
 401                 htonl(CPL_RX_PHYS_DSGL_PCIRLXORDER_V(0) |
 402                       CPL_RX_PHYS_DSGL_PCINOSNOOP_V(0) |
 403                       CPL_RX_PHYS_DSGL_PCITPHNTENB_V(0) |
 404                       CPL_RX_PHYS_DSGL_PCITPHNT_V(0) |
 405                       CPL_RX_PHYS_DSGL_DCAID_V(0) |
 406                       CPL_RX_PHYS_DSGL_NOOFSGENTR_V(nents));
 407         phys_cpl->rss_hdr_int.opcode = CPL_RX_PHYS_ADDR;
 408         phys_cpl->rss_hdr_int.qid = htons(sg_param->qid);
 409         phys_cpl->rss_hdr_int.hash_val = 0;
 410         to = (struct phys_sge_pairs *)((unsigned char *)phys_cpl +
 411                                        sizeof(struct cpl_rx_phys_dsgl));
 412
 413         for (i = 0; nents; to++) {
 414                 for (j = 0; j < 8 && nents; j++, nents--) {
 415                         out_buf_size -= sg_dma_len(sg);
 416                         to->len[j] = htons(sg_dma_len(sg));
 417                         to->addr[j] = cpu_to_be64(sg_dma_address(sg));
 418                         sg = sg_next(sg);
 419                 }
 420         }
 421         if (out_buf_size) {
 422                 j--;
 423                 to--;
 424                 to->len[j] = htons(ntohs(to->len[j]) + (out_buf_size));
 425         }
 426 }
 427
 428 static inline int map_writesg_phys_cpl(struct device *dev,
 429                                         struct cpl_rx_phys_dsgl *phys_cpl,
 430                                         struct scatterlist *sg,
 431                                         struct phys_sge_parm *sg_param)
 432 {
 433         if (!sg || !sg_param->nents)
 434                 return 0;
 435
 436         sg_param->nents = dma_map_sg(dev, sg, sg_param->nents, DMA_FROM_DEVICE);
 437         if (sg_param->nents == 0) {
 438                 pr_err("CHCR : DMA mapping failed\n");
 439                 return -EINVAL;
 440         }
 441         write_phys_cpl(phys_cpl, sg, sg_param);
 442         return 0;
 443 }
 444
 445 static inline int get_aead_subtype(struct crypto_aead *aead)
 446 {
 447         struct aead_alg *alg = crypto_aead_alg(aead);
 448         struct chcr_alg_template *chcr_crypto_alg =
 449                 container_of(alg, struct chcr_alg_template, alg.aead);
 450         return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK;
 451 }
 452
 453 static inline int get_cryptoalg_subtype(struct crypto_tfm *tfm)
 454 {
 455         struct crypto_alg *alg = tfm->__crt_alg;
 456         struct chcr_alg_template *chcr_crypto_alg =
 457                 container_of(alg, struct chcr_alg_template, alg.crypto);
 458
 459         return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK;
 460 }
 461
 462 static inline void write_buffer_to_skb(struct sk_buff *skb,
 463                                         unsigned int *frags,
 464                                         char *bfr,
 465                                         u8 bfr_len)
 466 {
 467         skb->len += bfr_len;
 468         skb->data_len += bfr_len;
 469         skb->truesize += bfr_len;
 470         get_page(virt_to_page(bfr));
 471         skb_fill_page_desc(skb, *frags, virt_to_page(bfr),
 472                            offset_in_page(bfr), bfr_len);
 473         (*frags)++;
 474 }
 475
 476
 477 static inline void
 478 write_sg_to_skb(struct sk_buff *skb, unsigned int *frags,
 479                         struct scatterlist *sg, unsigned int count)
 480 {
 481         struct page *spage;
 482         unsigned int page_len;
 483
 484         skb->len += count;
 485         skb->data_len += count;
 486         skb->truesize += count;
 487
 488         while (count > 0) {
 489                 if (!sg || (!(sg->length)))
 490                         break;
 491                 spage = sg_page(sg);
 492                 get_page(spage);
 493                 page_len = min(sg->length, count);
 494                 skb_fill_page_desc(skb, *frags, spage, sg->offset, page_len);
 495                 (*frags)++;
 496                 count -= page_len;
 497                 sg = sg_next(sg);
 498         }
 499 }
 500
 501 static int generate_copy_rrkey(struct ablk_ctx *ablkctx,
 502                                struct _key_ctx *key_ctx)
 503 {
 504         if (ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC) {
 505                 memcpy(key_ctx->key, ablkctx->rrkey, ablkctx->enckey_len);
 506         } else {
 507                 memcpy(key_ctx->key,
 508                        ablkctx->key + (ablkctx->enckey_len >> 1),
 509                        ablkctx->enckey_len >> 1);
 510                 memcpy(key_ctx->key + (ablkctx->enckey_len >> 1),
 511                        ablkctx->rrkey, ablkctx->enckey_len >> 1);
 512         }
 513         return 0;
 514 }
 515
 516 static inline void create_wreq(struct chcr_context *ctx,
 517                                struct chcr_wr *chcr_req,
 518                                void *req, struct sk_buff *skb,
 519                                int kctx_len, int hash_sz,
 520                                int is_iv,
 521                                unsigned int sc_len)
 522 {
 523         struct uld_ctx *u_ctx = ULD_CTX(ctx);
 524         int iv_loc = IV_DSGL;
 525         int qid = u_ctx->lldi.rxq_ids[ctx->tx_channel_id];
 526         unsigned int immdatalen = 0, nr_frags = 0;
 527
 528         if (is_ofld_imm(skb)) {
 529                 immdatalen = skb->data_len;
 530                 iv_loc = IV_IMMEDIATE;
 531         } else {
 532                 nr_frags = skb_shinfo(skb)->nr_frags;
 533         }
 534
 535         chcr_req->wreq.op_to_cctx_size = FILL_WR_OP_CCTX_SIZE(immdatalen,
 536                                 ((sizeof(chcr_req->key_ctx) + kctx_len) >> 4));
 537         chcr_req->wreq.pld_size_hash_size =
 538                 htonl(FW_CRYPTO_LOOKASIDE_WR_PLD_SIZE_V(sgl_lengths[nr_frags]) |
 539                       FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_V(hash_sz));
 540         chcr_req->wreq.len16_pkd =
 541                 htonl(FW_CRYPTO_LOOKASIDE_WR_LEN16_V(DIV_ROUND_UP(
 542                                     (calc_tx_flits_ofld(skb) * 8), 16)));
 543         chcr_req->wreq.cookie = cpu_to_be64((uintptr_t)req);
 544         chcr_req->wreq.rx_chid_to_rx_q_id =
 545                 FILL_WR_RX_Q_ID(ctx->dev->rx_channel_id, qid,
 546                                 is_iv ? iv_loc : IV_NOP, ctx->tx_channel_id);
 547
 548         chcr_req->ulptx.cmd_dest = FILL_ULPTX_CMD_DEST(ctx->dev->tx_channel_id,
 549                                                        qid);
 550         chcr_req->ulptx.len = htonl((DIV_ROUND_UP((calc_tx_flits_ofld(skb) * 8),
 551                                         16) - ((sizeof(chcr_req->wreq)) >> 4)));
 552
 553         chcr_req->sc_imm.cmd_more = FILL_CMD_MORE(immdatalen);
 554         chcr_req->sc_imm.len = cpu_to_be32(sizeof(struct cpl_tx_sec_pdu) +
 555                                    sizeof(chcr_req->key_ctx) +
 556                                    kctx_len + sc_len + immdatalen);
 557 }
 558
 559 /**
 560  *      create_cipher_wr - form the WR for cipher operations
 561  *      @req: cipher req.
 562  *      @ctx: crypto driver context of the request.
 563  *      @qid: ingress qid where response of this WR should be received.
 564  *      @op_type:       encryption or decryption
 565  */
 566 static struct sk_buff
 567 *create_cipher_wr(struct ablkcipher_request *req,
 568                   unsigned short qid,
 569                   unsigned short op_type)
 570 {
 571         struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
 572         struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm);
 573         struct uld_ctx *u_ctx = ULD_CTX(ctx);
 574         struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
 575         struct sk_buff *skb = NULL;
 576         struct chcr_wr *chcr_req;
 577         struct cpl_rx_phys_dsgl *phys_cpl;
 578         struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
 579         struct phys_sge_parm sg_param;
 580         unsigned int frags = 0, transhdr_len, phys_dsgl;
 581         unsigned int ivsize = crypto_ablkcipher_ivsize(tfm), kctx_len;
 582         gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
 583                         GFP_ATOMIC;
 584
 585         if (!req->info)
 586                 return ERR_PTR(-EINVAL);
 587         reqctx->dst_nents = sg_nents_for_len(req->dst, req->nbytes);
 588         if (reqctx->dst_nents <= 0) {
 589                 pr_err("AES:Invalid Destination sg lists\n");
 590                 return ERR_PTR(-EINVAL);
 591         }
 592         if ((ablkctx->enckey_len == 0) || (ivsize > AES_BLOCK_SIZE) ||
 593             (req->nbytes <= 0) || (req->nbytes % AES_BLOCK_SIZE)) {
 594                 pr_err("AES: Invalid value of Key Len %d nbytes %d IV Len %d\n",
 595                        ablkctx->enckey_len, req->nbytes, ivsize);
 596                 return ERR_PTR(-EINVAL);
 597         }
 598
 599         phys_dsgl = get_space_for_phys_dsgl(reqctx->dst_nents);
 600
 601         kctx_len = (DIV_ROUND_UP(ablkctx->enckey_len, 16) * 16);
 602         transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, phys_dsgl);
 603         skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags);
 604         if (!skb)
 605                 return ERR_PTR(-ENOMEM);
 606         skb_reserve(skb, sizeof(struct sge_opaque_hdr));
 607         chcr_req = (struct chcr_wr *)__skb_put(skb, transhdr_len);
 608         memset(chcr_req, 0, transhdr_len);
 609         chcr_req->sec_cpl.op_ivinsrtofst =
 610                 FILL_SEC_CPL_OP_IVINSR(ctx->dev->rx_channel_id, 2, 1);
 611
 612         chcr_req->sec_cpl.pldlen = htonl(ivsize + req->nbytes);
 613         chcr_req->sec_cpl.aadstart_cipherstop_hi =
 614                         FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, ivsize + 1, 0);
 615
 616         chcr_req->sec_cpl.cipherstop_lo_authinsert =
 617                         FILL_SEC_CPL_AUTHINSERT(0, 0, 0, 0);
 618         chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type, 0,
 619                                                          ablkctx->ciph_mode,
 620                                                          0, 0, ivsize >> 1);
 621         chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 0,
 622                                                           0, 1, phys_dsgl);
 623
 624         chcr_req->key_ctx.ctx_hdr = ablkctx->key_ctx_hdr;
 625         if (op_type == CHCR_DECRYPT_OP) {
 626                 generate_copy_rrkey(ablkctx, &chcr_req->key_ctx);
 627         } else {
 628                 if (ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC) {
 629                         memcpy(chcr_req->key_ctx.key, ablkctx->key,
 630                                ablkctx->enckey_len);
 631                 } else {
 632                         memcpy(chcr_req->key_ctx.key, ablkctx->key +
 633                                (ablkctx->enckey_len >> 1),
 634                                ablkctx->enckey_len >> 1);
 635                         memcpy(chcr_req->key_ctx.key +
 636                                (ablkctx->enckey_len >> 1),
 637                                ablkctx->key,
 638                                ablkctx->enckey_len >> 1);
 639                 }
 640         }
 641         phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
 642         sg_param.nents = reqctx->dst_nents;
 643         sg_param.obsize = req->nbytes;
 644         sg_param.qid = qid;
 645         sg_param.align = 1;
 646         if (map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, req->dst,
 647                                  &sg_param))
 648                 goto map_fail1;
 649
 650         skb_set_transport_header(skb, transhdr_len);
 651         memcpy(reqctx->iv, req->info, ivsize);
 652         write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize);
 653         write_sg_to_skb(skb, &frags, req->src, req->nbytes);
 654         create_wreq(ctx, chcr_req, req, skb, kctx_len, 0, 1,
 655                         sizeof(struct cpl_rx_phys_dsgl) + phys_dsgl);
 656         reqctx->skb = skb;
 657         skb_get(skb);
 658         return skb;
 659 map_fail1:
 660         kfree_skb(skb);
 661         return ERR_PTR(-ENOMEM);
 662 }
 663
 664 static int chcr_aes_cbc_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
 665                                unsigned int keylen)
 666 {
 667         struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm);
 668         struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
 669         unsigned int ck_size, context_size;
 670         u16 alignment = 0;
 671
 672         if (keylen == AES_KEYSIZE_128) {
 673                 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
 674         } else if (keylen == AES_KEYSIZE_192) {
 675                 alignment = 8;
 676                 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
 677         } else if (keylen == AES_KEYSIZE_256) {
 678                 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
 679         } else {
 680                 goto badkey_err;
 681         }
 682         memcpy(ablkctx->key, key, keylen);
 683         ablkctx->enckey_len = keylen;
 684         get_aes_decrypt_key(ablkctx->rrkey, ablkctx->key, keylen << 3);
 685         context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD +
 686                         keylen + alignment) >> 4;
 687
 688         ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY,
 689                                                 0, 0, context_size);
 690         ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CBC;
 691         return 0;
 692 badkey_err:
 693         crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
 694         ablkctx->enckey_len = 0;
 695         return -EINVAL;
 696 }
 697
 698 static int cxgb4_is_crypto_q_full(struct net_device *dev, unsigned int idx)
 699 {
 700         struct adapter *adap = netdev2adap(dev);
 701         struct sge_uld_txq_info *txq_info =
 702                 adap->sge.uld_txq_info[CXGB4_TX_CRYPTO];
 703         struct sge_uld_txq *txq;
 704         int ret = 0;
 705
 706         local_bh_disable();
 707         txq = &txq_info->uldtxq[idx];
 708         spin_lock(&txq->sendq.lock);
 709         if (txq->full)
 710                 ret = -1;
 711         spin_unlock(&txq->sendq.lock);
 712         local_bh_enable();
 713         return ret;
 714 }
 715
 716 static int chcr_aes_encrypt(struct ablkcipher_request *req)
 717 {
 718         struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
 719         struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm);
 720         struct uld_ctx *u_ctx = ULD_CTX(ctx);
 721         struct sk_buff *skb;
 722
 723         if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
 724                                             ctx->tx_channel_id))) {
 725                 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
 726                         return -EBUSY;
 727         }
 728
 729         skb = create_cipher_wr(req, u_ctx->lldi.rxq_ids[ctx->tx_channel_id],
 730                                CHCR_ENCRYPT_OP);
 731         if (IS_ERR(skb)) {
 732                 pr_err("chcr : %s : Failed to form WR. No memory\n", __func__);
 733                 return  PTR_ERR(skb);
 734         }
 735         skb->dev = u_ctx->lldi.ports[0];
 736         set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_channel_id);
 737         chcr_send_wr(skb);
 738         return -EINPROGRESS;
 739 }
 740
 741 static int chcr_aes_decrypt(struct ablkcipher_request *req)
 742 {
 743         struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
 744         struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm);
 745         struct uld_ctx *u_ctx = ULD_CTX(ctx);
 746         struct sk_buff *skb;
 747
 748         if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
 749                                             ctx->tx_channel_id))) {
 750                 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
 751                         return -EBUSY;
 752         }
 753
 754         skb = create_cipher_wr(req, u_ctx->lldi.rxq_ids[0],
 755                                CHCR_DECRYPT_OP);
 756         if (IS_ERR(skb)) {
 757                 pr_err("chcr : %s : Failed to form WR. No memory\n", __func__);
 758                 return PTR_ERR(skb);
 759         }
 760         skb->dev = u_ctx->lldi.ports[0];
 761         set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_channel_id);
 762         chcr_send_wr(skb);
 763         return -EINPROGRESS;
 764 }
 765
 766 static int chcr_device_init(struct chcr_context *ctx)
 767 {
 768         struct uld_ctx *u_ctx;
 769         unsigned int id;
 770         int err = 0, rxq_perchan, rxq_idx;
 771
 772         id = smp_processor_id();
 773         if (!ctx->dev) {
 774                 err = assign_chcr_device(&ctx->dev);
 775                 if (err) {
 776                         pr_err("chcr device assignment fails\n");
 777                         goto out;
 778                 }
 779                 u_ctx = ULD_CTX(ctx);
 780                 rxq_perchan = u_ctx->lldi.nrxq / u_ctx->lldi.nchan;
 781                 rxq_idx = ctx->dev->tx_channel_id * rxq_perchan;
 782                 rxq_idx += id % rxq_perchan;
 783                 spin_lock(&ctx->dev->lock_chcr_dev);
 784                 ctx->tx_channel_id = rxq_idx;
 785                 ctx->dev->tx_channel_id = !ctx->dev->tx_channel_id;
 786                 ctx->dev->rx_channel_id = 0;
 787                 spin_unlock(&ctx->dev->lock_chcr_dev);
 788         }
 789 out:
 790         return err;
 791 }
 792
 793 static int chcr_cra_init(struct crypto_tfm *tfm)
 794 {
 795         tfm->crt_ablkcipher.reqsize =  sizeof(struct chcr_blkcipher_req_ctx);
 796         return chcr_device_init(crypto_tfm_ctx(tfm));
 797 }
 798
 799 static int get_alg_config(struct algo_param *params,
 800                           unsigned int auth_size)
 801 {
 802         switch (auth_size) {
 803         case SHA1_DIGEST_SIZE:
 804                 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_160;
 805                 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA1;
 806                 params->result_size = SHA1_DIGEST_SIZE;
 807                 break;
 808         case SHA224_DIGEST_SIZE:
 809                 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
 810                 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA224;
 811                 params->result_size = SHA256_DIGEST_SIZE;
 812                 break;
 813         case SHA256_DIGEST_SIZE:
 814                 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
 815                 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA256;
 816                 params->result_size = SHA256_DIGEST_SIZE;
 817                 break;
 818         case SHA384_DIGEST_SIZE:
 819                 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512;
 820                 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA512_384;
 821                 params->result_size = SHA512_DIGEST_SIZE;
 822                 break;
 823         case SHA512_DIGEST_SIZE:
 824                 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512;
 825                 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA512_512;
 826                 params->result_size = SHA512_DIGEST_SIZE;
 827                 break;
 828         default:
 829                 pr_err("chcr : ERROR, unsupported digest size\n");
 830                 return -EINVAL;
 831         }
 832         return 0;
 833 }
 834
 835 static inline void chcr_free_shash(struct crypto_shash *base_hash)
 836 {
 837                 crypto_free_shash(base_hash);
 838 }
 839
 840 /**
 841  *      create_hash_wr - Create hash work request
 842  *      @req - Cipher req base
 843  */
 844 static struct sk_buff *create_hash_wr(struct ahash_request *req,
 845                                       struct hash_wr_param *param)
 846 {
 847         struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
 848         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 849         struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
 850         struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
 851         struct sk_buff *skb = NULL;
 852         struct chcr_wr *chcr_req;
 853         unsigned int frags = 0, transhdr_len, iopad_alignment = 0;
 854         unsigned int digestsize = crypto_ahash_digestsize(tfm);
 855         unsigned int kctx_len = 0;
 856         u8 hash_size_in_response = 0;
 857         gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
 858                 GFP_ATOMIC;
 859
 860         iopad_alignment = KEYCTX_ALIGN_PAD(digestsize);
 861         kctx_len = param->alg_prm.result_size + iopad_alignment;
 862         if (param->opad_needed)
 863                 kctx_len += param->alg_prm.result_size + iopad_alignment;
 864
 865         if (req_ctx->result)
 866                 hash_size_in_response = digestsize;
 867         else
 868                 hash_size_in_response = param->alg_prm.result_size;
 869         transhdr_len = HASH_TRANSHDR_SIZE(kctx_len);
 870         skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags);
 871         if (!skb)
 872                 return skb;
 873
 874         skb_reserve(skb, sizeof(struct sge_opaque_hdr));
 875         chcr_req = (struct chcr_wr *)__skb_put(skb, transhdr_len);
 876         memset(chcr_req, 0, transhdr_len);
 877
 878         chcr_req->sec_cpl.op_ivinsrtofst =
 879                 FILL_SEC_CPL_OP_IVINSR(ctx->dev->rx_channel_id, 2, 0);
 880         chcr_req->sec_cpl.pldlen = htonl(param->bfr_len + param->sg_len);
 881
 882         chcr_req->sec_cpl.aadstart_cipherstop_hi =
 883                 FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, 0, 0);
 884         chcr_req->sec_cpl.cipherstop_lo_authinsert =
 885                 FILL_SEC_CPL_AUTHINSERT(0, 1, 0, 0);
 886         chcr_req->sec_cpl.seqno_numivs =
 887                 FILL_SEC_CPL_SCMD0_SEQNO(0, 0, 0, param->alg_prm.auth_mode,
 888                                          param->opad_needed, 0);
 889
 890         chcr_req->sec_cpl.ivgen_hdrlen =
 891                 FILL_SEC_CPL_IVGEN_HDRLEN(param->last, param->more, 0, 1, 0, 0);
 892
 893         memcpy(chcr_req->key_ctx.key, req_ctx->partial_hash,
 894                param->alg_prm.result_size);
 895
 896         if (param->opad_needed)
 897                 memcpy(chcr_req->key_ctx.key +
 898                        ((param->alg_prm.result_size <= 32) ? 32 :
 899                         CHCR_HASH_MAX_DIGEST_SIZE),
 900                        hmacctx->opad, param->alg_prm.result_size);
 901
 902         chcr_req->key_ctx.ctx_hdr = FILL_KEY_CTX_HDR(CHCR_KEYCTX_NO_KEY,
 903                                             param->alg_prm.mk_size, 0,
 904                                             param->opad_needed,
 905                                             ((kctx_len +
 906                                              sizeof(chcr_req->key_ctx)) >> 4));
 907         chcr_req->sec_cpl.scmd1 = cpu_to_be64((u64)param->scmd1);
 908
 909         skb_set_transport_header(skb, transhdr_len);
 910         if (param->bfr_len != 0)
 911                 write_buffer_to_skb(skb, &frags, req_ctx->reqbfr,
 912                                     param->bfr_len);
 913         if (param->sg_len != 0)
 914                 write_sg_to_skb(skb, &frags, req->src, param->sg_len);
 915
 916         create_wreq(ctx, chcr_req, req, skb, kctx_len, hash_size_in_response, 0,
 917                         DUMMY_BYTES);
 918         req_ctx->skb = skb;
 919         skb_get(skb);
 920         return skb;
 921 }
 922
 923 static int chcr_ahash_update(struct ahash_request *req)
 924 {
 925         struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
 926         struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
 927         struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm));
 928         struct uld_ctx *u_ctx = NULL;
 929         struct sk_buff *skb;
 930         u8 remainder = 0, bs;
 931         unsigned int nbytes = req->nbytes;
 932         struct hash_wr_param params;
 933
 934         bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
 935
 936         u_ctx = ULD_CTX(ctx);
 937         if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
 938                                             ctx->tx_channel_id))) {
 939                 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
 940                         return -EBUSY;
 941         }
 942
 943         if (nbytes + req_ctx->reqlen >= bs) {
 944                 remainder = (nbytes + req_ctx->reqlen) % bs;
 945                 nbytes = nbytes + req_ctx->reqlen - remainder;
 946         } else {
 947                 sg_pcopy_to_buffer(req->src, sg_nents(req->src), req_ctx->reqbfr
 948                                    + req_ctx->reqlen, nbytes, 0);
 949                 req_ctx->reqlen += nbytes;
 950                 return 0;
 951         }
 952
 953         params.opad_needed = 0;
 954         params.more = 1;
 955         params.last = 0;
 956         params.sg_len = nbytes - req_ctx->reqlen;
 957         params.bfr_len = req_ctx->reqlen;
 958         params.scmd1 = 0;
 959         get_alg_config(&params.alg_prm, crypto_ahash_digestsize(rtfm));
 960         req_ctx->result = 0;
 961         req_ctx->data_len += params.sg_len + params.bfr_len;
 962         skb = create_hash_wr(req, &params);
 963         if (!skb)
 964                 return -ENOMEM;
 965
 966         if (remainder) {
 967                 u8 *temp;
 968                 /* Swap buffers */
 969                 temp = req_ctx->reqbfr;
 970                 req_ctx->reqbfr = req_ctx->skbfr;
 971                 req_ctx->skbfr = temp;
 972                 sg_pcopy_to_buffer(req->src, sg_nents(req->src),
 973                                    req_ctx->reqbfr, remainder, req->nbytes -
 974                                    remainder);
 975         }
 976         req_ctx->reqlen = remainder;
 977         skb->dev = u_ctx->lldi.ports[0];
 978         set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_channel_id);
 979         chcr_send_wr(skb);
 980
 981         return -EINPROGRESS;
 982 }
 983
 984 static void create_last_hash_block(char *bfr_ptr, unsigned int bs, u64 scmd1)
 985 {
 986         memset(bfr_ptr, 0, bs);
 987         *bfr_ptr = 0x80;
 988         if (bs == 64)
 989                 *(__be64 *)(bfr_ptr + 56) = cpu_to_be64(scmd1  << 3);
 990         else
 991                 *(__be64 *)(bfr_ptr + 120) =  cpu_to_be64(scmd1  << 3);
 992 }
 993
 994 static int chcr_ahash_final(struct ahash_request *req)
 995 {
 996         struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
 997         struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
 998         struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm));
 999         struct hash_wr_param params;
1000         struct sk_buff *skb;
1001         struct uld_ctx *u_ctx = NULL;
1002         u8 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
1003
1004         u_ctx = ULD_CTX(ctx);
1005         if (is_hmac(crypto_ahash_tfm(rtfm)))
1006                 params.opad_needed = 1;
1007         else
1008                 params.opad_needed = 0;
1009         params.sg_len = 0;
1010         get_alg_config(&params.alg_prm, crypto_ahash_digestsize(rtfm));
1011         req_ctx->result = 1;
1012         params.bfr_len = req_ctx->reqlen;
1013         req_ctx->data_len += params.bfr_len + params.sg_len;
1014         if (req_ctx->reqlen == 0) {
1015                 create_last_hash_block(req_ctx->reqbfr, bs, req_ctx->data_len);
1016                 params.last = 0;
1017                 params.more = 1;
1018                 params.scmd1 = 0;
1019                 params.bfr_len = bs;
1020
1021         } else {
1022                 params.scmd1 = req_ctx->data_len;
1023                 params.last = 1;
1024                 params.more = 0;
1025         }
1026         skb = create_hash_wr(req, &params);
1027         if (!skb)
1028                 return -ENOMEM;
1029
1030         skb->dev = u_ctx->lldi.ports[0];
1031         set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_channel_id);
1032         chcr_send_wr(skb);
1033         return -EINPROGRESS;
1034 }
1035
1036 static int chcr_ahash_finup(struct ahash_request *req)
1037 {
1038         struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1039         struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
1040         struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm));
1041         struct uld_ctx *u_ctx = NULL;
1042         struct sk_buff *skb;
1043         struct hash_wr_param params;
1044         u8  bs;
1045
1046         bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
1047         u_ctx = ULD_CTX(ctx);
1048
1049         if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
1050                                             ctx->tx_channel_id))) {
1051                 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
1052                         return -EBUSY;
1053         }
1054
1055         if (is_hmac(crypto_ahash_tfm(rtfm)))
1056                 params.opad_needed = 1;
1057         else
1058                 params.opad_needed = 0;
1059
1060         params.sg_len = req->nbytes;
1061         params.bfr_len = req_ctx->reqlen;
1062         get_alg_config(&params.alg_prm, crypto_ahash_digestsize(rtfm));
1063         req_ctx->data_len += params.bfr_len + params.sg_len;
1064         req_ctx->result = 1;
1065         if ((req_ctx->reqlen + req->nbytes) == 0) {
1066                 create_last_hash_block(req_ctx->reqbfr, bs, req_ctx->data_len);
1067                 params.last = 0;
1068                 params.more = 1;
1069                 params.scmd1 = 0;
1070                 params.bfr_len = bs;
1071         } else {
1072                 params.scmd1 = req_ctx->data_len;
1073                 params.last = 1;
1074                 params.more = 0;
1075         }
1076
1077         skb = create_hash_wr(req, &params);
1078         if (!skb)
1079                 return -ENOMEM;
1080
1081         skb->dev = u_ctx->lldi.ports[0];
1082         set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_channel_id);
1083         chcr_send_wr(skb);
1084
1085         return -EINPROGRESS;
1086 }
1087
1088 static int chcr_ahash_digest(struct ahash_request *req)
1089 {
1090         struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1091         struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
1092         struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm));
1093         struct uld_ctx *u_ctx = NULL;
1094         struct sk_buff *skb;
1095         struct hash_wr_param params;
1096         u8  bs;
1097
1098         rtfm->init(req);
1099         bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
1100
1101         u_ctx = ULD_CTX(ctx);
1102         if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
1103                                             ctx->tx_channel_id))) {
1104                 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
1105                         return -EBUSY;
1106         }
1107
1108         if (is_hmac(crypto_ahash_tfm(rtfm)))
1109                 params.opad_needed = 1;
1110         else
1111                 params.opad_needed = 0;
1112
1113         params.last = 0;
1114         params.more = 0;
1115         params.sg_len = req->nbytes;
1116         params.bfr_len = 0;
1117         params.scmd1 = 0;
1118         get_alg_config(&params.alg_prm, crypto_ahash_digestsize(rtfm));
1119         req_ctx->result = 1;
1120         req_ctx->data_len += params.bfr_len + params.sg_len;
1121
1122         if (req->nbytes == 0) {
1123                 create_last_hash_block(req_ctx->reqbfr, bs, 0);
1124                 params.more = 1;
1125                 params.bfr_len = bs;
1126         }
1127
1128         skb = create_hash_wr(req, &params);
1129         if (!skb)
1130                 return -ENOMEM;
1131
1132         skb->dev = u_ctx->lldi.ports[0];
1133         set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_channel_id);
1134         chcr_send_wr(skb);
1135         return -EINPROGRESS;
1136 }
1137
1138 static int chcr_ahash_export(struct ahash_request *areq, void *out)
1139 {
1140         struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1141         struct chcr_ahash_req_ctx *state = out;
1142
1143         state->reqlen = req_ctx->reqlen;
1144         state->data_len = req_ctx->data_len;
1145         memcpy(state->bfr1, req_ctx->reqbfr, req_ctx->reqlen);
1146         memcpy(state->partial_hash, req_ctx->partial_hash,
1147                CHCR_HASH_MAX_DIGEST_SIZE);
1148                 return 0;
1149 }
1150
1151 static int chcr_ahash_import(struct ahash_request *areq, const void *in)
1152 {
1153         struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1154         struct chcr_ahash_req_ctx *state = (struct chcr_ahash_req_ctx *)in;
1155
1156         req_ctx->reqlen = state->reqlen;
1157         req_ctx->data_len = state->data_len;
1158         req_ctx->reqbfr = req_ctx->bfr1;
1159         req_ctx->skbfr = req_ctx->bfr2;
1160         memcpy(req_ctx->bfr1, state->bfr1, CHCR_HASH_MAX_BLOCK_SIZE_128);
1161         memcpy(req_ctx->partial_hash, state->partial_hash,
1162                CHCR_HASH_MAX_DIGEST_SIZE);
1163         return 0;
1164 }
1165
1166 static int chcr_ahash_setkey(struct crypto_ahash *tfm, const u8 *key,
1167                              unsigned int keylen)
1168 {
1169         struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
1170         struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
1171         unsigned int digestsize = crypto_ahash_digestsize(tfm);
1172         unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
1173         unsigned int i, err = 0, updated_digestsize;
1174
1175         SHASH_DESC_ON_STACK(shash, hmacctx->base_hash);
1176
1177         /* use the key to calculate the ipad and opad. ipad will sent with the
1178          * first request's data. opad will be sent with the final hash result
1179          * ipad in hmacctx->ipad and opad in hmacctx->opad location
1180          */
1181         shash->tfm = hmacctx->base_hash;
1182         shash->flags = crypto_shash_get_flags(hmacctx->base_hash);
1183         if (keylen > bs) {
1184                 err = crypto_shash_digest(shash, key, keylen,
1185                                           hmacctx->ipad);
1186                 if (err)
1187                         goto out;
1188                 keylen = digestsize;
1189         } else {
1190                 memcpy(hmacctx->ipad, key, keylen);
1191         }
1192         memset(hmacctx->ipad + keylen, 0, bs - keylen);
1193         memcpy(hmacctx->opad, hmacctx->ipad, bs);
1194
1195         for (i = 0; i < bs / sizeof(int); i++) {
1196                 *((unsigned int *)(&hmacctx->ipad) + i) ^= IPAD_DATA;
1197                 *((unsigned int *)(&hmacctx->opad) + i) ^= OPAD_DATA;
1198         }
1199
1200         updated_digestsize = digestsize;
1201         if (digestsize == SHA224_DIGEST_SIZE)
1202                 updated_digestsize = SHA256_DIGEST_SIZE;
1203         else if (digestsize == SHA384_DIGEST_SIZE)
1204                 updated_digestsize = SHA512_DIGEST_SIZE;
1205         err = chcr_compute_partial_hash(shash, hmacctx->ipad,
1206                                         hmacctx->ipad, digestsize);
1207         if (err)
1208                 goto out;
1209         chcr_change_order(hmacctx->ipad, updated_digestsize);
1210
1211         err = chcr_compute_partial_hash(shash, hmacctx->opad,
1212                                         hmacctx->opad, digestsize);
1213         if (err)
1214                 goto out;
1215         chcr_change_order(hmacctx->opad, updated_digestsize);
1216 out:
1217         return err;
1218 }
1219
1220 static int chcr_aes_xts_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
1221                                unsigned int key_len)
1222 {
1223         struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm);
1224         struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
1225         unsigned short context_size = 0;
1226
1227         if ((key_len != (AES_KEYSIZE_128 << 1)) &&
1228             (key_len != (AES_KEYSIZE_256 << 1))) {
1229                 crypto_tfm_set_flags((struct crypto_tfm *)tfm,
1230                                      CRYPTO_TFM_RES_BAD_KEY_LEN);
1231                 ablkctx->enckey_len = 0;
1232                 return -EINVAL;
1233
1234         }
1235
1236         memcpy(ablkctx->key, key, key_len);
1237         ablkctx->enckey_len = key_len;
1238         get_aes_decrypt_key(ablkctx->rrkey, ablkctx->key, key_len << 2);
1239         context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + key_len) >> 4;
1240         ablkctx->key_ctx_hdr =
1241                 FILL_KEY_CTX_HDR((key_len == AES_KEYSIZE_256) ?
1242                                  CHCR_KEYCTX_CIPHER_KEY_SIZE_128 :
1243                                  CHCR_KEYCTX_CIPHER_KEY_SIZE_256,
1244                                  CHCR_KEYCTX_NO_KEY, 1,
1245                                  0, context_size);
1246         ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_XTS;
1247         return 0;
1248 }
1249
1250 static int chcr_sha_init(struct ahash_request *areq)
1251 {
1252         struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1253         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1254         int digestsize =  crypto_ahash_digestsize(tfm);
1255
1256         req_ctx->data_len = 0;
1257         req_ctx->reqlen = 0;
1258         req_ctx->reqbfr = req_ctx->bfr1;
1259         req_ctx->skbfr = req_ctx->bfr2;
1260         req_ctx->skb = NULL;
1261         req_ctx->result = 0;
1262         copy_hash_init_values(req_ctx->partial_hash, digestsize);
1263         return 0;
1264 }
1265
1266 static int chcr_sha_cra_init(struct crypto_tfm *tfm)
1267 {
1268         crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
1269                                  sizeof(struct chcr_ahash_req_ctx));
1270         return chcr_device_init(crypto_tfm_ctx(tfm));
1271 }
1272
1273 static int chcr_hmac_init(struct ahash_request *areq)
1274 {
1275         struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1276         struct crypto_ahash *rtfm = crypto_ahash_reqtfm(areq);
1277         struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm));
1278         struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
1279         unsigned int digestsize = crypto_ahash_digestsize(rtfm);
1280         unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
1281
1282         chcr_sha_init(areq);
1283         req_ctx->data_len = bs;
1284         if (is_hmac(crypto_ahash_tfm(rtfm))) {
1285                 if (digestsize == SHA224_DIGEST_SIZE)
1286                         memcpy(req_ctx->partial_hash, hmacctx->ipad,
1287                                SHA256_DIGEST_SIZE);
1288                 else if (digestsize == SHA384_DIGEST_SIZE)
1289                         memcpy(req_ctx->partial_hash, hmacctx->ipad,
1290                                SHA512_DIGEST_SIZE);
1291                 else
1292                         memcpy(req_ctx->partial_hash, hmacctx->ipad,
1293                                digestsize);
1294         }
1295         return 0;
1296 }
1297
1298 static int chcr_hmac_cra_init(struct crypto_tfm *tfm)
1299 {
1300         struct chcr_context *ctx = crypto_tfm_ctx(tfm);
1301         struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
1302         unsigned int digestsize =
1303                 crypto_ahash_digestsize(__crypto_ahash_cast(tfm));
1304
1305         crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
1306                                  sizeof(struct chcr_ahash_req_ctx));
1307         hmacctx->base_hash = chcr_alloc_shash(digestsize);
1308         if (IS_ERR(hmacctx->base_hash))
1309                 return PTR_ERR(hmacctx->base_hash);
1310         return chcr_device_init(crypto_tfm_ctx(tfm));
1311 }
1312
1313 static void chcr_hmac_cra_exit(struct crypto_tfm *tfm)
1314 {
1315         struct chcr_context *ctx = crypto_tfm_ctx(tfm);
1316         struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
1317
1318         if (hmacctx->base_hash) {
1319                 chcr_free_shash(hmacctx->base_hash);
1320                 hmacctx->base_hash = NULL;
1321         }
1322 }
1323
1324 static int chcr_copy_assoc(struct aead_request *req,
1325                                 struct chcr_aead_ctx *ctx)
1326 {
1327         SKCIPHER_REQUEST_ON_STACK(skreq, ctx->null);
1328
1329         skcipher_request_set_tfm(skreq, ctx->null);
1330         skcipher_request_set_callback(skreq, aead_request_flags(req),
1331                         NULL, NULL);
1332         skcipher_request_set_crypt(skreq, req->src, req->dst, req->assoclen,
1333                         NULL);
1334
1335         return crypto_skcipher_encrypt(skreq);
1336 }
1337
1338 static unsigned char get_hmac(unsigned int authsize)
1339 {
1340         switch (authsize) {
1341         case ICV_8:
1342                 return CHCR_SCMD_HMAC_CTRL_PL1;
1343         case ICV_10:
1344                 return CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366;
1345         case ICV_12:
1346                 return CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
1347         }
1348         return CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
1349 }
1350
1351
1352 static struct sk_buff *create_authenc_wr(struct aead_request *req,
1353                                          unsigned short qid,
1354                                          int size,
1355                                          unsigned short op_type)
1356 {
1357         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1358         struct chcr_context *ctx = crypto_aead_ctx(tfm);
1359         struct uld_ctx *u_ctx = ULD_CTX(ctx);
1360         struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
1361         struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx);
1362         struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
1363         struct sk_buff *skb = NULL;
1364         struct chcr_wr *chcr_req;
1365         struct cpl_rx_phys_dsgl *phys_cpl;
1366         struct phys_sge_parm sg_param;
1367         struct scatterlist *src;
1368         unsigned int frags = 0, transhdr_len;
1369         unsigned int ivsize = crypto_aead_ivsize(tfm), dst_size = 0;
1370         unsigned int   kctx_len = 0;
1371         unsigned short stop_offset = 0;
1372         unsigned int  assoclen = req->assoclen;
1373         unsigned int  authsize = crypto_aead_authsize(tfm);
1374         int err = 0;
1375         int null = 0;
1376         gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
1377                 GFP_ATOMIC;
1378
1379         if (aeadctx->enckey_len == 0 || (req->cryptlen == 0))
1380                 goto err;
1381
1382         if (op_type && req->cryptlen < crypto_aead_authsize(tfm))
1383                 goto err;
1384
1385         if (sg_nents_for_len(req->src, req->assoclen + req->cryptlen) < 0)
1386                 goto err;
1387         src = scatterwalk_ffwd(reqctx->srcffwd, req->src, req->assoclen);
1388         reqctx->dst = src;
1389
1390         if (req->src != req->dst) {
1391                 err = chcr_copy_assoc(req, aeadctx);
1392                 if (err)
1393                         return ERR_PTR(err);
1394                 reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, req->dst,
1395                                                req->assoclen);
1396         }
1397         if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_NULL) {
1398                 null = 1;
1399                 assoclen = 0;
1400         }
1401         reqctx->dst_nents = sg_nents_for_len(reqctx->dst, req->cryptlen +
1402                                              (op_type ? -authsize : authsize));
1403         if (reqctx->dst_nents <= 0) {
1404                 pr_err("AUTHENC:Invalid Destination sg entries\n");
1405                 goto err;
1406         }
1407         dst_size = get_space_for_phys_dsgl(reqctx->dst_nents);
1408         kctx_len = (ntohl(KEY_CONTEXT_CTX_LEN_V(aeadctx->key_ctx_hdr)) << 4)
1409                 - sizeof(chcr_req->key_ctx);
1410         transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
1411         skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags);
1412         if (!skb)
1413                 goto err;
1414
1415         /* LLD is going to write the sge hdr. */
1416         skb_reserve(skb, sizeof(struct sge_opaque_hdr));
1417
1418         /* Write WR */
1419         chcr_req = (struct chcr_wr *) __skb_put(skb, transhdr_len);
1420         memset(chcr_req, 0, transhdr_len);
1421
1422         stop_offset = (op_type == CHCR_ENCRYPT_OP) ? 0 : authsize;
1423
1424         /*
1425          * Input order  is AAD,IV and Payload. where IV should be included as
1426          * the part of authdata. All other fields should be filled according
1427          * to the hardware spec
1428          */
1429         chcr_req->sec_cpl.op_ivinsrtofst =
1430                 FILL_SEC_CPL_OP_IVINSR(ctx->dev->rx_channel_id, 2,
1431                                        (ivsize ? (assoclen + 1) : 0));
1432         chcr_req->sec_cpl.pldlen = htonl(assoclen + ivsize + req->cryptlen);
1433         chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
1434                                         assoclen ? 1 : 0, assoclen,
1435                                         assoclen + ivsize + 1,
1436                                         (stop_offset & 0x1F0) >> 4);
1437         chcr_req->sec_cpl.cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT(
1438                                         stop_offset & 0xF,
1439                                         null ? 0 : assoclen + ivsize + 1,
1440                                         stop_offset, stop_offset);
1441         chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type,
1442                                         (op_type == CHCR_ENCRYPT_OP) ? 1 : 0,
1443                                         CHCR_SCMD_CIPHER_MODE_AES_CBC,
1444                                         actx->auth_mode, aeadctx->hmac_ctrl,
1445                                         ivsize >> 1);
1446         chcr_req->sec_cpl.ivgen_hdrlen =  FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1,
1447                                          0, 1, dst_size);
1448
1449         chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr;
1450         if (op_type == CHCR_ENCRYPT_OP)
1451                 memcpy(chcr_req->key_ctx.key, aeadctx->key,
1452                        aeadctx->enckey_len);
1453         else
1454                 memcpy(chcr_req->key_ctx.key, actx->dec_rrkey,
1455                        aeadctx->enckey_len);
1456
1457         memcpy(chcr_req->key_ctx.key + (DIV_ROUND_UP(aeadctx->enckey_len, 16) <<
1458                                         4), actx->h_iopad, kctx_len -
1459                                 (DIV_ROUND_UP(aeadctx->enckey_len, 16) << 4));
1460
1461         phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
1462         sg_param.nents = reqctx->dst_nents;
1463         sg_param.obsize = req->cryptlen + (op_type ? -authsize : authsize);
1464         sg_param.qid = qid;
1465         sg_param.align = 0;
1466         if (map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, reqctx->dst,
1467                                   &sg_param))
1468                 goto dstmap_fail;
1469
1470         skb_set_transport_header(skb, transhdr_len);
1471
1472         if (assoclen) {
1473                 /* AAD buffer in */
1474                 write_sg_to_skb(skb, &frags, req->src, assoclen);
1475
1476         }
1477         write_buffer_to_skb(skb, &frags, req->iv, ivsize);
1478         write_sg_to_skb(skb, &frags, src, req->cryptlen);
1479         create_wreq(ctx, chcr_req, req, skb, kctx_len, size, 1,
1480                    sizeof(struct cpl_rx_phys_dsgl) + dst_size);
1481         reqctx->skb = skb;
1482         skb_get(skb);
1483
1484         return skb;
1485 dstmap_fail:
1486         /* ivmap_fail: */
1487         kfree_skb(skb);
1488 err:
1489         return ERR_PTR(-EINVAL);
1490 }
1491
1492 static void aes_gcm_empty_pld_pad(struct scatterlist *sg,
1493                                   unsigned short offset)
1494 {
1495         struct page *spage;
1496         unsigned char *addr;
1497
1498         spage = sg_page(sg);
1499         get_page(spage); /* so that it is not freed by NIC */
1500 #ifdef KMAP_ATOMIC_ARGS
1501         addr = kmap_atomic(spage, KM_SOFTIRQ0);
1502 #else
1503         addr = kmap_atomic(spage);
1504 #endif
1505         memset(addr + sg->offset, 0, offset + 1);
1506
1507         kunmap_atomic(addr);
1508 }
1509
1510 static int set_msg_len(u8 *block, unsigned int msglen, int csize)
1511 {
1512         __be32 data;
1513
1514         memset(block, 0, csize);
1515         block += csize;
1516
1517         if (csize >= 4)
1518                 csize = 4;
1519         else if (msglen > (unsigned int)(1 << (8 * csize)))
1520                 return -EOVERFLOW;
1521
1522         data = cpu_to_be32(msglen);
1523         memcpy(block - csize, (u8 *)&data + 4 - csize, csize);
1524
1525         return 0;
1526 }
1527
1528 static void generate_b0(struct aead_request *req,
1529                         struct chcr_aead_ctx *aeadctx,
1530                         unsigned short op_type)
1531 {
1532         unsigned int l, lp, m;
1533         int rc;
1534         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1535         struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
1536         u8 *b0 = reqctx->scratch_pad;
1537
1538         m = crypto_aead_authsize(aead);
1539
1540         memcpy(b0, reqctx->iv, 16);
1541
1542         lp = b0[0];
1543         l = lp + 1;
1544
1545         /* set m, bits 3-5 */
1546         *b0 |= (8 * ((m - 2) / 2));
1547
1548         /* set adata, bit 6, if associated data is used */
1549         if (req->assoclen)
1550                 *b0 |= 64;
1551         rc = set_msg_len(b0 + 16 - l,
1552                          (op_type == CHCR_DECRYPT_OP) ?
1553                          req->cryptlen - m : req->cryptlen, l);
1554 }
1555
1556 static inline int crypto_ccm_check_iv(const u8 *iv)
1557 {
1558         /* 2 <= L <= 8, so 1 <= L' <= 7. */
1559         if (iv[0] < 1 || iv[0] > 7)
1560                 return -EINVAL;
1561
1562         return 0;
1563 }
1564
1565 static int ccm_format_packet(struct aead_request *req,
1566                              struct chcr_aead_ctx *aeadctx,
1567                              unsigned int sub_type,
1568                              unsigned short op_type)
1569 {
1570         struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
1571         int rc = 0;
1572
1573         if (req->assoclen > T5_MAX_AAD_SIZE) {
1574                 pr_err("CCM: Unsupported AAD data. It should be < %d\n",
1575                        T5_MAX_AAD_SIZE);
1576                 return -EINVAL;
1577         }
1578         if (sub_type == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) {
1579                 reqctx->iv[0] = 3;
1580                 memcpy(reqctx->iv + 1, &aeadctx->salt[0], 3);
1581                 memcpy(reqctx->iv + 4, req->iv, 8);
1582                 memset(reqctx->iv + 12, 0, 4);
1583                 *((unsigned short *)(reqctx->scratch_pad + 16)) =
1584                         htons(req->assoclen - 8);
1585         } else {
1586                 memcpy(reqctx->iv, req->iv, 16);
1587                 *((unsigned short *)(reqctx->scratch_pad + 16)) =
1588                         htons(req->assoclen);
1589         }
1590         generate_b0(req, aeadctx, op_type);
1591         /* zero the ctr value */
1592         memset(reqctx->iv + 15 - reqctx->iv[0], 0, reqctx->iv[0] + 1);
1593         return rc;
1594 }
1595
1596 static void fill_sec_cpl_for_aead(struct cpl_tx_sec_pdu *sec_cpl,
1597                                   unsigned int dst_size,
1598                                   struct aead_request *req,
1599                                   unsigned short op_type,
1600                                           struct chcr_context *chcrctx)
1601 {
1602         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1603         unsigned int ivsize = AES_BLOCK_SIZE;
1604         unsigned int cipher_mode = CHCR_SCMD_CIPHER_MODE_AES_CCM;
1605         unsigned int mac_mode = CHCR_SCMD_AUTH_MODE_CBCMAC;
1606         unsigned int c_id = chcrctx->dev->rx_channel_id;
1607         unsigned int ccm_xtra;
1608         unsigned char tag_offset = 0, auth_offset = 0;
1609         unsigned char hmac_ctrl = get_hmac(crypto_aead_authsize(tfm));
1610         unsigned int assoclen;
1611
1612         if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309)
1613                 assoclen = req->assoclen - 8;
1614         else
1615                 assoclen = req->assoclen;
1616         ccm_xtra = CCM_B0_SIZE +
1617                 ((assoclen) ? CCM_AAD_FIELD_SIZE : 0);
1618
1619         auth_offset = req->cryptlen ?
1620                 (assoclen + ivsize + 1 + ccm_xtra) : 0;
1621         if (op_type == CHCR_DECRYPT_OP) {
1622                 if (crypto_aead_authsize(tfm) != req->cryptlen)
1623                         tag_offset = crypto_aead_authsize(tfm);
1624                 else
1625                         auth_offset = 0;
1626         }
1627
1628
1629         sec_cpl->op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR(c_id,
1630                                          2, (ivsize ?  (assoclen + 1) :  0) +
1631                                          ccm_xtra);
1632         sec_cpl->pldlen =
1633                 htonl(assoclen + ivsize + req->cryptlen + ccm_xtra);
1634         /* For CCM there wil be b0 always. So AAD start will be 1 always */
1635         sec_cpl->aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
1636                                         1, assoclen + ccm_xtra, assoclen
1637                                         + ivsize + 1 + ccm_xtra, 0);
1638
1639         sec_cpl->cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT(0,
1640                                         auth_offset, tag_offset,
1641                                         (op_type == CHCR_ENCRYPT_OP) ? 0 :
1642                                         crypto_aead_authsize(tfm));
1643         sec_cpl->seqno_numivs =  FILL_SEC_CPL_SCMD0_SEQNO(op_type,
1644                                         (op_type == CHCR_ENCRYPT_OP) ? 0 : 1,
1645                                         cipher_mode, mac_mode, hmac_ctrl,
1646                                         ivsize >> 1);
1647
1648         sec_cpl->ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, 0,
1649                                         1, dst_size);
1650 }
1651
1652 int aead_ccm_validate_input(unsigned short op_type,
1653                             struct aead_request *req,
1654                             struct chcr_aead_ctx *aeadctx,
1655                             unsigned int sub_type)
1656 {
1657         if (sub_type != CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) {
1658                 if (crypto_ccm_check_iv(req->iv)) {
1659                         pr_err("CCM: IV check fails\n");
1660                         return -EINVAL;
1661                 }
1662         } else {
1663                 if (req->assoclen != 16 && req->assoclen != 20) {
1664                         pr_err("RFC4309: Invalid AAD length %d\n",
1665                                req->assoclen);
1666                         return -EINVAL;
1667                 }
1668         }
1669         if (aeadctx->enckey_len == 0) {
1670                 pr_err("CCM: Encryption key not set\n");
1671                 return -EINVAL;
1672         }
1673         return 0;
1674 }
1675
1676 unsigned int fill_aead_req_fields(struct sk_buff *skb,
1677                                   struct aead_request *req,
1678                                   struct scatterlist *src,
1679                                   unsigned int ivsize,
1680                                   struct chcr_aead_ctx *aeadctx)
1681 {
1682         unsigned int frags = 0;
1683         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1684         struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
1685         /* b0 and aad length(if available) */
1686
1687         write_buffer_to_skb(skb, &frags, reqctx->scratch_pad, CCM_B0_SIZE +
1688                                 (req->assoclen ?  CCM_AAD_FIELD_SIZE : 0));
1689         if (req->assoclen) {
1690                 if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309)
1691                         write_sg_to_skb(skb, &frags, req->src,
1692                                         req->assoclen - 8);
1693                 else
1694                         write_sg_to_skb(skb, &frags, req->src, req->assoclen);
1695         }
1696         write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize);
1697         if (req->cryptlen)
1698                 write_sg_to_skb(skb, &frags, src, req->cryptlen);
1699
1700         return frags;
1701 }
1702
1703 static struct sk_buff *create_aead_ccm_wr(struct aead_request *req,
1704                                           unsigned short qid,
1705                                           int size,
1706                                           unsigned short op_type)
1707 {
1708         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1709         struct chcr_context *ctx = crypto_aead_ctx(tfm);
1710         struct uld_ctx *u_ctx = ULD_CTX(ctx);
1711         struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
1712         struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
1713         struct sk_buff *skb = NULL;
1714         struct chcr_wr *chcr_req;
1715         struct cpl_rx_phys_dsgl *phys_cpl;
1716         struct phys_sge_parm sg_param;
1717         struct scatterlist *src;
1718         unsigned int frags = 0, transhdr_len, ivsize = AES_BLOCK_SIZE;
1719         unsigned int dst_size = 0, kctx_len;
1720         unsigned int sub_type;
1721         unsigned int authsize = crypto_aead_authsize(tfm);
1722         int err = 0;
1723         gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
1724                 GFP_ATOMIC;
1725
1726
1727         if (op_type && req->cryptlen < crypto_aead_authsize(tfm))
1728                 goto err;
1729
1730         if (sg_nents_for_len(req->src, req->assoclen + req->cryptlen) < 0)
1731                 goto err;
1732         sub_type = get_aead_subtype(tfm);
1733         src = scatterwalk_ffwd(reqctx->srcffwd, req->src, req->assoclen);
1734         reqctx->dst = src;
1735
1736         if (req->src != req->dst) {
1737                 err = chcr_copy_assoc(req, aeadctx);
1738                 if (err) {
1739                         pr_err("AAD copy to destination buffer fails\n");
1740                         return ERR_PTR(err);
1741                 }
1742                 reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, req->dst,
1743                                                req->assoclen);
1744         }
1745         reqctx->dst_nents = sg_nents_for_len(reqctx->dst, req->cryptlen +
1746                                              (op_type ? -authsize : authsize));
1747         if (reqctx->dst_nents <= 0) {
1748                 pr_err("CCM:Invalid Destination sg entries\n");
1749                 goto err;
1750         }
1751
1752
1753         if (aead_ccm_validate_input(op_type, req, aeadctx, sub_type))
1754                 goto err;
1755
1756         dst_size = get_space_for_phys_dsgl(reqctx->dst_nents);
1757         kctx_len = ((DIV_ROUND_UP(aeadctx->enckey_len, 16)) << 4) * 2;
1758         transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
1759         skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)),  flags);
1760
1761         if (!skb)
1762                 goto err;
1763
1764         skb_reserve(skb, sizeof(struct sge_opaque_hdr));
1765
1766         chcr_req = (struct chcr_wr *) __skb_put(skb, transhdr_len);
1767         memset(chcr_req, 0, transhdr_len);
1768
1769         fill_sec_cpl_for_aead(&chcr_req->sec_cpl, dst_size, req, op_type, ctx);
1770
1771         chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr;
1772         memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len);
1773         memcpy(chcr_req->key_ctx.key + (DIV_ROUND_UP(aeadctx->enckey_len, 16) *
1774                                         16), aeadctx->key, aeadctx->enckey_len);
1775
1776         phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
1777         if (ccm_format_packet(req, aeadctx, sub_type, op_type))
1778                 goto dstmap_fail;
1779
1780         sg_param.nents = reqctx->dst_nents;
1781         sg_param.obsize = req->cryptlen + (op_type ? -authsize : authsize);
1782         sg_param.qid = qid;
1783         sg_param.align = 0;
1784         if (map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, reqctx->dst,
1785                                   &sg_param))
1786                 goto dstmap_fail;
1787
1788         skb_set_transport_header(skb, transhdr_len);
1789         frags = fill_aead_req_fields(skb, req, src, ivsize, aeadctx);
1790         create_wreq(ctx, chcr_req, req, skb, kctx_len, 0, 1,
1791                     sizeof(struct cpl_rx_phys_dsgl) + dst_size);
1792         reqctx->skb = skb;
1793         skb_get(skb);
1794         return skb;
1795 dstmap_fail:
1796         kfree_skb(skb);
1797         skb = NULL;
1798 err:
1799         return ERR_PTR(-EINVAL);
1800 }
1801
1802 static struct sk_buff *create_gcm_wr(struct aead_request *req,
1803                                      unsigned short qid,
1804                                      int size,
1805                                      unsigned short op_type)
1806 {
1807         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1808         struct chcr_context *ctx = crypto_aead_ctx(tfm);
1809         struct uld_ctx *u_ctx = ULD_CTX(ctx);
1810         struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
1811         struct chcr_aead_reqctx  *reqctx = aead_request_ctx(req);
1812         struct sk_buff *skb = NULL;
1813         struct chcr_wr *chcr_req;
1814         struct cpl_rx_phys_dsgl *phys_cpl;
1815         struct phys_sge_parm sg_param;
1816         struct scatterlist *src;
1817         unsigned int frags = 0, transhdr_len;
1818         unsigned int ivsize = AES_BLOCK_SIZE;
1819         unsigned int dst_size = 0, kctx_len;
1820         unsigned char tag_offset = 0;
1821         unsigned int crypt_len = 0;
1822         unsigned int authsize = crypto_aead_authsize(tfm);
1823         unsigned char hmac_ctrl = get_hmac(authsize);
1824         int err = 0;
1825         gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
1826                 GFP_ATOMIC;
1827
1828         /* validate key size */
1829         if (aeadctx->enckey_len == 0)
1830                 goto err;
1831
1832         if (op_type && req->cryptlen < crypto_aead_authsize(tfm))
1833                 goto err;
1834
1835         if (sg_nents_for_len(req->src, req->assoclen + req->cryptlen) < 0)
1836                 goto err;
1837
1838         src = scatterwalk_ffwd(reqctx->srcffwd, req->src, req->assoclen);
1839         reqctx->dst = src;
1840         if (req->src != req->dst) {
1841                 err = chcr_copy_assoc(req, aeadctx);
1842                 if (err)
1843                         return  ERR_PTR(err);
1844                 reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, req->dst,
1845                                                req->assoclen);
1846         }
1847
1848         if (!req->cryptlen)
1849                 /* null-payload is not supported in the hardware.
1850                  * software is sending block size
1851                  */
1852                 crypt_len = AES_BLOCK_SIZE;
1853         else
1854                 crypt_len = req->cryptlen;
1855         reqctx->dst_nents = sg_nents_for_len(reqctx->dst, req->cryptlen +
1856                                              (op_type ? -authsize : authsize));
1857         if (reqctx->dst_nents <= 0) {
1858                 pr_err("GCM:Invalid Destination sg entries\n");
1859                 goto err;
1860         }
1861
1862
1863         dst_size = get_space_for_phys_dsgl(reqctx->dst_nents);
1864         kctx_len = ((DIV_ROUND_UP(aeadctx->enckey_len, 16)) << 4) +
1865                 AEAD_H_SIZE;
1866         transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
1867         skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags);
1868         if (!skb)
1869                 goto err;
1870
1871         /* NIC driver is going to write the sge hdr. */
1872         skb_reserve(skb, sizeof(struct sge_opaque_hdr));
1873
1874         chcr_req = (struct chcr_wr *)__skb_put(skb, transhdr_len);
1875         memset(chcr_req, 0, transhdr_len);
1876
1877         if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106)
1878                 req->assoclen -= 8;
1879
1880         tag_offset = (op_type == CHCR_ENCRYPT_OP) ? 0 : authsize;
1881         chcr_req->sec_cpl.op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR(
1882                                         ctx->dev->rx_channel_id, 2, (ivsize ?
1883                                         (req->assoclen + 1) : 0));
1884         chcr_req->sec_cpl.pldlen = htonl(req->assoclen + ivsize + crypt_len);
1885         chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
1886                                         req->assoclen ? 1 : 0, req->assoclen,
1887                                         req->assoclen + ivsize + 1, 0);
1888         if (req->cryptlen) {
1889                 chcr_req->sec_cpl.cipherstop_lo_authinsert =
1890                         FILL_SEC_CPL_AUTHINSERT(0, req->assoclen + ivsize + 1,
1891                                                 tag_offset, tag_offset);
1892                 chcr_req->sec_cpl.seqno_numivs =
1893                         FILL_SEC_CPL_SCMD0_SEQNO(op_type, (op_type ==
1894                                         CHCR_ENCRYPT_OP) ? 1 : 0,
1895                                         CHCR_SCMD_CIPHER_MODE_AES_GCM,
1896                                         CHCR_SCMD_AUTH_MODE_GHASH, hmac_ctrl,
1897                                         ivsize >> 1);
1898         } else {
1899                 chcr_req->sec_cpl.cipherstop_lo_authinsert =
1900                         FILL_SEC_CPL_AUTHINSERT(0, 0, 0, 0);
1901                 chcr_req->sec_cpl.seqno_numivs =
1902                         FILL_SEC_CPL_SCMD0_SEQNO(op_type,
1903                                         (op_type ==  CHCR_ENCRYPT_OP) ?
1904                                         1 : 0, CHCR_SCMD_CIPHER_MODE_AES_CBC,
1905                                         0, 0, ivsize >> 1);
1906         }
1907         chcr_req->sec_cpl.ivgen_hdrlen =  FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1,
1908                                         0, 1, dst_size);
1909         chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr;
1910         memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len);
1911         memcpy(chcr_req->key_ctx.key + (DIV_ROUND_UP(aeadctx->enckey_len, 16) *
1912                                 16), GCM_CTX(aeadctx)->ghash_h, AEAD_H_SIZE);
1913
1914         /* prepare a 16 byte iv */
1915         /* S   A   L  T |  IV | 0x00000001 */
1916         if (get_aead_subtype(tfm) ==
1917             CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) {
1918                 memcpy(reqctx->iv, aeadctx->salt, 4);
1919                 memcpy(reqctx->iv + 4, req->iv, 8);
1920         } else {
1921                 memcpy(reqctx->iv, req->iv, 12);
1922         }
1923         *((unsigned int *)(reqctx->iv + 12)) = htonl(0x01);
1924
1925         phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
1926         sg_param.nents = reqctx->dst_nents;
1927         sg_param.obsize = req->cryptlen + (op_type ? -authsize : authsize);
1928         sg_param.qid = qid;
1929         sg_param.align = 0;
1930         if (map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, reqctx->dst,
1931                                   &sg_param))
1932                 goto dstmap_fail;
1933
1934         skb_set_transport_header(skb, transhdr_len);
1935
1936         write_sg_to_skb(skb, &frags, req->src, req->assoclen);
1937
1938         write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize);
1939
1940         if (req->cryptlen) {
1941                 write_sg_to_skb(skb, &frags, src, req->cryptlen);
1942         } else {
1943                 aes_gcm_empty_pld_pad(req->dst, authsize - 1);
1944                 write_sg_to_skb(skb, &frags, reqctx->dst, crypt_len);
1945
1946         }
1947
1948         create_wreq(ctx, chcr_req, req, skb, kctx_len, size, 1,
1949                         sizeof(struct cpl_rx_phys_dsgl) + dst_size);
1950         reqctx->skb = skb;
1951         skb_get(skb);
1952         return skb;
1953
1954 dstmap_fail:
1955         /* ivmap_fail: */
1956         kfree_skb(skb);
1957         skb = NULL;
1958 err:
1959         return skb;
1960 }
1961
1962
1963
1964 static int chcr_aead_cra_init(struct crypto_aead *tfm)
1965 {
1966         struct chcr_context *ctx = crypto_aead_ctx(tfm);
1967         struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
1968
1969         crypto_aead_set_reqsize(tfm, sizeof(struct chcr_aead_reqctx));
1970         aeadctx->null = crypto_get_default_null_skcipher();
1971         if (IS_ERR(aeadctx->null))
1972                 return PTR_ERR(aeadctx->null);
1973         return chcr_device_init(ctx);
1974 }
1975
1976 static void chcr_aead_cra_exit(struct crypto_aead *tfm)
1977 {
1978         crypto_put_default_null_skcipher();
1979 }
1980
1981 static int chcr_authenc_null_setauthsize(struct crypto_aead *tfm,
1982                                         unsigned int authsize)
1983 {
1984         struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm));
1985
1986         aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NOP;
1987         aeadctx->mayverify = VERIFY_HW;
1988         return 0;
1989 }
1990 static int chcr_authenc_setauthsize(struct crypto_aead *tfm,
1991                                     unsigned int authsize)
1992 {
1993         struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm));
1994         u32 maxauth = crypto_aead_maxauthsize(tfm);
1995
1996         /*SHA1 authsize in ipsec is 12 instead of 10 i.e maxauthsize / 2 is not
1997          * true for sha1. authsize == 12 condition should be before
1998          * authsize == (maxauth >> 1)
1999          */
2000         if (authsize == ICV_4) {
2001                 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1;
2002                 aeadctx->mayverify = VERIFY_HW;
2003         } else if (authsize == ICV_6) {
2004                 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL2;
2005                 aeadctx->mayverify = VERIFY_HW;
2006         } else if (authsize == ICV_10) {
2007                 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366;
2008                 aeadctx->mayverify = VERIFY_HW;
2009         } else if (authsize == ICV_12) {
2010                 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
2011                 aeadctx->mayverify = VERIFY_HW;
2012         } else if (authsize == ICV_14) {
2013                 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3;
2014                 aeadctx->mayverify = VERIFY_HW;
2015         } else if (authsize == (maxauth >> 1)) {
2016                 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
2017                 aeadctx->mayverify = VERIFY_HW;
2018         } else if (authsize == maxauth) {
2019                 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
2020                 aeadctx->mayverify = VERIFY_HW;
2021         } else {
2022                 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
2023                 aeadctx->mayverify = VERIFY_SW;
2024         }
2025         return 0;
2026 }
2027
2028
2029 static int chcr_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
2030 {
2031         struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm));
2032
2033         switch (authsize) {
2034         case ICV_4:
2035                 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1;
2036                 aeadctx->mayverify = VERIFY_HW;
2037                 break;
2038         case ICV_8:
2039                 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
2040                 aeadctx->mayverify = VERIFY_HW;
2041                 break;
2042         case ICV_12:
2043                  aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
2044                  aeadctx->mayverify = VERIFY_HW;
2045                 break;
2046         case ICV_14:
2047                  aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3;
2048                  aeadctx->mayverify = VERIFY_HW;
2049                 break;
2050         case ICV_16:
2051                 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
2052                 aeadctx->mayverify = VERIFY_HW;
2053                 break;
2054         case ICV_13:
2055         case ICV_15:
2056                 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
2057                 aeadctx->mayverify = VERIFY_SW;
2058                 break;
2059         default:
2060
2061                   crypto_tfm_set_flags((struct crypto_tfm *) tfm,
2062                         CRYPTO_TFM_RES_BAD_KEY_LEN);
2063                 return -EINVAL;
2064         }
2065         return 0;
2066 }
2067
2068 static int chcr_4106_4309_setauthsize(struct crypto_aead *tfm,
2069                                           unsigned int authsize)
2070 {
2071         struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm));
2072
2073         switch (authsize) {
2074         case ICV_8:
2075                 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
2076                 aeadctx->mayverify = VERIFY_HW;
2077                 break;
2078         case ICV_12:
2079                 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
2080                 aeadctx->mayverify = VERIFY_HW;
2081                 break;
2082         case ICV_16:
2083                 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
2084                 aeadctx->mayverify = VERIFY_HW;
2085                 break;
2086         default:
2087                 crypto_tfm_set_flags((struct crypto_tfm *)tfm,
2088                                      CRYPTO_TFM_RES_BAD_KEY_LEN);
2089                 return -EINVAL;
2090         }
2091         return 0;
2092 }
2093
2094 static int chcr_ccm_setauthsize(struct crypto_aead *tfm,
2095                                 unsigned int authsize)
2096 {
2097         struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm));
2098
2099         switch (authsize) {
2100         case ICV_4:
2101                 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1;
2102                 aeadctx->mayverify = VERIFY_HW;
2103                 break;
2104         case ICV_6:
2105                 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL2;
2106                 aeadctx->mayverify = VERIFY_HW;
2107                 break;
2108         case ICV_8:
2109                 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
2110                 aeadctx->mayverify = VERIFY_HW;
2111                 break;
2112         case ICV_10:
2113                 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366;
2114                 aeadctx->mayverify = VERIFY_HW;
2115                 break;
2116         case ICV_12:
2117                 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
2118                 aeadctx->mayverify = VERIFY_HW;
2119                 break;
2120         case ICV_14:
2121                 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3;
2122                 aeadctx->mayverify = VERIFY_HW;
2123                 break;
2124         case ICV_16:
2125                 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
2126                 aeadctx->mayverify = VERIFY_HW;
2127                 break;
2128         default:
2129                 crypto_tfm_set_flags((struct crypto_tfm *)tfm,
2130                                      CRYPTO_TFM_RES_BAD_KEY_LEN);
2131                 return -EINVAL;
2132         }
2133         return 0;
2134 }
2135
2136 static int chcr_aead_ccm_setkey(struct crypto_aead *aead,
2137                                 const u8 *key,
2138                                 unsigned int keylen)
2139 {
2140         struct chcr_context *ctx = crypto_aead_ctx(aead);
2141         struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
2142         unsigned char ck_size, mk_size;
2143         int key_ctx_size = 0;
2144
2145         memcpy(aeadctx->key, key, keylen);
2146         aeadctx->enckey_len = keylen;
2147         key_ctx_size = sizeof(struct _key_ctx) +
2148                 ((DIV_ROUND_UP(keylen, 16)) << 4)  * 2;
2149         if (keylen == AES_KEYSIZE_128) {
2150                 mk_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
2151                 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
2152         } else if (keylen == AES_KEYSIZE_192) {
2153                 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
2154                 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_192;
2155         } else if (keylen == AES_KEYSIZE_256) {
2156                 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
2157                 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
2158         } else {
2159                 crypto_tfm_set_flags((struct crypto_tfm *)aead,
2160                                      CRYPTO_TFM_RES_BAD_KEY_LEN);
2161                 aeadctx->enckey_len = 0;
2162                 return  -EINVAL;
2163         }
2164         aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, mk_size, 0, 0,
2165                                                 key_ctx_size >> 4);
2166         return 0;
2167 }
2168
2169 static int chcr_aead_rfc4309_setkey(struct crypto_aead *aead, const u8 *key,
2170                                     unsigned int keylen)
2171 {
2172         struct chcr_context *ctx = crypto_aead_ctx(aead);
2173          struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
2174
2175         if (keylen < 3) {
2176                 crypto_tfm_set_flags((struct crypto_tfm *)aead,
2177                                      CRYPTO_TFM_RES_BAD_KEY_LEN);
2178                 aeadctx->enckey_len = 0;
2179                 return  -EINVAL;
2180         }
2181         keylen -= 3;
2182         memcpy(aeadctx->salt, key + keylen, 3);
2183         return chcr_aead_ccm_setkey(aead, key, keylen);
2184 }
2185
2186 static int chcr_gcm_setkey(struct crypto_aead *aead, const u8 *key,
2187                            unsigned int keylen)
2188 {
2189         struct chcr_context *ctx = crypto_aead_ctx(aead);
2190         struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
2191         struct chcr_gcm_ctx *gctx = GCM_CTX(aeadctx);
2192         struct crypto_cipher *cipher;
2193         unsigned int ck_size;
2194         int ret = 0, key_ctx_size = 0;
2195
2196         if (get_aead_subtype(aead) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106 &&
2197             keylen > 3) {
2198                 keylen -= 4;  /* nonce/salt is present in the last 4 bytes */
2199                 memcpy(aeadctx->salt, key + keylen, 4);
2200         }
2201         if (keylen == AES_KEYSIZE_128) {
2202                 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
2203         } else if (keylen == AES_KEYSIZE_192) {
2204                 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
2205         } else if (keylen == AES_KEYSIZE_256) {
2206                 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
2207         } else {
2208                 crypto_tfm_set_flags((struct crypto_tfm *)aead,
2209                                      CRYPTO_TFM_RES_BAD_KEY_LEN);
2210                 aeadctx->enckey_len = 0;
2211                 pr_err("GCM: Invalid key length %d", keylen);
2212                 ret = -EINVAL;
2213                 goto out;
2214         }
2215
2216         memcpy(aeadctx->key, key, keylen);
2217         aeadctx->enckey_len = keylen;
2218         key_ctx_size = sizeof(struct _key_ctx) +
2219                 ((DIV_ROUND_UP(keylen, 16)) << 4) +
2220                 AEAD_H_SIZE;
2221                 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size,
2222                                                 CHCR_KEYCTX_MAC_KEY_SIZE_128,
2223                                                 0, 0,
2224                                                 key_ctx_size >> 4);
2225         /* Calculate the H = CIPH(K, 0 repeated 16 times).
2226          * It will go in key context
2227          */
2228         cipher = crypto_alloc_cipher("aes-generic", 0, 0);
2229         if (IS_ERR(cipher)) {
2230                 aeadctx->enckey_len = 0;
2231                 ret = -ENOMEM;
2232                 goto out;
2233         }
2234
2235         ret = crypto_cipher_setkey(cipher, key, keylen);
2236         if (ret) {
2237                 aeadctx->enckey_len = 0;
2238                 goto out1;
2239         }
2240         memset(gctx->ghash_h, 0, AEAD_H_SIZE);
2241         crypto_cipher_encrypt_one(cipher, gctx->ghash_h, gctx->ghash_h);
2242
2243 out1:
2244         crypto_free_cipher(cipher);
2245 out:
2246         return ret;
2247 }
2248
2249 static int chcr_authenc_setkey(struct crypto_aead *authenc, const u8 *key,
2250                                    unsigned int keylen)
2251 {
2252         struct chcr_context *ctx = crypto_aead_ctx(authenc);
2253         struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
2254         struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx);
2255         /* it contains auth and cipher key both*/
2256         struct crypto_authenc_keys keys;
2257         unsigned int bs;
2258         unsigned int max_authsize = crypto_aead_alg(authenc)->maxauthsize;
2259         int err = 0, i, key_ctx_len = 0;
2260         unsigned char ck_size = 0;
2261         unsigned char pad[CHCR_HASH_MAX_BLOCK_SIZE_128] = { 0 };
2262         struct crypto_shash *base_hash = NULL;
2263         struct algo_param param;
2264         int align;
2265         u8 *o_ptr = NULL;
2266
2267         if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) {
2268                 crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN);
2269                 goto out;
2270         }
2271
2272         if (get_alg_config(&param, max_authsize)) {
2273                 pr_err("chcr : Unsupported digest size\n");
2274                 goto out;
2275         }
2276         if (keys.enckeylen == AES_KEYSIZE_128) {
2277                 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
2278         } else if (keys.enckeylen == AES_KEYSIZE_192) {
2279                 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
2280         } else if (keys.enckeylen == AES_KEYSIZE_256) {
2281                 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
2282         } else {
2283                 pr_err("chcr : Unsupported cipher key\n");
2284                 goto out;
2285         }
2286
2287         /* Copy only encryption key. We use authkey to generate h(ipad) and
2288          * h(opad) so authkey is not needed again. authkeylen size have the
2289          * size of the hash digest size.
2290          */
2291         memcpy(aeadctx->key, keys.enckey, keys.enckeylen);
2292         aeadctx->enckey_len = keys.enckeylen;
2293         get_aes_decrypt_key(actx->dec_rrkey, aeadctx->key,
2294                             aeadctx->enckey_len << 3);
2295
2296         base_hash  = chcr_alloc_shash(max_authsize);
2297         if (IS_ERR(base_hash)) {
2298                 pr_err("chcr : Base driver cannot be loaded\n");
2299                 goto out;
2300         }
2301         {
2302                 SHASH_DESC_ON_STACK(shash, base_hash);
2303                 shash->tfm = base_hash;
2304                 shash->flags = crypto_shash_get_flags(base_hash);
2305                 bs = crypto_shash_blocksize(base_hash);
2306                 align = KEYCTX_ALIGN_PAD(max_authsize);
2307                 o_ptr =  actx->h_iopad + param.result_size + align;
2308
2309                 if (keys.authkeylen > bs) {
2310                         err = crypto_shash_digest(shash, keys.authkey,
2311                                                   keys.authkeylen,
2312                                                   o_ptr);
2313                         if (err) {
2314                                 pr_err("chcr : Base driver cannot be loaded\n");
2315                                 goto out;
2316                         }
2317                         keys.authkeylen = max_authsize;
2318                 } else
2319                         memcpy(o_ptr, keys.authkey, keys.authkeylen);
2320
2321                 /* Compute the ipad-digest*/
2322                 memset(pad + keys.authkeylen, 0, bs - keys.authkeylen);
2323                 memcpy(pad, o_ptr, keys.authkeylen);
2324                 for (i = 0; i < bs >> 2; i++)
2325                         *((unsigned int *)pad + i) ^= IPAD_DATA;
2326
2327                 if (chcr_compute_partial_hash(shash, pad, actx->h_iopad,
2328                                               max_authsize))
2329                         goto out;
2330                 /* Compute the opad-digest */
2331                 memset(pad + keys.authkeylen, 0, bs - keys.authkeylen);
2332                 memcpy(pad, o_ptr, keys.authkeylen);
2333                 for (i = 0; i < bs >> 2; i++)
2334                         *((unsigned int *)pad + i) ^= OPAD_DATA;
2335
2336                 if (chcr_compute_partial_hash(shash, pad, o_ptr, max_authsize))
2337                         goto out;
2338
2339                 /* convert the ipad and opad digest to network order */
2340                 chcr_change_order(actx->h_iopad, param.result_size);
2341                 chcr_change_order(o_ptr, param.result_size);
2342                 key_ctx_len = sizeof(struct _key_ctx) +
2343                         ((DIV_ROUND_UP(keys.enckeylen, 16)) << 4) +
2344                         (param.result_size + align) * 2;
2345                 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, param.mk_size,
2346                                                 0, 1, key_ctx_len >> 4);
2347                 actx->auth_mode = param.auth_mode;
2348                 chcr_free_shash(base_hash);
2349
2350                 return 0;
2351         }
2352 out:
2353         aeadctx->enckey_len = 0;
2354         if (base_hash)
2355                 chcr_free_shash(base_hash);
2356         return -EINVAL;
2357 }
2358
2359 static int chcr_aead_digest_null_setkey(struct crypto_aead *authenc,
2360                                         const u8 *key, unsigned int keylen)
2361 {
2362         struct chcr_context *ctx = crypto_aead_ctx(authenc);
2363         struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
2364         struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx);
2365         struct crypto_authenc_keys keys;
2366
2367         /* it contains auth and cipher key both*/
2368         int key_ctx_len = 0;
2369         unsigned char ck_size = 0;
2370
2371         if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) {
2372                 crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN);
2373                 goto out;
2374         }
2375         if (keys.enckeylen == AES_KEYSIZE_128) {
2376                 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
2377         } else if (keys.enckeylen == AES_KEYSIZE_192) {
2378                 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
2379         } else if (keys.enckeylen == AES_KEYSIZE_256) {
2380                 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
2381         } else {
2382                 pr_err("chcr : Unsupported cipher key\n");
2383                 goto out;
2384         }
2385         memcpy(aeadctx->key, keys.enckey, keys.enckeylen);
2386         aeadctx->enckey_len = keys.enckeylen;
2387         get_aes_decrypt_key(actx->dec_rrkey, aeadctx->key,
2388                                     aeadctx->enckey_len << 3);
2389         key_ctx_len =  sizeof(struct _key_ctx)
2390                 + ((DIV_ROUND_UP(keys.enckeylen, 16)) << 4);
2391
2392         aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY, 0,
2393                                                 0, key_ctx_len >> 4);
2394         actx->auth_mode = CHCR_SCMD_AUTH_MODE_NOP;
2395         return 0;
2396 out:
2397         aeadctx->enckey_len = 0;
2398         return -EINVAL;
2399 }
2400 static int chcr_aead_encrypt(struct aead_request *req)
2401 {
2402         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2403         struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2404
2405         reqctx->verify = VERIFY_HW;
2406
2407         switch (get_aead_subtype(tfm)) {
2408         case CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC:
2409         case CRYPTO_ALG_SUB_TYPE_AEAD_NULL:
2410                 return chcr_aead_op(req, CHCR_ENCRYPT_OP, 0,
2411                                     create_authenc_wr);
2412         case CRYPTO_ALG_SUB_TYPE_AEAD_CCM:
2413         case CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309:
2414                 return chcr_aead_op(req, CHCR_ENCRYPT_OP, 0,
2415                                     create_aead_ccm_wr);
2416         default:
2417                 return chcr_aead_op(req, CHCR_ENCRYPT_OP, 0,
2418                                     create_gcm_wr);
2419         }
2420 }
2421
2422 static int chcr_aead_decrypt(struct aead_request *req)
2423 {
2424         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2425         struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm));
2426         struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2427         int size;
2428
2429         if (aeadctx->mayverify == VERIFY_SW) {
2430                 size = crypto_aead_maxauthsize(tfm);
2431                 reqctx->verify = VERIFY_SW;
2432         } else {
2433                 size = 0;
2434                 reqctx->verify = VERIFY_HW;
2435         }
2436
2437         switch (get_aead_subtype(tfm)) {
2438         case CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC:
2439         case CRYPTO_ALG_SUB_TYPE_AEAD_NULL:
2440                 return chcr_aead_op(req, CHCR_DECRYPT_OP, size,
2441                                     create_authenc_wr);
2442         case CRYPTO_ALG_SUB_TYPE_AEAD_CCM:
2443         case CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309:
2444                 return chcr_aead_op(req, CHCR_DECRYPT_OP, size,
2445                                     create_aead_ccm_wr);
2446         default:
2447                 return chcr_aead_op(req, CHCR_DECRYPT_OP, size,
2448                                     create_gcm_wr);
2449         }
2450 }
2451
2452 static int chcr_aead_op(struct aead_request *req,
2453                           unsigned short op_type,
2454                           int size,
2455                           create_wr_t create_wr_fn)
2456 {
2457         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2458         struct chcr_context *ctx = crypto_aead_ctx(tfm);
2459         struct uld_ctx *u_ctx;
2460         struct sk_buff *skb;
2461
2462         if (!ctx->dev) {
2463                 pr_err("chcr : %s : No crypto device.\n", __func__);
2464                 return -ENXIO;
2465         }
2466         u_ctx = ULD_CTX(ctx);
2467         if (cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
2468                                    ctx->tx_channel_id)) {
2469                 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
2470                         return -EBUSY;
2471         }
2472
2473         /* Form a WR from req */
2474         skb = create_wr_fn(req, u_ctx->lldi.rxq_ids[ctx->tx_channel_id], size,
2475                            op_type);
2476
2477         if (IS_ERR(skb) || skb == NULL) {
2478                 pr_err("chcr : %s : failed to form WR. No memory\n", __func__);
2479                 return PTR_ERR(skb);
2480         }
2481
2482         skb->dev = u_ctx->lldi.ports[0];
2483         set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_channel_id);
2484         chcr_send_wr(skb);
2485         return -EINPROGRESS;
2486 }
2487 static struct chcr_alg_template driver_algs[] = {
2488         /* AES-CBC */
2489         {
2490                 .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2491                 .is_registered = 0,
2492                 .alg.crypto = {
2493                         .cra_name               = "cbc(aes)",
2494                         .cra_driver_name        = "cbc-aes-chcr",
2495                         .cra_priority           = CHCR_CRA_PRIORITY,
2496                         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER |
2497                                 CRYPTO_ALG_ASYNC,
2498                         .cra_blocksize          = AES_BLOCK_SIZE,
2499                         .cra_ctxsize            = sizeof(struct chcr_context)
2500                                 + sizeof(struct ablk_ctx),
2501                         .cra_alignmask          = 0,
2502                         .cra_type               = &crypto_ablkcipher_type,
2503                         .cra_module             = THIS_MODULE,
2504                         .cra_init               = chcr_cra_init,
2505                         .cra_exit               = NULL,
2506                         .cra_u.ablkcipher       = {
2507                                 .min_keysize    = AES_MIN_KEY_SIZE,
2508                                 .max_keysize    = AES_MAX_KEY_SIZE,
2509                                 .ivsize         = AES_BLOCK_SIZE,
2510                                 .setkey                 = chcr_aes_cbc_setkey,
2511                                 .encrypt                = chcr_aes_encrypt,
2512                                 .decrypt                = chcr_aes_decrypt,
2513                         }
2514                 }
2515         },
2516         {
2517                 .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2518                 .is_registered = 0,
2519                 .alg.crypto =   {
2520                         .cra_name               = "xts(aes)",
2521                         .cra_driver_name        = "xts-aes-chcr",
2522                         .cra_priority           = CHCR_CRA_PRIORITY,
2523                         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER |
2524                                 CRYPTO_ALG_ASYNC,
2525                         .cra_blocksize          = AES_BLOCK_SIZE,
2526                         .cra_ctxsize            = sizeof(struct chcr_context) +
2527                                 sizeof(struct ablk_ctx),
2528                         .cra_alignmask          = 0,
2529                         .cra_type               = &crypto_ablkcipher_type,
2530                         .cra_module             = THIS_MODULE,
2531                         .cra_init               = chcr_cra_init,
2532                         .cra_exit               = NULL,
2533                         .cra_u = {
2534                                 .ablkcipher = {
2535                                         .min_keysize    = 2 * AES_MIN_KEY_SIZE,
2536                                         .max_keysize    = 2 * AES_MAX_KEY_SIZE,
2537                                         .ivsize         = AES_BLOCK_SIZE,
2538                                         .setkey         = chcr_aes_xts_setkey,
2539                                         .encrypt        = chcr_aes_encrypt,
2540                                         .decrypt        = chcr_aes_decrypt,
2541                                 }
2542                         }
2543                 }
2544         },
2545         /* SHA */
2546         {
2547                 .type = CRYPTO_ALG_TYPE_AHASH,
2548                 .is_registered = 0,
2549                 .alg.hash = {
2550                         .halg.digestsize = SHA1_DIGEST_SIZE,
2551                         .halg.base = {
2552                                 .cra_name = "sha1",
2553                                 .cra_driver_name = "sha1-chcr",
2554                                 .cra_blocksize = SHA1_BLOCK_SIZE,
2555                         }
2556                 }
2557         },
2558         {
2559                 .type = CRYPTO_ALG_TYPE_AHASH,
2560                 .is_registered = 0,
2561                 .alg.hash = {
2562                         .halg.digestsize = SHA256_DIGEST_SIZE,
2563                         .halg.base = {
2564                                 .cra_name = "sha256",
2565                                 .cra_driver_name = "sha256-chcr",
2566                                 .cra_blocksize = SHA256_BLOCK_SIZE,
2567                         }
2568                 }
2569         },
2570         {
2571                 .type = CRYPTO_ALG_TYPE_AHASH,
2572                 .is_registered = 0,
2573                 .alg.hash = {
2574                         .halg.digestsize = SHA224_DIGEST_SIZE,
2575                         .halg.base = {
2576                                 .cra_name = "sha224",
2577                                 .cra_driver_name = "sha224-chcr",
2578                                 .cra_blocksize = SHA224_BLOCK_SIZE,
2579                         }
2580                 }
2581         },
2582         {
2583                 .type = CRYPTO_ALG_TYPE_AHASH,
2584                 .is_registered = 0,
2585                 .alg.hash = {
2586                         .halg.digestsize = SHA384_DIGEST_SIZE,
2587                         .halg.base = {
2588                                 .cra_name = "sha384",
2589                                 .cra_driver_name = "sha384-chcr",
2590                                 .cra_blocksize = SHA384_BLOCK_SIZE,
2591                         }
2592                 }
2593         },
2594         {
2595                 .type = CRYPTO_ALG_TYPE_AHASH,
2596                 .is_registered = 0,
2597                 .alg.hash = {
2598                         .halg.digestsize = SHA512_DIGEST_SIZE,
2599                         .halg.base = {
2600                                 .cra_name = "sha512",
2601                                 .cra_driver_name = "sha512-chcr",
2602                                 .cra_blocksize = SHA512_BLOCK_SIZE,
2603                         }
2604                 }
2605         },
2606         /* HMAC */
2607         {
2608                 .type = CRYPTO_ALG_TYPE_HMAC,
2609                 .is_registered = 0,
2610                 .alg.hash = {
2611                         .halg.digestsize = SHA1_DIGEST_SIZE,
2612                         .halg.base = {
2613                                 .cra_name = "hmac(sha1)",
2614                                 .cra_driver_name = "hmac-sha1-chcr",
2615                                 .cra_blocksize = SHA1_BLOCK_SIZE,
2616                         }
2617                 }
2618         },
2619         {
2620                 .type = CRYPTO_ALG_TYPE_HMAC,
2621                 .is_registered = 0,
2622                 .alg.hash = {
2623                         .halg.digestsize = SHA224_DIGEST_SIZE,
2624                         .halg.base = {
2625                                 .cra_name = "hmac(sha224)",
2626                                 .cra_driver_name = "hmac-sha224-chcr",
2627                                 .cra_blocksize = SHA224_BLOCK_SIZE,
2628                         }
2629                 }
2630         },
2631         {
2632                 .type = CRYPTO_ALG_TYPE_HMAC,
2633                 .is_registered = 0,
2634                 .alg.hash = {
2635                         .halg.digestsize = SHA256_DIGEST_SIZE,
2636                         .halg.base = {
2637                                 .cra_name = "hmac(sha256)",
2638                                 .cra_driver_name = "hmac-sha256-chcr",
2639                                 .cra_blocksize = SHA256_BLOCK_SIZE,
2640                         }
2641                 }
2642         },
2643         {
2644                 .type = CRYPTO_ALG_TYPE_HMAC,
2645                 .is_registered = 0,
2646                 .alg.hash = {
2647                         .halg.digestsize = SHA384_DIGEST_SIZE,
2648                         .halg.base = {
2649                                 .cra_name = "hmac(sha384)",
2650                                 .cra_driver_name = "hmac-sha384-chcr",
2651                                 .cra_blocksize = SHA384_BLOCK_SIZE,
2652                         }
2653                 }
2654         },
2655         {
2656                 .type = CRYPTO_ALG_TYPE_HMAC,
2657                 .is_registered = 0,
2658                 .alg.hash = {
2659                         .halg.digestsize = SHA512_DIGEST_SIZE,
2660                         .halg.base = {
2661                                 .cra_name = "hmac(sha512)",
2662                                 .cra_driver_name = "hmac-sha512-chcr",
2663                                 .cra_blocksize = SHA512_BLOCK_SIZE,
2664                         }
2665                 }
2666         },
2667         /* Add AEAD Algorithms */
2668         {
2669                 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_GCM,
2670                 .is_registered = 0,
2671                 .alg.aead = {
2672                         .base = {
2673                                 .cra_name = "gcm(aes)",
2674                                 .cra_driver_name = "gcm-aes-chcr",
2675                                 .cra_blocksize  = 1,
2676                                 .cra_ctxsize =  sizeof(struct chcr_context) +
2677                                                 sizeof(struct chcr_aead_ctx) +
2678                                                 sizeof(struct chcr_gcm_ctx),
2679                         },
2680                         .ivsize = 12,
2681                         .maxauthsize = GHASH_DIGEST_SIZE,
2682                         .setkey = chcr_gcm_setkey,
2683                         .setauthsize = chcr_gcm_setauthsize,
2684                 }
2685         },
2686         {
2687                 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106,
2688                 .is_registered = 0,
2689                 .alg.aead = {
2690                         .base = {
2691                                 .cra_name = "rfc4106(gcm(aes))",
2692                                 .cra_driver_name = "rfc4106-gcm-aes-chcr",
2693                                 .cra_blocksize   = 1,
2694                                 .cra_ctxsize =  sizeof(struct chcr_context) +
2695                                                 sizeof(struct chcr_aead_ctx) +
2696                                                 sizeof(struct chcr_gcm_ctx),
2697
2698                         },
2699                         .ivsize = 8,
2700                         .maxauthsize    = GHASH_DIGEST_SIZE,
2701                         .setkey = chcr_gcm_setkey,
2702                         .setauthsize    = chcr_4106_4309_setauthsize,
2703                 }
2704         },
2705         {
2706                 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_CCM,
2707                 .is_registered = 0,
2708                 .alg.aead = {
2709                         .base = {
2710                                 .cra_name = "ccm(aes)",
2711                                 .cra_driver_name = "ccm-aes-chcr",
2712                                 .cra_blocksize   = 1,
2713                                 .cra_ctxsize =  sizeof(struct chcr_context) +
2714                                                 sizeof(struct chcr_aead_ctx),
2715
2716                         },
2717                         .ivsize = AES_BLOCK_SIZE,
2718                         .maxauthsize    = GHASH_DIGEST_SIZE,
2719                         .setkey = chcr_aead_ccm_setkey,
2720                         .setauthsize    = chcr_ccm_setauthsize,
2721                 }
2722         },
2723         {
2724                 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309,
2725                 .is_registered = 0,
2726                 .alg.aead = {
2727                         .base = {
2728                                 .cra_name = "rfc4309(ccm(aes))",
2729                                 .cra_driver_name = "rfc4309-ccm-aes-chcr",
2730                                 .cra_blocksize   = 1,
2731                                 .cra_ctxsize =  sizeof(struct chcr_context) +
2732                                                 sizeof(struct chcr_aead_ctx),
2733
2734                         },
2735                         .ivsize = 8,
2736                         .maxauthsize    = GHASH_DIGEST_SIZE,
2737                         .setkey = chcr_aead_rfc4309_setkey,
2738                         .setauthsize = chcr_4106_4309_setauthsize,
2739                 }
2740         },
2741         {
2742                 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC,
2743                 .is_registered = 0,
2744                 .alg.aead = {
2745                         .base = {
2746                                 .cra_name = "authenc(hmac(sha1),cbc(aes))",
2747                                 .cra_driver_name =
2748                                         "authenc-hmac-sha1-cbc-aes-chcr",
2749                                 .cra_blocksize   = AES_BLOCK_SIZE,
2750                                 .cra_ctxsize =  sizeof(struct chcr_context) +
2751                                                 sizeof(struct chcr_aead_ctx) +
2752                                                 sizeof(struct chcr_authenc_ctx),
2753
2754                         },
2755                         .ivsize = AES_BLOCK_SIZE,
2756                         .maxauthsize = SHA1_DIGEST_SIZE,
2757                         .setkey = chcr_authenc_setkey,
2758                         .setauthsize = chcr_authenc_setauthsize,
2759                 }
2760         },
2761         {
2762                 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC,
2763                 .is_registered = 0,
2764                 .alg.aead = {
2765                         .base = {
2766
2767                                 .cra_name = "authenc(hmac(sha256),cbc(aes))",
2768                                 .cra_driver_name =
2769                                         "authenc-hmac-sha256-cbc-aes-chcr",
2770                                 .cra_blocksize   = AES_BLOCK_SIZE,
2771                                 .cra_ctxsize =  sizeof(struct chcr_context) +
2772                                                 sizeof(struct chcr_aead_ctx) +
2773                                                 sizeof(struct chcr_authenc_ctx),
2774
2775                         },
2776                         .ivsize = AES_BLOCK_SIZE,
2777                         .maxauthsize    = SHA256_DIGEST_SIZE,
2778                         .setkey = chcr_authenc_setkey,
2779                         .setauthsize = chcr_authenc_setauthsize,
2780                 }
2781         },
2782         {
2783                 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC,
2784                 .is_registered = 0,
2785                 .alg.aead = {
2786                         .base = {
2787                                 .cra_name = "authenc(hmac(sha224),cbc(aes))",
2788                                 .cra_driver_name =
2789                                         "authenc-hmac-sha224-cbc-aes-chcr",
2790                                 .cra_blocksize   = AES_BLOCK_SIZE,
2791                                 .cra_ctxsize =  sizeof(struct chcr_context) +
2792                                                 sizeof(struct chcr_aead_ctx) +
2793                                                 sizeof(struct chcr_authenc_ctx),
2794                         },
2795                         .ivsize = AES_BLOCK_SIZE,
2796                         .maxauthsize = SHA224_DIGEST_SIZE,
2797                         .setkey = chcr_authenc_setkey,
2798                         .setauthsize = chcr_authenc_setauthsize,
2799                 }
2800         },
2801         {
2802                 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC,
2803                 .is_registered = 0,
2804                 .alg.aead = {
2805                         .base = {
2806                                 .cra_name = "authenc(hmac(sha384),cbc(aes))",
2807                                 .cra_driver_name =
2808                                         "authenc-hmac-sha384-cbc-aes-chcr",
2809                                 .cra_blocksize   = AES_BLOCK_SIZE,
2810                                 .cra_ctxsize =  sizeof(struct chcr_context) +
2811                                                 sizeof(struct chcr_aead_ctx) +
2812                                                 sizeof(struct chcr_authenc_ctx),
2813
2814                         },
2815                         .ivsize = AES_BLOCK_SIZE,
2816                         .maxauthsize = SHA384_DIGEST_SIZE,
2817                         .setkey = chcr_authenc_setkey,
2818                         .setauthsize = chcr_authenc_setauthsize,
2819                 }
2820         },
2821         {
2822                 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC,
2823                 .is_registered = 0,
2824                 .alg.aead = {
2825                         .base = {
2826                                 .cra_name = "authenc(hmac(sha512),cbc(aes))",
2827                                 .cra_driver_name =
2828                                         "authenc-hmac-sha512-cbc-aes-chcr",
2829                                 .cra_blocksize   = AES_BLOCK_SIZE,
2830                                 .cra_ctxsize =  sizeof(struct chcr_context) +
2831                                                 sizeof(struct chcr_aead_ctx) +
2832                                                 sizeof(struct chcr_authenc_ctx),
2833
2834                         },
2835                         .ivsize = AES_BLOCK_SIZE,
2836                         .maxauthsize = SHA512_DIGEST_SIZE,
2837                         .setkey = chcr_authenc_setkey,
2838                         .setauthsize = chcr_authenc_setauthsize,
2839                 }
2840         },
2841         {
2842                 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_NULL,
2843                 .is_registered = 0,
2844                 .alg.aead = {
2845                         .base = {
2846                                 .cra_name = "authenc(digest_null,cbc(aes))",
2847                                 .cra_driver_name =
2848                                         "authenc-digest_null-cbc-aes-chcr",
2849                                 .cra_blocksize   = AES_BLOCK_SIZE,
2850                                 .cra_ctxsize =  sizeof(struct chcr_context) +
2851                                                 sizeof(struct chcr_aead_ctx) +
2852                                                 sizeof(struct chcr_authenc_ctx),
2853
2854                         },
2855                         .ivsize  = AES_BLOCK_SIZE,
2856                         .maxauthsize = 0,
2857                         .setkey  = chcr_aead_digest_null_setkey,
2858                         .setauthsize = chcr_authenc_null_setauthsize,
2859                 }
2860         },
2861 };
2862
2863 /*
2864  *      chcr_unregister_alg - Deregister crypto algorithms with
2865  *      kernel framework.
2866  */
2867 static int chcr_unregister_alg(void)
2868 {
2869         int i;
2870
2871         for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
2872                 switch (driver_algs[i].type & CRYPTO_ALG_TYPE_MASK) {
2873                 case CRYPTO_ALG_TYPE_ABLKCIPHER:
2874                         if (driver_algs[i].is_registered)
2875                                 crypto_unregister_alg(
2876                                                 &driver_algs[i].alg.crypto);
2877                         break;
2878                 case CRYPTO_ALG_TYPE_AEAD:
2879                         if (driver_algs[i].is_registered)
2880                                 crypto_unregister_aead(
2881                                                 &driver_algs[i].alg.aead);
2882                         break;
2883                 case CRYPTO_ALG_TYPE_AHASH:
2884                         if (driver_algs[i].is_registered)
2885                                 crypto_unregister_ahash(
2886                                                 &driver_algs[i].alg.hash);
2887                         break;
2888                 }
2889                 driver_algs[i].is_registered = 0;
2890         }
2891         return 0;
2892 }
2893
2894 #define SZ_AHASH_CTX sizeof(struct chcr_context)
2895 #define SZ_AHASH_H_CTX (sizeof(struct chcr_context) + sizeof(struct hmac_ctx))
2896 #define SZ_AHASH_REQ_CTX sizeof(struct chcr_ahash_req_ctx)
2897 #define AHASH_CRA_FLAGS (CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC)
2898
2899 /*
2900  *      chcr_register_alg - Register crypto algorithms with kernel framework.
2901  */
2902 static int chcr_register_alg(void)
2903 {
2904         struct crypto_alg ai;
2905         struct ahash_alg *a_hash;
2906         int err = 0, i;
2907         char *name = NULL;
2908
2909         for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
2910                 if (driver_algs[i].is_registered)
2911                         continue;
2912                 switch (driver_algs[i].type & CRYPTO_ALG_TYPE_MASK) {
2913                 case CRYPTO_ALG_TYPE_ABLKCIPHER:
2914                         err = crypto_register_alg(&driver_algs[i].alg.crypto);
2915                         name = driver_algs[i].alg.crypto.cra_driver_name;
2916                         break;
2917                 case CRYPTO_ALG_TYPE_AEAD:
2918                         driver_algs[i].alg.aead.base.cra_priority =
2919                                 CHCR_CRA_PRIORITY;
2920                         driver_algs[i].alg.aead.base.cra_flags =
2921                                 CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC;
2922                         driver_algs[i].alg.aead.encrypt = chcr_aead_encrypt;
2923                         driver_algs[i].alg.aead.decrypt = chcr_aead_decrypt;
2924                         driver_algs[i].alg.aead.init = chcr_aead_cra_init;
2925                         driver_algs[i].alg.aead.exit = chcr_aead_cra_exit;
2926                         driver_algs[i].alg.aead.base.cra_module = THIS_MODULE;
2927                         err = crypto_register_aead(&driver_algs[i].alg.aead);
2928                         name = driver_algs[i].alg.aead.base.cra_driver_name;
2929                         break;
2930                 case CRYPTO_ALG_TYPE_AHASH:
2931                         a_hash = &driver_algs[i].alg.hash;
2932                         a_hash->update = chcr_ahash_update;
2933                         a_hash->final = chcr_ahash_final;
2934                         a_hash->finup = chcr_ahash_finup;
2935                         a_hash->digest = chcr_ahash_digest;
2936                         a_hash->export = chcr_ahash_export;
2937                         a_hash->import = chcr_ahash_import;
2938                         a_hash->halg.statesize = SZ_AHASH_REQ_CTX;
2939                         a_hash->halg.base.cra_priority = CHCR_CRA_PRIORITY;
2940                         a_hash->halg.base.cra_module = THIS_MODULE;
2941                         a_hash->halg.base.cra_flags = AHASH_CRA_FLAGS;
2942                         a_hash->halg.base.cra_alignmask = 0;
2943                         a_hash->halg.base.cra_exit = NULL;
2944                         a_hash->halg.base.cra_type = &crypto_ahash_type;
2945
2946                         if (driver_algs[i].type == CRYPTO_ALG_TYPE_HMAC) {
2947                                 a_hash->halg.base.cra_init = chcr_hmac_cra_init;
2948                                 a_hash->halg.base.cra_exit = chcr_hmac_cra_exit;
2949                                 a_hash->init = chcr_hmac_init;
2950                                 a_hash->setkey = chcr_ahash_setkey;
2951                                 a_hash->halg.base.cra_ctxsize = SZ_AHASH_H_CTX;
2952                         } else {
2953                                 a_hash->init = chcr_sha_init;
2954                                 a_hash->halg.base.cra_ctxsize = SZ_AHASH_CTX;
2955                                 a_hash->halg.base.cra_init = chcr_sha_cra_init;
2956                         }
2957                         err = crypto_register_ahash(&driver_algs[i].alg.hash);
2958                         ai = driver_algs[i].alg.hash.halg.base;
2959                         name = ai.cra_driver_name;
2960                         break;
2961                 }
2962                 if (err) {
2963                         pr_err("chcr : %s : Algorithm registration failed\n",
2964                                name);
2965                         goto register_err;
2966                 } else {
2967                         driver_algs[i].is_registered = 1;
2968                 }
2969         }
2970         return 0;
2971
2972 register_err:
2973         chcr_unregister_alg();
2974         return err;
2975 }
2976
2977 /*
2978  *      start_crypto - Register the crypto algorithms.
2979  *      This should called once when the first device comesup. After this
2980  *      kernel will start calling driver APIs for crypto operations.
2981  */
2982 int start_crypto(void)
2983 {
2984         return chcr_register_alg();
2985 }
2986
2987 /*
2988  *      stop_crypto - Deregister all the crypto algorithms with kernel.
2989  *      This should be called once when the last device goes down. After this
2990  *      kernel will not call the driver API for crypto operations.
2991  */
2992 int stop_crypto(void)
2993 {
2994         chcr_unregister_alg();
2995         return 0;
2996 }