drivers/crypto/talitos.c

   1 /*
   2  * talitos - Freescale Integrated Security Engine (SEC) device driver
   3  *
   4  * Copyright (c) 2008-2011 Freescale Semiconductor, Inc.
   5  *
   6  * Scatterlist Crypto API glue code copied from files with the following:
   7  * Copyright (c) 2006-2007 Herbert Xu <herbert@gondor.apana.org.au>
   8  *
   9  * Crypto algorithm registration code copied from hifn driver:
  10  * 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru>
  11  * All rights reserved.
  12  *
  13  * This program is free software; you can redistribute it and/or modify
  14  * it under the terms of the GNU General Public License as published by
  15  * the Free Software Foundation; either version 2 of the License, or
  16  * (at your option) any later version.
  17  *
  18  * This program is distributed in the hope that it will be useful,
  19  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  20  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  21  * GNU General Public License for more details.
  22  *
  23  * You should have received a copy of the GNU General Public License
  24  * along with this program; if not, write to the Free Software
  25  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  26  */
  27
  28 #include <linux/kernel.h>
  29 #include <linux/module.h>
  30 #include <linux/mod_devicetable.h>
  31 #include <linux/device.h>
  32 #include <linux/interrupt.h>
  33 #include <linux/crypto.h>
  34 #include <linux/hw_random.h>
  35 #include <linux/of_address.h>
  36 #include <linux/of_irq.h>
  37 #include <linux/of_platform.h>
  38 #include <linux/dma-mapping.h>
  39 #include <linux/io.h>
  40 #include <linux/spinlock.h>
  41 #include <linux/rtnetlink.h>
  42 #include <linux/slab.h>
  43
  44 #include <crypto/algapi.h>
  45 #include <crypto/aes.h>
  46 #include <crypto/des.h>
  47 #include <crypto/sha.h>
  48 #include <crypto/md5.h>
  49 #include <crypto/internal/aead.h>
  50 #include <crypto/authenc.h>
  51 #include <crypto/skcipher.h>
  52 #include <crypto/hash.h>
  53 #include <crypto/internal/hash.h>
  54 #include <crypto/scatterwalk.h>
  55
  56 #include "talitos.h"
  57
  58 static void to_talitos_ptr(struct talitos_ptr *ptr, dma_addr_t dma_addr,
  59                            bool is_sec1)
  60 {
  61         ptr->ptr = cpu_to_be32(lower_32_bits(dma_addr));
  62         if (!is_sec1)
  63                 ptr->eptr = upper_32_bits(dma_addr);
  64 }
  65
  66 static void copy_talitos_ptr(struct talitos_ptr *dst_ptr,
  67                              struct talitos_ptr *src_ptr, bool is_sec1)
  68 {
  69         dst_ptr->ptr = src_ptr->ptr;
  70         if (!is_sec1)
  71                 dst_ptr->eptr = src_ptr->eptr;
  72 }
  73
  74 static void to_talitos_ptr_len(struct talitos_ptr *ptr, unsigned int len,
  75                                bool is_sec1)
  76 {
  77         if (is_sec1) {
  78                 ptr->res = 0;
  79                 ptr->len1 = cpu_to_be16(len);
  80         } else {
  81                 ptr->len = cpu_to_be16(len);
  82         }
  83 }
  84
  85 static unsigned short from_talitos_ptr_len(struct talitos_ptr *ptr,
  86                                            bool is_sec1)
  87 {
  88         if (is_sec1)
  89                 return be16_to_cpu(ptr->len1);
  90         else
  91                 return be16_to_cpu(ptr->len);
  92 }
  93
  94 static void to_talitos_ptr_extent_clear(struct talitos_ptr *ptr, bool is_sec1)
  95 {
  96         if (!is_sec1)
  97                 ptr->j_extent = 0;
  98 }
  99
 100 /*
 101  * map virtual single (contiguous) pointer to h/w descriptor pointer
 102  */
 103 static void map_single_talitos_ptr(struct device *dev,
 104                                    struct talitos_ptr *ptr,
 105                                    unsigned int len, void *data,
 106                                    enum dma_data_direction dir)
 107 {
 108         dma_addr_t dma_addr = dma_map_single(dev, data, len, dir);
 109         struct talitos_private *priv = dev_get_drvdata(dev);
 110         bool is_sec1 = has_ftr_sec1(priv);
 111
 112         to_talitos_ptr_len(ptr, len, is_sec1);
 113         to_talitos_ptr(ptr, dma_addr, is_sec1);
 114         to_talitos_ptr_extent_clear(ptr, is_sec1);
 115 }
 116
 117 /*
 118  * unmap bus single (contiguous) h/w descriptor pointer
 119  */
 120 static void unmap_single_talitos_ptr(struct device *dev,
 121                                      struct talitos_ptr *ptr,
 122                                      enum dma_data_direction dir)
 123 {
 124         struct talitos_private *priv = dev_get_drvdata(dev);
 125         bool is_sec1 = has_ftr_sec1(priv);
 126
 127         dma_unmap_single(dev, be32_to_cpu(ptr->ptr),
 128                          from_talitos_ptr_len(ptr, is_sec1), dir);
 129 }
 130
 131 static int reset_channel(struct device *dev, int ch)
 132 {
 133         struct talitos_private *priv = dev_get_drvdata(dev);
 134         unsigned int timeout = TALITOS_TIMEOUT;
 135         bool is_sec1 = has_ftr_sec1(priv);
 136
 137         if (is_sec1) {
 138                 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,
 139                           TALITOS1_CCCR_LO_RESET);
 140
 141                 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR_LO) &
 142                         TALITOS1_CCCR_LO_RESET) && --timeout)
 143                         cpu_relax();
 144         } else {
 145                 setbits32(priv->chan[ch].reg + TALITOS_CCCR,
 146                           TALITOS2_CCCR_RESET);
 147
 148                 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) &
 149                         TALITOS2_CCCR_RESET) && --timeout)
 150                         cpu_relax();
 151         }
 152
 153         if (timeout == 0) {
 154                 dev_err(dev, "failed to reset channel %d\n", ch);
 155                 return -EIO;
 156         }
 157
 158         /* set 36-bit addressing, done writeback enable and done IRQ enable */
 159         setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, TALITOS_CCCR_LO_EAE |
 160                   TALITOS_CCCR_LO_CDWE | TALITOS_CCCR_LO_CDIE);
 161
 162         /* and ICCR writeback, if available */
 163         if (priv->features & TALITOS_FTR_HW_AUTH_CHECK)
 164                 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,
 165                           TALITOS_CCCR_LO_IWSE);
 166
 167         return 0;
 168 }
 169
 170 static int reset_device(struct device *dev)
 171 {
 172         struct talitos_private *priv = dev_get_drvdata(dev);
 173         unsigned int timeout = TALITOS_TIMEOUT;
 174         bool is_sec1 = has_ftr_sec1(priv);
 175         u32 mcr = is_sec1 ? TALITOS1_MCR_SWR : TALITOS2_MCR_SWR;
 176
 177         setbits32(priv->reg + TALITOS_MCR, mcr);
 178
 179         while ((in_be32(priv->reg + TALITOS_MCR) & mcr)
 180                && --timeout)
 181                 cpu_relax();
 182
 183         if (priv->irq[1]) {
 184                 mcr = TALITOS_MCR_RCA1 | TALITOS_MCR_RCA3;
 185                 setbits32(priv->reg + TALITOS_MCR, mcr);
 186         }
 187
 188         if (timeout == 0) {
 189                 dev_err(dev, "failed to reset device\n");
 190                 return -EIO;
 191         }
 192
 193         return 0;
 194 }
 195
 196 /*
 197  * Reset and initialize the device
 198  */
 199 static int init_device(struct device *dev)
 200 {
 201         struct talitos_private *priv = dev_get_drvdata(dev);
 202         int ch, err;
 203         bool is_sec1 = has_ftr_sec1(priv);
 204
 205         /*
 206          * Master reset
 207          * errata documentation: warning: certain SEC interrupts
 208          * are not fully cleared by writing the MCR:SWR bit,
 209          * set bit twice to completely reset
 210          */
 211         err = reset_device(dev);
 212         if (err)
 213                 return err;
 214
 215         err = reset_device(dev);
 216         if (err)
 217                 return err;
 218
 219         /* reset channels */
 220         for (ch = 0; ch < priv->num_channels; ch++) {
 221                 err = reset_channel(dev, ch);
 222                 if (err)
 223                         return err;
 224         }
 225
 226         /* enable channel done and error interrupts */
 227         if (is_sec1) {
 228                 clrbits32(priv->reg + TALITOS_IMR, TALITOS1_IMR_INIT);
 229                 clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT);
 230                 /* disable parity error check in DEU (erroneous? test vect.) */
 231                 setbits32(priv->reg_deu + TALITOS_EUICR, TALITOS1_DEUICR_KPE);
 232         } else {
 233                 setbits32(priv->reg + TALITOS_IMR, TALITOS2_IMR_INIT);
 234                 setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT);
 235         }
 236
 237         /* disable integrity check error interrupts (use writeback instead) */
 238         if (priv->features & TALITOS_FTR_HW_AUTH_CHECK)
 239                 setbits32(priv->reg_mdeu + TALITOS_EUICR_LO,
 240                           TALITOS_MDEUICR_LO_ICE);
 241
 242         return 0;
 243 }
 244
 245 /**
 246  * talitos_submit - submits a descriptor to the device for processing
 247  * @dev:        the SEC device to be used
 248  * @ch:         the SEC device channel to be used
 249  * @desc:       the descriptor to be processed by the device
 250  * @callback:   whom to call when processing is complete
 251  * @context:    a handle for use by caller (optional)
 252  *
 253  * desc must contain valid dma-mapped (bus physical) address pointers.
 254  * callback must check err and feedback in descriptor header
 255  * for device processing status.
 256  */
 257 int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc,
 258                    void (*callback)(struct device *dev,
 259                                     struct talitos_desc *desc,
 260                                     void *context, int error),
 261                    void *context)
 262 {
 263         struct talitos_private *priv = dev_get_drvdata(dev);
 264         struct talitos_request *request;
 265         unsigned long flags;
 266         int head;
 267         bool is_sec1 = has_ftr_sec1(priv);
 268
 269         spin_lock_irqsave(&priv->chan[ch].head_lock, flags);
 270
 271         if (!atomic_inc_not_zero(&priv->chan[ch].submit_count)) {
 272                 /* h/w fifo is full */
 273                 spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);
 274                 return -EAGAIN;
 275         }
 276
 277         head = priv->chan[ch].head;
 278         request = &priv->chan[ch].fifo[head];
 279
 280         /* map descriptor and save caller data */
 281         if (is_sec1) {
 282                 desc->hdr1 = desc->hdr;
 283                 desc->next_desc = 0;
 284                 request->dma_desc = dma_map_single(dev, &desc->hdr1,
 285                                                    TALITOS_DESC_SIZE,
 286                                                    DMA_BIDIRECTIONAL);
 287         } else {
 288                 request->dma_desc = dma_map_single(dev, desc,
 289                                                    TALITOS_DESC_SIZE,
 290                                                    DMA_BIDIRECTIONAL);
 291         }
 292         request->callback = callback;
 293         request->context = context;
 294
 295         /* increment fifo head */
 296         priv->chan[ch].head = (priv->chan[ch].head + 1) & (priv->fifo_len - 1);
 297
 298         smp_wmb();
 299         request->desc = desc;
 300
 301         /* GO! */
 302         wmb();
 303         out_be32(priv->chan[ch].reg + TALITOS_FF,
 304                  upper_32_bits(request->dma_desc));
 305         out_be32(priv->chan[ch].reg + TALITOS_FF_LO,
 306                  lower_32_bits(request->dma_desc));
 307
 308         spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);
 309
 310         return -EINPROGRESS;
 311 }
 312 EXPORT_SYMBOL(talitos_submit);
 313
 314 /*
 315  * process what was done, notify callback of error if not
 316  */
 317 static void flush_channel(struct device *dev, int ch, int error, int reset_ch)
 318 {
 319         struct talitos_private *priv = dev_get_drvdata(dev);
 320         struct talitos_request *request, saved_req;
 321         unsigned long flags;
 322         int tail, status;
 323         bool is_sec1 = has_ftr_sec1(priv);
 324
 325         spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);
 326
 327         tail = priv->chan[ch].tail;
 328         while (priv->chan[ch].fifo[tail].desc) {
 329                 __be32 hdr;
 330
 331                 request = &priv->chan[ch].fifo[tail];
 332
 333                 /* descriptors with their done bits set don't get the error */
 334                 rmb();
 335                 hdr = is_sec1 ? request->desc->hdr1 : request->desc->hdr;
 336
 337                 if ((hdr & DESC_HDR_DONE) == DESC_HDR_DONE)
 338                         status = 0;
 339                 else
 340                         if (!error)
 341                                 break;
 342                         else
 343                                 status = error;
 344
 345                 dma_unmap_single(dev, request->dma_desc,
 346                                  TALITOS_DESC_SIZE,
 347                                  DMA_BIDIRECTIONAL);
 348
 349                 /* copy entries so we can call callback outside lock */
 350                 saved_req.desc = request->desc;
 351                 saved_req.callback = request->callback;
 352                 saved_req.context = request->context;
 353
 354                 /* release request entry in fifo */
 355                 smp_wmb();
 356                 request->desc = NULL;
 357
 358                 /* increment fifo tail */
 359                 priv->chan[ch].tail = (tail + 1) & (priv->fifo_len - 1);
 360
 361                 spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);
 362
 363                 atomic_dec(&priv->chan[ch].submit_count);
 364
 365                 saved_req.callback(dev, saved_req.desc, saved_req.context,
 366                                    status);
 367                 /* channel may resume processing in single desc error case */
 368                 if (error && !reset_ch && status == error)
 369                         return;
 370                 spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);
 371                 tail = priv->chan[ch].tail;
 372         }
 373
 374         spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);
 375 }
 376
 377 /*
 378  * process completed requests for channels that have done status
 379  */
 380 #define DEF_TALITOS1_DONE(name, ch_done_mask)                           \
 381 static void talitos1_done_##name(unsigned long data)                    \
 382 {                                                                       \
 383         struct device *dev = (struct device *)data;                     \
 384         struct talitos_private *priv = dev_get_drvdata(dev);            \
 385         unsigned long flags;                                            \
 386                                                                         \
 387         if (ch_done_mask & 0x10000000)                                  \
 388                 flush_channel(dev, 0, 0, 0);                    \
 389         if (priv->num_channels == 1)                                    \
 390                 goto out;                                               \
 391         if (ch_done_mask & 0x40000000)                                  \
 392                 flush_channel(dev, 1, 0, 0);                    \
 393         if (ch_done_mask & 0x00010000)                                  \
 394                 flush_channel(dev, 2, 0, 0);                    \
 395         if (ch_done_mask & 0x00040000)                                  \
 396                 flush_channel(dev, 3, 0, 0);                    \
 397                                                                         \
 398 out:                                                                    \
 399         /* At this point, all completed channels have been processed */ \
 400         /* Unmask done interrupts for channels completed later on. */   \
 401         spin_lock_irqsave(&priv->reg_lock, flags);                      \
 402         clrbits32(priv->reg + TALITOS_IMR, ch_done_mask);               \
 403         clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT);    \
 404         spin_unlock_irqrestore(&priv->reg_lock, flags);                 \
 405 }
 406
 407 DEF_TALITOS1_DONE(4ch, TALITOS1_ISR_4CHDONE)
 408
 409 #define DEF_TALITOS2_DONE(name, ch_done_mask)                           \
 410 static void talitos2_done_##name(unsigned long data)                    \
 411 {                                                                       \
 412         struct device *dev = (struct device *)data;                     \
 413         struct talitos_private *priv = dev_get_drvdata(dev);            \
 414         unsigned long flags;                                            \
 415                                                                         \
 416         if (ch_done_mask & 1)                                           \
 417                 flush_channel(dev, 0, 0, 0);                            \
 418         if (priv->num_channels == 1)                                    \
 419                 goto out;                                               \
 420         if (ch_done_mask & (1 << 2))                                    \
 421                 flush_channel(dev, 1, 0, 0);                            \
 422         if (ch_done_mask & (1 << 4))                                    \
 423                 flush_channel(dev, 2, 0, 0);                            \
 424         if (ch_done_mask & (1 << 6))                                    \
 425                 flush_channel(dev, 3, 0, 0);                            \
 426                                                                         \
 427 out:                                                                    \
 428         /* At this point, all completed channels have been processed */ \
 429         /* Unmask done interrupts for channels completed later on. */   \
 430         spin_lock_irqsave(&priv->reg_lock, flags);                      \
 431         setbits32(priv->reg + TALITOS_IMR, ch_done_mask);               \
 432         setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT);    \
 433         spin_unlock_irqrestore(&priv->reg_lock, flags);                 \
 434 }
 435
 436 DEF_TALITOS2_DONE(4ch, TALITOS2_ISR_4CHDONE)
 437 DEF_TALITOS2_DONE(ch0_2, TALITOS2_ISR_CH_0_2_DONE)
 438 DEF_TALITOS2_DONE(ch1_3, TALITOS2_ISR_CH_1_3_DONE)
 439
 440 /*
 441  * locate current (offending) descriptor
 442  */
 443 static u32 current_desc_hdr(struct device *dev, int ch)
 444 {
 445         struct talitos_private *priv = dev_get_drvdata(dev);
 446         int tail, iter;
 447         dma_addr_t cur_desc;
 448
 449         cur_desc = ((u64)in_be32(priv->chan[ch].reg + TALITOS_CDPR)) << 32;
 450         cur_desc |= in_be32(priv->chan[ch].reg + TALITOS_CDPR_LO);
 451
 452         if (!cur_desc) {
 453                 dev_err(dev, "CDPR is NULL, giving up search for offending descriptor\n");
 454                 return 0;
 455         }
 456
 457         tail = priv->chan[ch].tail;
 458
 459         iter = tail;
 460         while (priv->chan[ch].fifo[iter].dma_desc != cur_desc) {
 461                 iter = (iter + 1) & (priv->fifo_len - 1);
 462                 if (iter == tail) {
 463                         dev_err(dev, "couldn't locate current descriptor\n");
 464                         return 0;
 465                 }
 466         }
 467
 468         return priv->chan[ch].fifo[iter].desc->hdr;
 469 }
 470
 471 /*
 472  * user diagnostics; report root cause of error based on execution unit status
 473  */
 474 static void report_eu_error(struct device *dev, int ch, u32 desc_hdr)
 475 {
 476         struct talitos_private *priv = dev_get_drvdata(dev);
 477         int i;
 478
 479         if (!desc_hdr)
 480                 desc_hdr = in_be32(priv->chan[ch].reg + TALITOS_DESCBUF);
 481
 482         switch (desc_hdr & DESC_HDR_SEL0_MASK) {
 483         case DESC_HDR_SEL0_AFEU:
 484                 dev_err(dev, "AFEUISR 0x%08x_%08x\n",
 485                         in_be32(priv->reg_afeu + TALITOS_EUISR),
 486                         in_be32(priv->reg_afeu + TALITOS_EUISR_LO));
 487                 break;
 488         case DESC_HDR_SEL0_DEU:
 489                 dev_err(dev, "DEUISR 0x%08x_%08x\n",
 490                         in_be32(priv->reg_deu + TALITOS_EUISR),
 491                         in_be32(priv->reg_deu + TALITOS_EUISR_LO));
 492                 break;
 493         case DESC_HDR_SEL0_MDEUA:
 494         case DESC_HDR_SEL0_MDEUB:
 495                 dev_err(dev, "MDEUISR 0x%08x_%08x\n",
 496                         in_be32(priv->reg_mdeu + TALITOS_EUISR),
 497                         in_be32(priv->reg_mdeu + TALITOS_EUISR_LO));
 498                 break;
 499         case DESC_HDR_SEL0_RNG:
 500                 dev_err(dev, "RNGUISR 0x%08x_%08x\n",
 501                         in_be32(priv->reg_rngu + TALITOS_ISR),
 502                         in_be32(priv->reg_rngu + TALITOS_ISR_LO));
 503                 break;
 504         case DESC_HDR_SEL0_PKEU:
 505                 dev_err(dev, "PKEUISR 0x%08x_%08x\n",
 506                         in_be32(priv->reg_pkeu + TALITOS_EUISR),
 507                         in_be32(priv->reg_pkeu + TALITOS_EUISR_LO));
 508                 break;
 509         case DESC_HDR_SEL0_AESU:
 510                 dev_err(dev, "AESUISR 0x%08x_%08x\n",
 511                         in_be32(priv->reg_aesu + TALITOS_EUISR),
 512                         in_be32(priv->reg_aesu + TALITOS_EUISR_LO));
 513                 break;
 514         case DESC_HDR_SEL0_CRCU:
 515                 dev_err(dev, "CRCUISR 0x%08x_%08x\n",
 516                         in_be32(priv->reg_crcu + TALITOS_EUISR),
 517                         in_be32(priv->reg_crcu + TALITOS_EUISR_LO));
 518                 break;
 519         case DESC_HDR_SEL0_KEU:
 520                 dev_err(dev, "KEUISR 0x%08x_%08x\n",
 521                         in_be32(priv->reg_pkeu + TALITOS_EUISR),
 522                         in_be32(priv->reg_pkeu + TALITOS_EUISR_LO));
 523                 break;
 524         }
 525
 526         switch (desc_hdr & DESC_HDR_SEL1_MASK) {
 527         case DESC_HDR_SEL1_MDEUA:
 528         case DESC_HDR_SEL1_MDEUB:
 529                 dev_err(dev, "MDEUISR 0x%08x_%08x\n",
 530                         in_be32(priv->reg_mdeu + TALITOS_EUISR),
 531                         in_be32(priv->reg_mdeu + TALITOS_EUISR_LO));
 532                 break;
 533         case DESC_HDR_SEL1_CRCU:
 534                 dev_err(dev, "CRCUISR 0x%08x_%08x\n",
 535                         in_be32(priv->reg_crcu + TALITOS_EUISR),
 536                         in_be32(priv->reg_crcu + TALITOS_EUISR_LO));
 537                 break;
 538         }
 539
 540         for (i = 0; i < 8; i++)
 541                 dev_err(dev, "DESCBUF 0x%08x_%08x\n",
 542                         in_be32(priv->chan[ch].reg + TALITOS_DESCBUF + 8*i),
 543                         in_be32(priv->chan[ch].reg + TALITOS_DESCBUF_LO + 8*i));
 544 }
 545
 546 /*
 547  * recover from error interrupts
 548  */
 549 static void talitos_error(struct device *dev, u32 isr, u32 isr_lo)
 550 {
 551         struct talitos_private *priv = dev_get_drvdata(dev);
 552         unsigned int timeout = TALITOS_TIMEOUT;
 553         int ch, error, reset_dev = 0;
 554         u32 v_lo;
 555         bool is_sec1 = has_ftr_sec1(priv);
 556         int reset_ch = is_sec1 ? 1 : 0; /* only SEC2 supports continuation */
 557
 558         for (ch = 0; ch < priv->num_channels; ch++) {
 559                 /* skip channels without errors */
 560                 if (is_sec1) {
 561                         /* bits 29, 31, 17, 19 */
 562                         if (!(isr & (1 << (29 + (ch & 1) * 2 - (ch & 2) * 6))))
 563                                 continue;
 564                 } else {
 565                         if (!(isr & (1 << (ch * 2 + 1))))
 566                                 continue;
 567                 }
 568
 569                 error = -EINVAL;
 570
 571                 v_lo = in_be32(priv->chan[ch].reg + TALITOS_CCPSR_LO);
 572
 573                 if (v_lo & TALITOS_CCPSR_LO_DOF) {
 574                         dev_err(dev, "double fetch fifo overflow error\n");
 575                         error = -EAGAIN;
 576                         reset_ch = 1;
 577                 }
 578                 if (v_lo & TALITOS_CCPSR_LO_SOF) {
 579                         /* h/w dropped descriptor */
 580                         dev_err(dev, "single fetch fifo overflow error\n");
 581                         error = -EAGAIN;
 582                 }
 583                 if (v_lo & TALITOS_CCPSR_LO_MDTE)
 584                         dev_err(dev, "master data transfer error\n");
 585                 if (v_lo & TALITOS_CCPSR_LO_SGDLZ)
 586                         dev_err(dev, is_sec1 ? "pointeur not complete error\n"
 587                                              : "s/g data length zero error\n");
 588                 if (v_lo & TALITOS_CCPSR_LO_FPZ)
 589                         dev_err(dev, is_sec1 ? "parity error\n"
 590                                              : "fetch pointer zero error\n");
 591                 if (v_lo & TALITOS_CCPSR_LO_IDH)
 592                         dev_err(dev, "illegal descriptor header error\n");
 593                 if (v_lo & TALITOS_CCPSR_LO_IEU)
 594                         dev_err(dev, is_sec1 ? "static assignment error\n"
 595                                              : "invalid exec unit error\n");
 596                 if (v_lo & TALITOS_CCPSR_LO_EU)
 597                         report_eu_error(dev, ch, current_desc_hdr(dev, ch));
 598                 if (!is_sec1) {
 599                         if (v_lo & TALITOS_CCPSR_LO_GB)
 600                                 dev_err(dev, "gather boundary error\n");
 601                         if (v_lo & TALITOS_CCPSR_LO_GRL)
 602                                 dev_err(dev, "gather return/length error\n");
 603                         if (v_lo & TALITOS_CCPSR_LO_SB)
 604                                 dev_err(dev, "scatter boundary error\n");
 605                         if (v_lo & TALITOS_CCPSR_LO_SRL)
 606                                 dev_err(dev, "scatter return/length error\n");
 607                 }
 608
 609                 flush_channel(dev, ch, error, reset_ch);
 610
 611                 if (reset_ch) {
 612                         reset_channel(dev, ch);
 613                 } else {
 614                         setbits32(priv->chan[ch].reg + TALITOS_CCCR,
 615                                   TALITOS2_CCCR_CONT);
 616                         setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, 0);
 617                         while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) &
 618                                TALITOS2_CCCR_CONT) && --timeout)
 619                                 cpu_relax();
 620                         if (timeout == 0) {
 621                                 dev_err(dev, "failed to restart channel %d\n",
 622                                         ch);
 623                                 reset_dev = 1;
 624                         }
 625                 }
 626         }
 627         if (reset_dev || (is_sec1 && isr & ~TALITOS1_ISR_4CHERR) ||
 628             (!is_sec1 && isr & ~TALITOS2_ISR_4CHERR) || isr_lo) {
 629                 if (is_sec1 && (isr_lo & TALITOS1_ISR_TEA_ERR))
 630                         dev_err(dev, "TEA error: ISR 0x%08x_%08x\n",
 631                                 isr, isr_lo);
 632                 else
 633                         dev_err(dev, "done overflow, internal time out, or "
 634                                 "rngu error: ISR 0x%08x_%08x\n", isr, isr_lo);
 635
 636                 /* purge request queues */
 637                 for (ch = 0; ch < priv->num_channels; ch++)
 638                         flush_channel(dev, ch, -EIO, 1);
 639
 640                 /* reset and reinitialize the device */
 641                 init_device(dev);
 642         }
 643 }
 644
 645 #define DEF_TALITOS1_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet)          \
 646 static irqreturn_t talitos1_interrupt_##name(int irq, void *data)              \
 647 {                                                                              \
 648         struct device *dev = data;                                             \
 649         struct talitos_private *priv = dev_get_drvdata(dev);                   \
 650         u32 isr, isr_lo;                                                       \
 651         unsigned long flags;                                                   \
 652                                                                                \
 653         spin_lock_irqsave(&priv->reg_lock, flags);                             \
 654         isr = in_be32(priv->reg + TALITOS_ISR);                                \
 655         isr_lo = in_be32(priv->reg + TALITOS_ISR_LO);                          \
 656         /* Acknowledge interrupt */                                            \
 657         out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \
 658         out_be32(priv->reg + TALITOS_ICR_LO, isr_lo);                          \
 659                                                                                \
 660         if (unlikely(isr & ch_err_mask || isr_lo & TALITOS1_IMR_LO_INIT)) {    \
 661                 spin_unlock_irqrestore(&priv->reg_lock, flags);                \
 662                 talitos_error(dev, isr & ch_err_mask, isr_lo);                 \
 663         }                                                                      \
 664         else {                                                                 \
 665                 if (likely(isr & ch_done_mask)) {                              \
 666                         /* mask further done interrupts. */                    \
 667                         setbits32(priv->reg + TALITOS_IMR, ch_done_mask);      \
 668                         /* done_task will unmask done interrupts at exit */    \
 669                         tasklet_schedule(&priv->done_task[tlet]);              \
 670                 }                                                              \
 671                 spin_unlock_irqrestore(&priv->reg_lock, flags);                \
 672         }                                                                      \
 673                                                                                \
 674         return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED :  \
 675                                                                 IRQ_NONE;      \
 676 }
 677
 678 DEF_TALITOS1_INTERRUPT(4ch, TALITOS1_ISR_4CHDONE, TALITOS1_ISR_4CHERR, 0)
 679
 680 #define DEF_TALITOS2_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet)          \
 681 static irqreturn_t talitos2_interrupt_##name(int irq, void *data)              \
 682 {                                                                              \
 683         struct device *dev = data;                                             \
 684         struct talitos_private *priv = dev_get_drvdata(dev);                   \
 685         u32 isr, isr_lo;                                                       \
 686         unsigned long flags;                                                   \
 687                                                                                \
 688         spin_lock_irqsave(&priv->reg_lock, flags);                             \
 689         isr = in_be32(priv->reg + TALITOS_ISR);                                \
 690         isr_lo = in_be32(priv->reg + TALITOS_ISR_LO);                          \
 691         /* Acknowledge interrupt */                                            \
 692         out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \
 693         out_be32(priv->reg + TALITOS_ICR_LO, isr_lo);                          \
 694                                                                                \
 695         if (unlikely(isr & ch_err_mask || isr_lo)) {                           \
 696                 spin_unlock_irqrestore(&priv->reg_lock, flags);                \
 697                 talitos_error(dev, isr & ch_err_mask, isr_lo);                 \
 698         }                                                                      \
 699         else {                                                                 \
 700                 if (likely(isr & ch_done_mask)) {                              \
 701                         /* mask further done interrupts. */                    \
 702                         clrbits32(priv->reg + TALITOS_IMR, ch_done_mask);      \
 703                         /* done_task will unmask done interrupts at exit */    \
 704                         tasklet_schedule(&priv->done_task[tlet]);              \
 705                 }                                                              \
 706                 spin_unlock_irqrestore(&priv->reg_lock, flags);                \
 707         }                                                                      \
 708                                                                                \
 709         return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED :  \
 710                                                                 IRQ_NONE;      \
 711 }
 712
 713 DEF_TALITOS2_INTERRUPT(4ch, TALITOS2_ISR_4CHDONE, TALITOS2_ISR_4CHERR, 0)
 714 DEF_TALITOS2_INTERRUPT(ch0_2, TALITOS2_ISR_CH_0_2_DONE, TALITOS2_ISR_CH_0_2_ERR,
 715                        0)
 716 DEF_TALITOS2_INTERRUPT(ch1_3, TALITOS2_ISR_CH_1_3_DONE, TALITOS2_ISR_CH_1_3_ERR,
 717                        1)
 718
 719 /*
 720  * hwrng
 721  */
 722 static int talitos_rng_data_present(struct hwrng *rng, int wait)
 723 {
 724         struct device *dev = (struct device *)rng->priv;
 725         struct talitos_private *priv = dev_get_drvdata(dev);
 726         u32 ofl;
 727         int i;
 728
 729         for (i = 0; i < 20; i++) {
 730                 ofl = in_be32(priv->reg_rngu + TALITOS_EUSR_LO) &
 731                       TALITOS_RNGUSR_LO_OFL;
 732                 if (ofl || !wait)
 733                         break;
 734                 udelay(10);
 735         }
 736
 737         return !!ofl;
 738 }
 739
 740 static int talitos_rng_data_read(struct hwrng *rng, u32 *data)
 741 {
 742         struct device *dev = (struct device *)rng->priv;
 743         struct talitos_private *priv = dev_get_drvdata(dev);
 744
 745         /* rng fifo requires 64-bit accesses */
 746         *data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO);
 747         *data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO_LO);
 748
 749         return sizeof(u32);
 750 }
 751
 752 static int talitos_rng_init(struct hwrng *rng)
 753 {
 754         struct device *dev = (struct device *)rng->priv;
 755         struct talitos_private *priv = dev_get_drvdata(dev);
 756         unsigned int timeout = TALITOS_TIMEOUT;
 757
 758         setbits32(priv->reg_rngu + TALITOS_EURCR_LO, TALITOS_RNGURCR_LO_SR);
 759         while (!(in_be32(priv->reg_rngu + TALITOS_EUSR_LO)
 760                  & TALITOS_RNGUSR_LO_RD)
 761                && --timeout)
 762                 cpu_relax();
 763         if (timeout == 0) {
 764                 dev_err(dev, "failed to reset rng hw\n");
 765                 return -ENODEV;
 766         }
 767
 768         /* start generating */
 769         setbits32(priv->reg_rngu + TALITOS_EUDSR_LO, 0);
 770
 771         return 0;
 772 }
 773
 774 static int talitos_register_rng(struct device *dev)
 775 {
 776         struct talitos_private *priv = dev_get_drvdata(dev);
 777         int err;
 778
 779         priv->rng.name          = dev_driver_string(dev),
 780         priv->rng.init          = talitos_rng_init,
 781         priv->rng.data_present  = talitos_rng_data_present,
 782         priv->rng.data_read     = talitos_rng_data_read,
 783         priv->rng.priv          = (unsigned long)dev;
 784
 785         err = hwrng_register(&priv->rng);
 786         if (!err)
 787                 priv->rng_registered = true;
 788
 789         return err;
 790 }
 791
 792 static void talitos_unregister_rng(struct device *dev)
 793 {
 794         struct talitos_private *priv = dev_get_drvdata(dev);
 795
 796         if (!priv->rng_registered)
 797                 return;
 798
 799         hwrng_unregister(&priv->rng);
 800         priv->rng_registered = false;
 801 }
 802
 803 /*
 804  * crypto alg
 805  */
 806 #define TALITOS_CRA_PRIORITY            3000
 807 #define TALITOS_MAX_KEY_SIZE            96
 808 #define TALITOS_MAX_IV_LENGTH           16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
 809
 810 struct talitos_ctx {
 811         struct device *dev;
 812         int ch;
 813         __be32 desc_hdr_template;
 814         u8 key[TALITOS_MAX_KEY_SIZE];
 815         u8 iv[TALITOS_MAX_IV_LENGTH];
 816         unsigned int keylen;
 817         unsigned int enckeylen;
 818         unsigned int authkeylen;
 819 };
 820
 821 #define HASH_MAX_BLOCK_SIZE             SHA512_BLOCK_SIZE
 822 #define TALITOS_MDEU_MAX_CONTEXT_SIZE   TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512
 823
 824 struct talitos_ahash_req_ctx {
 825         u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)];
 826         unsigned int hw_context_size;
 827         u8 buf[HASH_MAX_BLOCK_SIZE];
 828         u8 bufnext[HASH_MAX_BLOCK_SIZE];
 829         unsigned int swinit;
 830         unsigned int first;
 831         unsigned int last;
 832         unsigned int to_hash_later;
 833         unsigned int nbuf;
 834         struct scatterlist bufsl[2];
 835         struct scatterlist *psrc;
 836 };
 837
 838 static int aead_setkey(struct crypto_aead *authenc,
 839                        const u8 *key, unsigned int keylen)
 840 {
 841         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
 842         struct crypto_authenc_keys keys;
 843
 844         if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
 845                 goto badkey;
 846
 847         if (keys.authkeylen + keys.enckeylen > TALITOS_MAX_KEY_SIZE)
 848                 goto badkey;
 849
 850         memcpy(ctx->key, keys.authkey, keys.authkeylen);
 851         memcpy(&ctx->key[keys.authkeylen], keys.enckey, keys.enckeylen);
 852
 853         ctx->keylen = keys.authkeylen + keys.enckeylen;
 854         ctx->enckeylen = keys.enckeylen;
 855         ctx->authkeylen = keys.authkeylen;
 856
 857         return 0;
 858
 859 badkey:
 860         crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN);
 861         return -EINVAL;
 862 }
 863
 864 /*
 865  * talitos_edesc - s/w-extended descriptor
 866  * @src_nents: number of segments in input scatterlist
 867  * @dst_nents: number of segments in output scatterlist
 868  * @icv_ool: whether ICV is out-of-line
 869  * @iv_dma: dma address of iv for checking continuity and link table
 870  * @dma_len: length of dma mapped link_tbl space
 871  * @dma_link_tbl: bus physical address of link_tbl/buf
 872  * @desc: h/w descriptor
 873  * @link_tbl: input and output h/w link tables (if {src,dst}_nents > 1) (SEC2)
 874  * @buf: input and output buffeur (if {src,dst}_nents > 1) (SEC1)
 875  *
 876  * if decrypting (with authcheck), or either one of src_nents or dst_nents
 877  * is greater than 1, an integrity check value is concatenated to the end
 878  * of link_tbl data
 879  */
 880 struct talitos_edesc {
 881         int src_nents;
 882         int dst_nents;
 883         bool icv_ool;
 884         dma_addr_t iv_dma;
 885         int dma_len;
 886         dma_addr_t dma_link_tbl;
 887         struct talitos_desc desc;
 888         union {
 889                 struct talitos_ptr link_tbl[0];
 890                 u8 buf[0];
 891         };
 892 };
 893
 894 static void talitos_sg_unmap(struct device *dev,
 895                              struct talitos_edesc *edesc,
 896                              struct scatterlist *src,
 897                              struct scatterlist *dst)
 898 {
 899         unsigned int src_nents = edesc->src_nents ? : 1;
 900         unsigned int dst_nents = edesc->dst_nents ? : 1;
 901
 902         if (src != dst) {
 903                 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
 904
 905                 if (dst) {
 906                         dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE);
 907                 }
 908         } else
 909                 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
 910 }
 911
 912 static void ipsec_esp_unmap(struct device *dev,
 913                             struct talitos_edesc *edesc,
 914                             struct aead_request *areq)
 915 {
 916         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[6], DMA_FROM_DEVICE);
 917         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[3], DMA_TO_DEVICE);
 918         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE);
 919         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[0], DMA_TO_DEVICE);
 920
 921         talitos_sg_unmap(dev, edesc, areq->src, areq->dst);
 922
 923         if (edesc->dma_len)
 924                 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
 925                                  DMA_BIDIRECTIONAL);
 926 }
 927
 928 /*
 929  * ipsec_esp descriptor callbacks
 930  */
 931 static void ipsec_esp_encrypt_done(struct device *dev,
 932                                    struct talitos_desc *desc, void *context,
 933                                    int err)
 934 {
 935         struct aead_request *areq = context;
 936         struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
 937         unsigned int authsize = crypto_aead_authsize(authenc);
 938         struct talitos_edesc *edesc;
 939         struct scatterlist *sg;
 940         void *icvdata;
 941
 942         edesc = container_of(desc, struct talitos_edesc, desc);
 943
 944         ipsec_esp_unmap(dev, edesc, areq);
 945
 946         /* copy the generated ICV to dst */
 947         if (edesc->icv_ool) {
 948                 icvdata = &edesc->link_tbl[edesc->src_nents +
 949                                            edesc->dst_nents + 2];
 950                 sg = sg_last(areq->dst, edesc->dst_nents);
 951                 memcpy((char *)sg_virt(sg) + sg->length - authsize,
 952                        icvdata, authsize);
 953         }
 954
 955         kfree(edesc);
 956
 957         aead_request_complete(areq, err);
 958 }
 959
 960 static void ipsec_esp_decrypt_swauth_done(struct device *dev,
 961                                           struct talitos_desc *desc,
 962                                           void *context, int err)
 963 {
 964         struct aead_request *req = context;
 965         struct crypto_aead *authenc = crypto_aead_reqtfm(req);
 966         unsigned int authsize = crypto_aead_authsize(authenc);
 967         struct talitos_edesc *edesc;
 968         struct scatterlist *sg;
 969         char *oicv, *icv;
 970
 971         edesc = container_of(desc, struct talitos_edesc, desc);
 972
 973         ipsec_esp_unmap(dev, edesc, req);
 974
 975         if (!err) {
 976                 /* auth check */
 977                 sg = sg_last(req->dst, edesc->dst_nents ? : 1);
 978                 icv = (char *)sg_virt(sg) + sg->length - authsize;
 979
 980                 if (edesc->dma_len) {
 981                         oicv = (char *)&edesc->link_tbl[edesc->src_nents +
 982                                                         edesc->dst_nents + 2];
 983                         if (edesc->icv_ool)
 984                                 icv = oicv + authsize;
 985                 } else
 986                         oicv = (char *)&edesc->link_tbl[0];
 987
 988                 err = crypto_memneq(oicv, icv, authsize) ? -EBADMSG : 0;
 989         }
 990
 991         kfree(edesc);
 992
 993         aead_request_complete(req, err);
 994 }
 995
 996 static void ipsec_esp_decrypt_hwauth_done(struct device *dev,
 997                                           struct talitos_desc *desc,
 998                                           void *context, int err)
 999 {
1000         struct aead_request *req = context;
1001         struct talitos_edesc *edesc;
1002
1003         edesc = container_of(desc, struct talitos_edesc, desc);
1004
1005         ipsec_esp_unmap(dev, edesc, req);
1006
1007         /* check ICV auth status */
1008         if (!err && ((desc->hdr_lo & DESC_HDR_LO_ICCR1_MASK) !=
1009                      DESC_HDR_LO_ICCR1_PASS))
1010                 err = -EBADMSG;
1011
1012         kfree(edesc);
1013
1014         aead_request_complete(req, err);
1015 }
1016
1017 /*
1018  * convert scatterlist to SEC h/w link table format
1019  * stop at cryptlen bytes
1020  */
1021 static int sg_to_link_tbl_offset(struct scatterlist *sg, int sg_count,
1022                                  unsigned int offset, int cryptlen,
1023                                  struct talitos_ptr *link_tbl_ptr)
1024 {
1025         int n_sg = sg_count;
1026         int count = 0;
1027
1028         while (cryptlen && sg && n_sg--) {
1029                 unsigned int len = sg_dma_len(sg);
1030
1031                 if (offset >= len) {
1032                         offset -= len;
1033                         goto next;
1034                 }
1035
1036                 len -= offset;
1037
1038                 if (len > cryptlen)
1039                         len = cryptlen;
1040
1041                 to_talitos_ptr(link_tbl_ptr + count,
1042                                sg_dma_address(sg) + offset, 0);
1043                 link_tbl_ptr[count].len = cpu_to_be16(len);
1044                 link_tbl_ptr[count].j_extent = 0;
1045                 count++;
1046                 cryptlen -= len;
1047                 offset = 0;
1048
1049 next:
1050                 sg = sg_next(sg);
1051         }
1052
1053         /* tag end of link table */
1054         if (count > 0)
1055                 link_tbl_ptr[count - 1].j_extent = DESC_PTR_LNKTBL_RETURN;
1056
1057         return count;
1058 }
1059
1060 static inline int sg_to_link_tbl(struct scatterlist *sg, int sg_count,
1061                                  int cryptlen,
1062                                  struct talitos_ptr *link_tbl_ptr)
1063 {
1064         return sg_to_link_tbl_offset(sg, sg_count, 0, cryptlen,
1065                                      link_tbl_ptr);
1066 }
1067
1068 /*
1069  * fill in and submit ipsec_esp descriptor
1070  */
1071 static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq,
1072                      void (*callback)(struct device *dev,
1073                                       struct talitos_desc *desc,
1074                                       void *context, int error))
1075 {
1076         struct crypto_aead *aead = crypto_aead_reqtfm(areq);
1077         unsigned int authsize = crypto_aead_authsize(aead);
1078         struct talitos_ctx *ctx = crypto_aead_ctx(aead);
1079         struct device *dev = ctx->dev;
1080         struct talitos_desc *desc = &edesc->desc;
1081         unsigned int cryptlen = areq->cryptlen;
1082         unsigned int ivsize = crypto_aead_ivsize(aead);
1083         int tbl_off = 0;
1084         int sg_count, ret;
1085         int sg_link_tbl_len;
1086
1087         /* hmac key */
1088         map_single_talitos_ptr(dev, &desc->ptr[0], ctx->authkeylen, &ctx->key,
1089                                DMA_TO_DEVICE);
1090
1091         sg_count = dma_map_sg(dev, areq->src, edesc->src_nents ?: 1,
1092                               (areq->src == areq->dst) ? DMA_BIDIRECTIONAL
1093                                                            : DMA_TO_DEVICE);
1094         /* hmac data */
1095         desc->ptr[1].len = cpu_to_be16(areq->assoclen);
1096         if (sg_count > 1 &&
1097             (ret = sg_to_link_tbl_offset(areq->src, sg_count, 0,
1098                                          areq->assoclen,
1099                                          &edesc->link_tbl[tbl_off])) > 1) {
1100                 to_talitos_ptr(&desc->ptr[1], edesc->dma_link_tbl + tbl_off *
1101                                sizeof(struct talitos_ptr), 0);
1102                 desc->ptr[1].j_extent = DESC_PTR_LNKTBL_JUMP;
1103
1104                 dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1105                                            edesc->dma_len, DMA_BIDIRECTIONAL);
1106
1107                 tbl_off += ret;
1108         } else {
1109                 to_talitos_ptr(&desc->ptr[1], sg_dma_address(areq->src), 0);
1110                 desc->ptr[1].j_extent = 0;
1111         }
1112
1113         /* cipher iv */
1114         to_talitos_ptr(&desc->ptr[2], edesc->iv_dma, 0);
1115         desc->ptr[2].len = cpu_to_be16(ivsize);
1116         desc->ptr[2].j_extent = 0;
1117
1118         /* cipher key */
1119         map_single_talitos_ptr(dev, &desc->ptr[3], ctx->enckeylen,
1120                                (char *)&ctx->key + ctx->authkeylen,
1121                                DMA_TO_DEVICE);
1122
1123         /*
1124          * cipher in
1125          * map and adjust cipher len to aead request cryptlen.
1126          * extent is bytes of HMAC postpended to ciphertext,
1127          * typically 12 for ipsec
1128          */
1129         desc->ptr[4].len = cpu_to_be16(cryptlen);
1130         desc->ptr[4].j_extent = authsize;
1131
1132         sg_link_tbl_len = cryptlen;
1133         if (edesc->desc.hdr & DESC_HDR_MODE1_MDEU_CICV)
1134                 sg_link_tbl_len += authsize;
1135
1136         if (sg_count == 1) {
1137                 to_talitos_ptr(&desc->ptr[4], sg_dma_address(areq->src) +
1138                                areq->assoclen, 0);
1139         } else if ((ret = sg_to_link_tbl_offset(areq->src, sg_count,
1140                                                 areq->assoclen, sg_link_tbl_len,
1141                                                 &edesc->link_tbl[tbl_off])) >
1142                    1) {
1143                 desc->ptr[4].j_extent |= DESC_PTR_LNKTBL_JUMP;
1144                 to_talitos_ptr(&desc->ptr[4], edesc->dma_link_tbl +
1145                                               tbl_off *
1146                                               sizeof(struct talitos_ptr), 0);
1147                 dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1148                                            edesc->dma_len,
1149                                            DMA_BIDIRECTIONAL);
1150                 tbl_off += ret;
1151         } else {
1152                 copy_talitos_ptr(&desc->ptr[4], &edesc->link_tbl[tbl_off], 0);
1153         }
1154
1155         /* cipher out */
1156         desc->ptr[5].len = cpu_to_be16(cryptlen);
1157         desc->ptr[5].j_extent = authsize;
1158
1159         if (areq->src != areq->dst)
1160                 sg_count = dma_map_sg(dev, areq->dst, edesc->dst_nents ? : 1,
1161                                       DMA_FROM_DEVICE);
1162
1163         edesc->icv_ool = false;
1164
1165         if (sg_count == 1) {
1166                 to_talitos_ptr(&desc->ptr[5], sg_dma_address(areq->dst) +
1167                                areq->assoclen, 0);
1168         } else if ((sg_count =
1169                         sg_to_link_tbl_offset(areq->dst, sg_count,
1170                                               areq->assoclen, cryptlen,
1171                                               &edesc->link_tbl[tbl_off])) > 1) {
1172                 struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off];
1173
1174                 to_talitos_ptr(&desc->ptr[5], edesc->dma_link_tbl +
1175                                tbl_off * sizeof(struct talitos_ptr), 0);
1176
1177                 /* Add an entry to the link table for ICV data */
1178                 tbl_ptr += sg_count - 1;
1179                 tbl_ptr->j_extent = 0;
1180                 tbl_ptr++;
1181                 tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN;
1182                 tbl_ptr->len = cpu_to_be16(authsize);
1183
1184                 /* icv data follows link tables */
1185                 to_talitos_ptr(tbl_ptr, edesc->dma_link_tbl +
1186                                         (edesc->src_nents + edesc->dst_nents +
1187                                          2) * sizeof(struct talitos_ptr) +
1188                                         authsize, 0);
1189                 desc->ptr[5].j_extent |= DESC_PTR_LNKTBL_JUMP;
1190                 dma_sync_single_for_device(ctx->dev, edesc->dma_link_tbl,
1191                                            edesc->dma_len, DMA_BIDIRECTIONAL);
1192
1193                 edesc->icv_ool = true;
1194         } else {
1195                 copy_talitos_ptr(&desc->ptr[5], &edesc->link_tbl[tbl_off], 0);
1196         }
1197
1198         /* iv out */
1199         map_single_talitos_ptr(dev, &desc->ptr[6], ivsize, ctx->iv,
1200                                DMA_FROM_DEVICE);
1201
1202         ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
1203         if (ret != -EINPROGRESS) {
1204                 ipsec_esp_unmap(dev, edesc, areq);
1205                 kfree(edesc);
1206         }
1207         return ret;
1208 }
1209
1210 /*
1211  * allocate and map the extended descriptor
1212  */
1213 static struct talitos_edesc *talitos_edesc_alloc(struct device *dev,
1214                                                  struct scatterlist *src,
1215                                                  struct scatterlist *dst,
1216                                                  u8 *iv,
1217                                                  unsigned int assoclen,
1218                                                  unsigned int cryptlen,
1219                                                  unsigned int authsize,
1220                                                  unsigned int ivsize,
1221                                                  int icv_stashing,
1222                                                  u32 cryptoflags,
1223                                                  bool encrypt)
1224 {
1225         struct talitos_edesc *edesc;
1226         int src_nents, dst_nents, alloc_len, dma_len;
1227         dma_addr_t iv_dma = 0;
1228         gfp_t flags = cryptoflags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
1229                       GFP_ATOMIC;
1230         struct talitos_private *priv = dev_get_drvdata(dev);
1231         bool is_sec1 = has_ftr_sec1(priv);
1232         int max_len = is_sec1 ? TALITOS1_MAX_DATA_LEN : TALITOS2_MAX_DATA_LEN;
1233         void *err;
1234
1235         if (cryptlen + authsize > max_len) {
1236                 dev_err(dev, "length exceeds h/w max limit\n");
1237                 return ERR_PTR(-EINVAL);
1238         }
1239
1240         if (ivsize)
1241                 iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE);
1242
1243         if (!dst || dst == src) {
1244                 src_nents = sg_nents_for_len(src,
1245                                              assoclen + cryptlen + authsize);
1246                 if (src_nents < 0) {
1247                         dev_err(dev, "Invalid number of src SG.\n");
1248                         err = ERR_PTR(-EINVAL);
1249                         goto error_sg;
1250                 }
1251                 src_nents = (src_nents == 1) ? 0 : src_nents;
1252                 dst_nents = dst ? src_nents : 0;
1253         } else { /* dst && dst != src*/
1254                 src_nents = sg_nents_for_len(src, assoclen + cryptlen +
1255                                                  (encrypt ? 0 : authsize));
1256                 if (src_nents < 0) {
1257                         dev_err(dev, "Invalid number of src SG.\n");
1258                         err = ERR_PTR(-EINVAL);
1259                         goto error_sg;
1260                 }
1261                 src_nents = (src_nents == 1) ? 0 : src_nents;
1262                 dst_nents = sg_nents_for_len(dst, assoclen + cryptlen +
1263                                                  (encrypt ? authsize : 0));
1264                 if (dst_nents < 0) {
1265                         dev_err(dev, "Invalid number of dst SG.\n");
1266                         err = ERR_PTR(-EINVAL);
1267                         goto error_sg;
1268                 }
1269                 dst_nents = (dst_nents == 1) ? 0 : dst_nents;
1270         }
1271
1272         /*
1273          * allocate space for base edesc plus the link tables,
1274          * allowing for two separate entries for AD and generated ICV (+ 2),
1275          * and space for two sets of ICVs (stashed and generated)
1276          */
1277         alloc_len = sizeof(struct talitos_edesc);
1278         if (src_nents || dst_nents) {
1279                 if (is_sec1)
1280                         dma_len = (src_nents ? cryptlen : 0) +
1281                                   (dst_nents ? cryptlen : 0);
1282                 else
1283                         dma_len = (src_nents + dst_nents + 2) *
1284                                   sizeof(struct talitos_ptr) + authsize * 2;
1285                 alloc_len += dma_len;
1286         } else {
1287                 dma_len = 0;
1288                 alloc_len += icv_stashing ? authsize : 0;
1289         }
1290
1291         edesc = kmalloc(alloc_len, GFP_DMA | flags);
1292         if (!edesc) {
1293                 dev_err(dev, "could not allocate edescriptor\n");
1294                 err = ERR_PTR(-ENOMEM);
1295                 goto error_sg;
1296         }
1297
1298         edesc->src_nents = src_nents;
1299         edesc->dst_nents = dst_nents;
1300         edesc->iv_dma = iv_dma;
1301         edesc->dma_len = dma_len;
1302         if (dma_len)
1303                 edesc->dma_link_tbl = dma_map_single(dev, &edesc->link_tbl[0],
1304                                                      edesc->dma_len,
1305                                                      DMA_BIDIRECTIONAL);
1306
1307         return edesc;
1308 error_sg:
1309         if (iv_dma)
1310                 dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE);
1311         return err;
1312 }
1313
1314 static struct talitos_edesc *aead_edesc_alloc(struct aead_request *areq, u8 *iv,
1315                                               int icv_stashing, bool encrypt)
1316 {
1317         struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
1318         unsigned int authsize = crypto_aead_authsize(authenc);
1319         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1320         unsigned int ivsize = crypto_aead_ivsize(authenc);
1321
1322         return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst,
1323                                    iv, areq->assoclen, areq->cryptlen,
1324                                    authsize, ivsize, icv_stashing,
1325                                    areq->base.flags, encrypt);
1326 }
1327
1328 static int aead_encrypt(struct aead_request *req)
1329 {
1330         struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1331         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1332         struct talitos_edesc *edesc;
1333
1334         /* allocate extended descriptor */
1335         edesc = aead_edesc_alloc(req, req->iv, 0, true);
1336         if (IS_ERR(edesc))
1337                 return PTR_ERR(edesc);
1338
1339         /* set encrypt */
1340         edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1341
1342         return ipsec_esp(edesc, req, ipsec_esp_encrypt_done);
1343 }
1344
1345 static int aead_decrypt(struct aead_request *req)
1346 {
1347         struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1348         unsigned int authsize = crypto_aead_authsize(authenc);
1349         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1350         struct talitos_private *priv = dev_get_drvdata(ctx->dev);
1351         struct talitos_edesc *edesc;
1352         struct scatterlist *sg;
1353         void *icvdata;
1354
1355         req->cryptlen -= authsize;
1356
1357         /* allocate extended descriptor */
1358         edesc = aead_edesc_alloc(req, req->iv, 1, false);
1359         if (IS_ERR(edesc))
1360                 return PTR_ERR(edesc);
1361
1362         if ((priv->features & TALITOS_FTR_HW_AUTH_CHECK) &&
1363             ((!edesc->src_nents && !edesc->dst_nents) ||
1364              priv->features & TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT)) {
1365
1366                 /* decrypt and check the ICV */
1367                 edesc->desc.hdr = ctx->desc_hdr_template |
1368                                   DESC_HDR_DIR_INBOUND |
1369                                   DESC_HDR_MODE1_MDEU_CICV;
1370
1371                 /* reset integrity check result bits */
1372                 edesc->desc.hdr_lo = 0;
1373
1374                 return ipsec_esp(edesc, req, ipsec_esp_decrypt_hwauth_done);
1375         }
1376
1377         /* Have to check the ICV with software */
1378         edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
1379
1380         /* stash incoming ICV for later cmp with ICV generated by the h/w */
1381         if (edesc->dma_len)
1382                 icvdata = (char *)&edesc->link_tbl[edesc->src_nents +
1383                                                    edesc->dst_nents + 2];
1384         else
1385                 icvdata = &edesc->link_tbl[0];
1386
1387         sg = sg_last(req->src, edesc->src_nents ? : 1);
1388
1389         memcpy(icvdata, (char *)sg_virt(sg) + sg->length - authsize, authsize);
1390
1391         return ipsec_esp(edesc, req, ipsec_esp_decrypt_swauth_done);
1392 }
1393
1394 static int ablkcipher_setkey(struct crypto_ablkcipher *cipher,
1395                              const u8 *key, unsigned int keylen)
1396 {
1397         struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1398
1399         memcpy(&ctx->key, key, keylen);
1400         ctx->keylen = keylen;
1401
1402         return 0;
1403 }
1404
1405 static void unmap_sg_talitos_ptr(struct device *dev, struct scatterlist *src,
1406                                  struct scatterlist *dst, unsigned int len,
1407                                  struct talitos_edesc *edesc)
1408 {
1409         struct talitos_private *priv = dev_get_drvdata(dev);
1410         bool is_sec1 = has_ftr_sec1(priv);
1411
1412         if (is_sec1) {
1413                 if (!edesc->src_nents) {
1414                         dma_unmap_sg(dev, src, 1,
1415                                      dst != src ? DMA_TO_DEVICE
1416                                                 : DMA_BIDIRECTIONAL);
1417                 }
1418                 if (dst && edesc->dst_nents) {
1419                         dma_sync_single_for_device(dev,
1420                                                    edesc->dma_link_tbl + len,
1421                                                    len, DMA_FROM_DEVICE);
1422                         sg_copy_from_buffer(dst, edesc->dst_nents ? : 1,
1423                                             edesc->buf + len, len);
1424                 } else if (dst && dst != src) {
1425                         dma_unmap_sg(dev, dst, 1, DMA_FROM_DEVICE);
1426                 }
1427         } else {
1428                 talitos_sg_unmap(dev, edesc, src, dst);
1429         }
1430 }
1431
1432 static void common_nonsnoop_unmap(struct device *dev,
1433                                   struct talitos_edesc *edesc,
1434                                   struct ablkcipher_request *areq)
1435 {
1436         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);
1437
1438         unmap_sg_talitos_ptr(dev, areq->src, areq->dst, areq->nbytes, edesc);
1439         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE);
1440         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1], DMA_TO_DEVICE);
1441
1442         if (edesc->dma_len)
1443                 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1444                                  DMA_BIDIRECTIONAL);
1445 }
1446
1447 static void ablkcipher_done(struct device *dev,
1448                             struct talitos_desc *desc, void *context,
1449                             int err)
1450 {
1451         struct ablkcipher_request *areq = context;
1452         struct talitos_edesc *edesc;
1453
1454         edesc = container_of(desc, struct talitos_edesc, desc);
1455
1456         common_nonsnoop_unmap(dev, edesc, areq);
1457
1458         kfree(edesc);
1459
1460         areq->base.complete(&areq->base, err);
1461 }
1462
1463 int map_sg_in_talitos_ptr(struct device *dev, struct scatterlist *src,
1464                           unsigned int len, struct talitos_edesc *edesc,
1465                           enum dma_data_direction dir, struct talitos_ptr *ptr)
1466 {
1467         int sg_count;
1468         struct talitos_private *priv = dev_get_drvdata(dev);
1469         bool is_sec1 = has_ftr_sec1(priv);
1470
1471         to_talitos_ptr_len(ptr, len, is_sec1);
1472
1473         if (is_sec1) {
1474                 sg_count = edesc->src_nents ? : 1;
1475
1476                 if (sg_count == 1) {
1477                         dma_map_sg(dev, src, 1, dir);
1478                         to_talitos_ptr(ptr, sg_dma_address(src), is_sec1);
1479                 } else {
1480                         sg_copy_to_buffer(src, sg_count, edesc->buf, len);
1481                         to_talitos_ptr(ptr, edesc->dma_link_tbl, is_sec1);
1482                         dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1483                                                    len, DMA_TO_DEVICE);
1484                 }
1485         } else {
1486                 to_talitos_ptr_extent_clear(ptr, is_sec1);
1487
1488                 sg_count = dma_map_sg(dev, src, edesc->src_nents ? : 1, dir);
1489
1490                 if (sg_count == 1) {
1491                         to_talitos_ptr(ptr, sg_dma_address(src), is_sec1);
1492                 } else {
1493                         sg_count = sg_to_link_tbl(src, sg_count, len,
1494                                                   &edesc->link_tbl[0]);
1495                         if (sg_count > 1) {
1496                                 to_talitos_ptr(ptr, edesc->dma_link_tbl, 0);
1497                                 ptr->j_extent |= DESC_PTR_LNKTBL_JUMP;
1498                                 dma_sync_single_for_device(dev,
1499                                                            edesc->dma_link_tbl,
1500                                                            edesc->dma_len,
1501                                                            DMA_BIDIRECTIONAL);
1502                         } else {
1503                                 /* Only one segment now, so no link tbl needed*/
1504                                 to_talitos_ptr(ptr, sg_dma_address(src),
1505                                                is_sec1);
1506                         }
1507                 }
1508         }
1509         return sg_count;
1510 }
1511
1512 void map_sg_out_talitos_ptr(struct device *dev, struct scatterlist *dst,
1513                             unsigned int len, struct talitos_edesc *edesc,
1514                             enum dma_data_direction dir,
1515                             struct talitos_ptr *ptr, int sg_count)
1516 {
1517         struct talitos_private *priv = dev_get_drvdata(dev);
1518         bool is_sec1 = has_ftr_sec1(priv);
1519
1520         if (dir != DMA_NONE)
1521                 sg_count = dma_map_sg(dev, dst, edesc->dst_nents ? : 1, dir);
1522
1523         to_talitos_ptr_len(ptr, len, is_sec1);
1524
1525         if (is_sec1) {
1526                 if (sg_count == 1) {
1527                         if (dir != DMA_NONE)
1528                                 dma_map_sg(dev, dst, 1, dir);
1529                         to_talitos_ptr(ptr, sg_dma_address(dst), is_sec1);
1530                 } else {
1531                         to_talitos_ptr(ptr, edesc->dma_link_tbl + len, is_sec1);
1532                         dma_sync_single_for_device(dev,
1533                                                    edesc->dma_link_tbl + len,
1534                                                    len, DMA_FROM_DEVICE);
1535                 }
1536         } else {
1537                 to_talitos_ptr_extent_clear(ptr, is_sec1);
1538
1539                 if (sg_count == 1) {
1540                         to_talitos_ptr(ptr, sg_dma_address(dst), is_sec1);
1541                 } else {
1542                         struct talitos_ptr *link_tbl_ptr =
1543                                 &edesc->link_tbl[edesc->src_nents + 1];
1544
1545                         to_talitos_ptr(ptr, edesc->dma_link_tbl +
1546                                             (edesc->src_nents + 1) *
1547                                              sizeof(struct talitos_ptr), 0);
1548                         ptr->j_extent |= DESC_PTR_LNKTBL_JUMP;
1549                         sg_to_link_tbl(dst, sg_count, len, link_tbl_ptr);
1550                         dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1551                                                    edesc->dma_len,
1552                                                    DMA_BIDIRECTIONAL);
1553                 }
1554         }
1555 }
1556
1557 static int common_nonsnoop(struct talitos_edesc *edesc,
1558                            struct ablkcipher_request *areq,
1559                            void (*callback) (struct device *dev,
1560                                              struct talitos_desc *desc,
1561                                              void *context, int error))
1562 {
1563         struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1564         struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1565         struct device *dev = ctx->dev;
1566         struct talitos_desc *desc = &edesc->desc;
1567         unsigned int cryptlen = areq->nbytes;
1568         unsigned int ivsize = crypto_ablkcipher_ivsize(cipher);
1569         int sg_count, ret;
1570         struct talitos_private *priv = dev_get_drvdata(dev);
1571         bool is_sec1 = has_ftr_sec1(priv);
1572
1573         /* first DWORD empty */
1574         desc->ptr[0] = zero_entry;
1575
1576         /* cipher iv */
1577         to_talitos_ptr(&desc->ptr[1], edesc->iv_dma, is_sec1);
1578         to_talitos_ptr_len(&desc->ptr[1], ivsize, is_sec1);
1579         to_talitos_ptr_extent_clear(&desc->ptr[1], is_sec1);
1580
1581         /* cipher key */
1582         map_single_talitos_ptr(dev, &desc->ptr[2], ctx->keylen,
1583                                (char *)&ctx->key, DMA_TO_DEVICE);
1584
1585         /*
1586          * cipher in
1587          */
1588         sg_count = map_sg_in_talitos_ptr(dev, areq->src, cryptlen, edesc,
1589                                          (areq->src == areq->dst) ?
1590                                           DMA_BIDIRECTIONAL : DMA_TO_DEVICE,
1591                                           &desc->ptr[3]);
1592
1593         /* cipher out */
1594         map_sg_out_talitos_ptr(dev, areq->dst, cryptlen, edesc,
1595                                (areq->src == areq->dst) ? DMA_NONE
1596                                                         : DMA_FROM_DEVICE,
1597                                &desc->ptr[4], sg_count);
1598
1599         /* iv out */
1600         map_single_talitos_ptr(dev, &desc->ptr[5], ivsize, ctx->iv,
1601                                DMA_FROM_DEVICE);
1602
1603         /* last DWORD empty */
1604         desc->ptr[6] = zero_entry;
1605
1606         ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
1607         if (ret != -EINPROGRESS) {
1608                 common_nonsnoop_unmap(dev, edesc, areq);
1609                 kfree(edesc);
1610         }
1611         return ret;
1612 }
1613
1614 static struct talitos_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request *
1615                                                     areq, bool encrypt)
1616 {
1617         struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1618         struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1619         unsigned int ivsize = crypto_ablkcipher_ivsize(cipher);
1620
1621         return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst,
1622                                    areq->info, 0, areq->nbytes, 0, ivsize, 0,
1623                                    areq->base.flags, encrypt);
1624 }
1625
1626 static int ablkcipher_encrypt(struct ablkcipher_request *areq)
1627 {
1628         struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1629         struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1630         struct talitos_edesc *edesc;
1631
1632         /* allocate extended descriptor */
1633         edesc = ablkcipher_edesc_alloc(areq, true);
1634         if (IS_ERR(edesc))
1635                 return PTR_ERR(edesc);
1636
1637         /* set encrypt */
1638         edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1639
1640         return common_nonsnoop(edesc, areq, ablkcipher_done);
1641 }
1642
1643 static int ablkcipher_decrypt(struct ablkcipher_request *areq)
1644 {
1645         struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1646         struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1647         struct talitos_edesc *edesc;
1648
1649         /* allocate extended descriptor */
1650         edesc = ablkcipher_edesc_alloc(areq, false);
1651         if (IS_ERR(edesc))
1652                 return PTR_ERR(edesc);
1653
1654         edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
1655
1656         return common_nonsnoop(edesc, areq, ablkcipher_done);
1657 }
1658
1659 static void common_nonsnoop_hash_unmap(struct device *dev,
1660                                        struct talitos_edesc *edesc,
1661                                        struct ahash_request *areq)
1662 {
1663         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1664         struct talitos_private *priv = dev_get_drvdata(dev);
1665         bool is_sec1 = has_ftr_sec1(priv);
1666
1667         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);
1668
1669         unmap_sg_talitos_ptr(dev, req_ctx->psrc, NULL, 0, edesc);
1670
1671         /* When using hashctx-in, must unmap it. */
1672         if (from_talitos_ptr_len(&edesc->desc.ptr[1], is_sec1))
1673                 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1],
1674                                          DMA_TO_DEVICE);
1675
1676         if (from_talitos_ptr_len(&edesc->desc.ptr[2], is_sec1))
1677                 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2],
1678                                          DMA_TO_DEVICE);
1679
1680         if (edesc->dma_len)
1681                 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1682                                  DMA_BIDIRECTIONAL);
1683
1684 }
1685
1686 static void ahash_done(struct device *dev,
1687                        struct talitos_desc *desc, void *context,
1688                        int err)
1689 {
1690         struct ahash_request *areq = context;
1691         struct talitos_edesc *edesc =
1692                  container_of(desc, struct talitos_edesc, desc);
1693         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1694
1695         if (!req_ctx->last && req_ctx->to_hash_later) {
1696                 /* Position any partial block for next update/final/finup */
1697                 memcpy(req_ctx->buf, req_ctx->bufnext, req_ctx->to_hash_later);
1698                 req_ctx->nbuf = req_ctx->to_hash_later;
1699         }
1700         common_nonsnoop_hash_unmap(dev, edesc, areq);
1701
1702         kfree(edesc);
1703
1704         areq->base.complete(&areq->base, err);
1705 }
1706
1707 /*
1708  * SEC1 doesn't like hashing of 0 sized message, so we do the padding
1709  * ourself and submit a padded block
1710  */
1711 void talitos_handle_buggy_hash(struct talitos_ctx *ctx,
1712                                struct talitos_edesc *edesc,
1713                                struct talitos_ptr *ptr)
1714 {
1715         static u8 padded_hash[64] = {
1716                 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1717                 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1718                 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1719                 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1720         };
1721
1722         pr_err_once("Bug in SEC1, padding ourself\n");
1723         edesc->desc.hdr &= ~DESC_HDR_MODE0_MDEU_PAD;
1724         map_single_talitos_ptr(ctx->dev, ptr, sizeof(padded_hash),
1725                                (char *)padded_hash, DMA_TO_DEVICE);
1726 }
1727
1728 static int common_nonsnoop_hash(struct talitos_edesc *edesc,
1729                                 struct ahash_request *areq, unsigned int length,
1730                                 void (*callback) (struct device *dev,
1731                                                   struct talitos_desc *desc,
1732                                                   void *context, int error))
1733 {
1734         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1735         struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1736         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1737         struct device *dev = ctx->dev;
1738         struct talitos_desc *desc = &edesc->desc;
1739         int ret;
1740         struct talitos_private *priv = dev_get_drvdata(dev);
1741         bool is_sec1 = has_ftr_sec1(priv);
1742
1743         /* first DWORD empty */
1744         desc->ptr[0] = zero_entry;
1745
1746         /* hash context in */
1747         if (!req_ctx->first || req_ctx->swinit) {
1748                 map_single_talitos_ptr(dev, &desc->ptr[1],
1749                                        req_ctx->hw_context_size,
1750                                        (char *)req_ctx->hw_context,
1751                                        DMA_TO_DEVICE);
1752                 req_ctx->swinit = 0;
1753         } else {
1754                 desc->ptr[1] = zero_entry;
1755                 /* Indicate next op is not the first. */
1756                 req_ctx->first = 0;
1757         }
1758
1759         /* HMAC key */
1760         if (ctx->keylen)
1761                 map_single_talitos_ptr(dev, &desc->ptr[2], ctx->keylen,
1762                                        (char *)&ctx->key, DMA_TO_DEVICE);
1763         else
1764                 desc->ptr[2] = zero_entry;
1765
1766         /*
1767          * data in
1768          */
1769         map_sg_in_talitos_ptr(dev, req_ctx->psrc, length, edesc,
1770                               DMA_TO_DEVICE, &desc->ptr[3]);
1771
1772         /* fifth DWORD empty */
1773         desc->ptr[4] = zero_entry;
1774
1775         /* hash/HMAC out -or- hash context out */
1776         if (req_ctx->last)
1777                 map_single_talitos_ptr(dev, &desc->ptr[5],
1778                                        crypto_ahash_digestsize(tfm),
1779                                        areq->result, DMA_FROM_DEVICE);
1780         else
1781                 map_single_talitos_ptr(dev, &desc->ptr[5],
1782                                        req_ctx->hw_context_size,
1783                                        req_ctx->hw_context, DMA_FROM_DEVICE);
1784
1785         /* last DWORD empty */
1786         desc->ptr[6] = zero_entry;
1787
1788         if (is_sec1 && from_talitos_ptr_len(&desc->ptr[3], true) == 0)
1789                 talitos_handle_buggy_hash(ctx, edesc, &desc->ptr[3]);
1790
1791         ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
1792         if (ret != -EINPROGRESS) {
1793                 common_nonsnoop_hash_unmap(dev, edesc, areq);
1794                 kfree(edesc);
1795         }
1796         return ret;
1797 }
1798
1799 static struct talitos_edesc *ahash_edesc_alloc(struct ahash_request *areq,
1800                                                unsigned int nbytes)
1801 {
1802         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1803         struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1804         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1805
1806         return talitos_edesc_alloc(ctx->dev, req_ctx->psrc, NULL, NULL, 0,
1807                                    nbytes, 0, 0, 0, areq->base.flags, false);
1808 }
1809
1810 static int ahash_init(struct ahash_request *areq)
1811 {
1812         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1813         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1814
1815         /* Initialize the context */
1816         req_ctx->nbuf = 0;
1817         req_ctx->first = 1; /* first indicates h/w must init its context */
1818         req_ctx->swinit = 0; /* assume h/w init of context */
1819         req_ctx->hw_context_size =
1820                 (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE)
1821                         ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256
1822                         : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512;
1823
1824         return 0;
1825 }
1826
1827 /*
1828  * on h/w without explicit sha224 support, we initialize h/w context
1829  * manually with sha224 constants, and tell it to run sha256.
1830  */
1831 static int ahash_init_sha224_swinit(struct ahash_request *areq)
1832 {
1833         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1834
1835         ahash_init(areq);
1836         req_ctx->swinit = 1;/* prevent h/w initting context with sha256 values*/
1837
1838         req_ctx->hw_context[0] = SHA224_H0;
1839         req_ctx->hw_context[1] = SHA224_H1;
1840         req_ctx->hw_context[2] = SHA224_H2;
1841         req_ctx->hw_context[3] = SHA224_H3;
1842         req_ctx->hw_context[4] = SHA224_H4;
1843         req_ctx->hw_context[5] = SHA224_H5;
1844         req_ctx->hw_context[6] = SHA224_H6;
1845         req_ctx->hw_context[7] = SHA224_H7;
1846
1847         /* init 64-bit count */
1848         req_ctx->hw_context[8] = 0;
1849         req_ctx->hw_context[9] = 0;
1850
1851         return 0;
1852 }
1853
1854 static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes)
1855 {
1856         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1857         struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1858         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1859         struct talitos_edesc *edesc;
1860         unsigned int blocksize =
1861                         crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
1862         unsigned int nbytes_to_hash;
1863         unsigned int to_hash_later;
1864         unsigned int nsg;
1865         int nents;
1866
1867         if (!req_ctx->last && (nbytes + req_ctx->nbuf <= blocksize)) {
1868                 /* Buffer up to one whole block */
1869                 nents = sg_nents_for_len(areq->src, nbytes);
1870                 if (nents < 0) {
1871                         dev_err(ctx->dev, "Invalid number of src SG.\n");
1872                         return nents;
1873                 }
1874                 sg_copy_to_buffer(areq->src, nents,
1875                                   req_ctx->buf + req_ctx->nbuf, nbytes);
1876                 req_ctx->nbuf += nbytes;
1877                 return 0;
1878         }
1879
1880         /* At least (blocksize + 1) bytes are available to hash */
1881         nbytes_to_hash = nbytes + req_ctx->nbuf;
1882         to_hash_later = nbytes_to_hash & (blocksize - 1);
1883
1884         if (req_ctx->last)
1885                 to_hash_later = 0;
1886         else if (to_hash_later)
1887                 /* There is a partial block. Hash the full block(s) now */
1888                 nbytes_to_hash -= to_hash_later;
1889         else {
1890                 /* Keep one block buffered */
1891                 nbytes_to_hash -= blocksize;
1892                 to_hash_later = blocksize;
1893         }
1894
1895         /* Chain in any previously buffered data */
1896         if (req_ctx->nbuf) {
1897                 nsg = (req_ctx->nbuf < nbytes_to_hash) ? 2 : 1;
1898                 sg_init_table(req_ctx->bufsl, nsg);
1899                 sg_set_buf(req_ctx->bufsl, req_ctx->buf, req_ctx->nbuf);
1900                 if (nsg > 1)
1901                         sg_chain(req_ctx->bufsl, 2, areq->src);
1902                 req_ctx->psrc = req_ctx->bufsl;
1903         } else
1904                 req_ctx->psrc = areq->src;
1905
1906         if (to_hash_later) {
1907                 nents = sg_nents_for_len(areq->src, nbytes);
1908                 if (nents < 0) {
1909                         dev_err(ctx->dev, "Invalid number of src SG.\n");
1910                         return nents;
1911                 }
1912                 sg_pcopy_to_buffer(areq->src, nents,
1913                                       req_ctx->bufnext,
1914                                       to_hash_later,
1915                                       nbytes - to_hash_later);
1916         }
1917         req_ctx->to_hash_later = to_hash_later;
1918
1919         /* Allocate extended descriptor */
1920         edesc = ahash_edesc_alloc(areq, nbytes_to_hash);
1921         if (IS_ERR(edesc))
1922                 return PTR_ERR(edesc);
1923
1924         edesc->desc.hdr = ctx->desc_hdr_template;
1925
1926         /* On last one, request SEC to pad; otherwise continue */
1927         if (req_ctx->last)
1928                 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_PAD;
1929         else
1930                 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_CONT;
1931
1932         /* request SEC to INIT hash. */
1933         if (req_ctx->first && !req_ctx->swinit)
1934                 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_INIT;
1935
1936         /* When the tfm context has a keylen, it's an HMAC.
1937          * A first or last (ie. not middle) descriptor must request HMAC.
1938          */
1939         if (ctx->keylen && (req_ctx->first || req_ctx->last))
1940                 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC;
1941
1942         return common_nonsnoop_hash(edesc, areq, nbytes_to_hash,
1943                                     ahash_done);
1944 }
1945
1946 static int ahash_update(struct ahash_request *areq)
1947 {
1948         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1949
1950         req_ctx->last = 0;
1951
1952         return ahash_process_req(areq, areq->nbytes);
1953 }
1954
1955 static int ahash_final(struct ahash_request *areq)
1956 {
1957         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1958
1959         req_ctx->last = 1;
1960
1961         return ahash_process_req(areq, 0);
1962 }
1963
1964 static int ahash_finup(struct ahash_request *areq)
1965 {
1966         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1967
1968         req_ctx->last = 1;
1969
1970         return ahash_process_req(areq, areq->nbytes);
1971 }
1972
1973 static int ahash_digest(struct ahash_request *areq)
1974 {
1975         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1976         struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
1977
1978         ahash->init(areq);
1979         req_ctx->last = 1;
1980
1981         return ahash_process_req(areq, areq->nbytes);
1982 }
1983
1984 struct keyhash_result {
1985         struct completion completion;
1986         int err;
1987 };
1988
1989 static void keyhash_complete(struct crypto_async_request *req, int err)
1990 {
1991         struct keyhash_result *res = req->data;
1992
1993         if (err == -EINPROGRESS)
1994                 return;
1995
1996         res->err = err;
1997         complete(&res->completion);
1998 }
1999
2000 static int keyhash(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen,
2001                    u8 *hash)
2002 {
2003         struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2004
2005         struct scatterlist sg[1];
2006         struct ahash_request *req;
2007         struct keyhash_result hresult;
2008         int ret;
2009
2010         init_completion(&hresult.completion);
2011
2012         req = ahash_request_alloc(tfm, GFP_KERNEL);
2013         if (!req)
2014                 return -ENOMEM;
2015
2016         /* Keep tfm keylen == 0 during hash of the long key */
2017         ctx->keylen = 0;
2018         ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
2019                                    keyhash_complete, &hresult);
2020
2021         sg_init_one(&sg[0], key, keylen);
2022
2023         ahash_request_set_crypt(req, sg, hash, keylen);
2024         ret = crypto_ahash_digest(req);
2025         switch (ret) {
2026         case 0:
2027                 break;
2028         case -EINPROGRESS:
2029         case -EBUSY:
2030                 ret = wait_for_completion_interruptible(
2031                         &hresult.completion);
2032                 if (!ret)
2033                         ret = hresult.err;
2034                 break;
2035         default:
2036                 break;
2037         }
2038         ahash_request_free(req);
2039
2040         return ret;
2041 }
2042
2043 static int ahash_setkey(struct crypto_ahash *tfm, const u8 *key,
2044                         unsigned int keylen)
2045 {
2046         struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2047         unsigned int blocksize =
2048                         crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
2049         unsigned int digestsize = crypto_ahash_digestsize(tfm);
2050         unsigned int keysize = keylen;
2051         u8 hash[SHA512_DIGEST_SIZE];
2052         int ret;
2053
2054         if (keylen <= blocksize)
2055                 memcpy(ctx->key, key, keysize);
2056         else {
2057                 /* Must get the hash of the long key */
2058                 ret = keyhash(tfm, key, keylen, hash);
2059
2060                 if (ret) {
2061                         crypto_ahash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
2062                         return -EINVAL;
2063                 }
2064
2065                 keysize = digestsize;
2066                 memcpy(ctx->key, hash, digestsize);
2067         }
2068
2069         ctx->keylen = keysize;
2070
2071         return 0;
2072 }
2073
2074
2075 struct talitos_alg_template {
2076         u32 type;
2077         union {
2078                 struct crypto_alg crypto;
2079                 struct ahash_alg hash;
2080                 struct aead_alg aead;
2081         } alg;
2082         __be32 desc_hdr_template;
2083 };
2084
2085 static struct talitos_alg_template driver_algs[] = {
2086         /* AEAD algorithms.  These use a single-pass ipsec_esp descriptor */
2087         {       .type = CRYPTO_ALG_TYPE_AEAD,
2088                 .alg.aead = {
2089                         .base = {
2090                                 .cra_name = "authenc(hmac(sha1),cbc(aes))",
2091                                 .cra_driver_name = "authenc-hmac-sha1-"
2092                                                    "cbc-aes-talitos",
2093                                 .cra_blocksize = AES_BLOCK_SIZE,
2094                                 .cra_flags = CRYPTO_ALG_ASYNC,
2095                         },
2096                         .ivsize = AES_BLOCK_SIZE,
2097                         .maxauthsize = SHA1_DIGEST_SIZE,
2098                 },
2099                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2100                                      DESC_HDR_SEL0_AESU |
2101                                      DESC_HDR_MODE0_AESU_CBC |
2102                                      DESC_HDR_SEL1_MDEUA |
2103                                      DESC_HDR_MODE1_MDEU_INIT |
2104                                      DESC_HDR_MODE1_MDEU_PAD |
2105                                      DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2106         },
2107         {       .type = CRYPTO_ALG_TYPE_AEAD,
2108                 .alg.aead = {
2109                         .base = {
2110                                 .cra_name = "authenc(hmac(sha1),"
2111                                             "cbc(des3_ede))",
2112                                 .cra_driver_name = "authenc-hmac-sha1-"
2113                                                    "cbc-3des-talitos",
2114                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2115                                 .cra_flags = CRYPTO_ALG_ASYNC,
2116                         },
2117                         .ivsize = DES3_EDE_BLOCK_SIZE,
2118                         .maxauthsize = SHA1_DIGEST_SIZE,
2119                 },
2120                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2121                                      DESC_HDR_SEL0_DEU |
2122                                      DESC_HDR_MODE0_DEU_CBC |
2123                                      DESC_HDR_MODE0_DEU_3DES |
2124                                      DESC_HDR_SEL1_MDEUA |
2125                                      DESC_HDR_MODE1_MDEU_INIT |
2126                                      DESC_HDR_MODE1_MDEU_PAD |
2127                                      DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2128         },
2129         {       .type = CRYPTO_ALG_TYPE_AEAD,
2130                 .alg.aead = {
2131                         .base = {
2132                                 .cra_name = "authenc(hmac(sha224),cbc(aes))",
2133                                 .cra_driver_name = "authenc-hmac-sha224-"
2134                                                    "cbc-aes-talitos",
2135                                 .cra_blocksize = AES_BLOCK_SIZE,
2136                                 .cra_flags = CRYPTO_ALG_ASYNC,
2137                         },
2138                         .ivsize = AES_BLOCK_SIZE,
2139                         .maxauthsize = SHA224_DIGEST_SIZE,
2140                 },
2141                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2142                                      DESC_HDR_SEL0_AESU |
2143                                      DESC_HDR_MODE0_AESU_CBC |
2144                                      DESC_HDR_SEL1_MDEUA |
2145                                      DESC_HDR_MODE1_MDEU_INIT |
2146                                      DESC_HDR_MODE1_MDEU_PAD |
2147                                      DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2148         },
2149         {       .type = CRYPTO_ALG_TYPE_AEAD,
2150                 .alg.aead = {
2151                         .base = {
2152                                 .cra_name = "authenc(hmac(sha224),"
2153                                             "cbc(des3_ede))",
2154                                 .cra_driver_name = "authenc-hmac-sha224-"
2155                                                    "cbc-3des-talitos",
2156                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2157                                 .cra_flags = CRYPTO_ALG_ASYNC,
2158                         },
2159                         .ivsize = DES3_EDE_BLOCK_SIZE,
2160                         .maxauthsize = SHA224_DIGEST_SIZE,
2161                 },
2162                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2163                                      DESC_HDR_SEL0_DEU |
2164                                      DESC_HDR_MODE0_DEU_CBC |
2165                                      DESC_HDR_MODE0_DEU_3DES |
2166                                      DESC_HDR_SEL1_MDEUA |
2167                                      DESC_HDR_MODE1_MDEU_INIT |
2168                                      DESC_HDR_MODE1_MDEU_PAD |
2169                                      DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2170         },
2171         {       .type = CRYPTO_ALG_TYPE_AEAD,
2172                 .alg.aead = {
2173                         .base = {
2174                                 .cra_name = "authenc(hmac(sha256),cbc(aes))",
2175                                 .cra_driver_name = "authenc-hmac-sha256-"
2176                                                    "cbc-aes-talitos",
2177                                 .cra_blocksize = AES_BLOCK_SIZE,
2178                                 .cra_flags = CRYPTO_ALG_ASYNC,
2179                         },
2180                         .ivsize = AES_BLOCK_SIZE,
2181                         .maxauthsize = SHA256_DIGEST_SIZE,
2182                 },
2183                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2184                                      DESC_HDR_SEL0_AESU |
2185                                      DESC_HDR_MODE0_AESU_CBC |
2186                                      DESC_HDR_SEL1_MDEUA |
2187                                      DESC_HDR_MODE1_MDEU_INIT |
2188                                      DESC_HDR_MODE1_MDEU_PAD |
2189                                      DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2190         },
2191         {       .type = CRYPTO_ALG_TYPE_AEAD,
2192                 .alg.aead = {
2193                         .base = {
2194                                 .cra_name = "authenc(hmac(sha256),"
2195                                             "cbc(des3_ede))",
2196                                 .cra_driver_name = "authenc-hmac-sha256-"
2197                                                    "cbc-3des-talitos",
2198                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2199                                 .cra_flags = CRYPTO_ALG_ASYNC,
2200                         },
2201                         .ivsize = DES3_EDE_BLOCK_SIZE,
2202                         .maxauthsize = SHA256_DIGEST_SIZE,
2203                 },
2204                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2205                                      DESC_HDR_SEL0_DEU |
2206                                      DESC_HDR_MODE0_DEU_CBC |
2207                                      DESC_HDR_MODE0_DEU_3DES |
2208                                      DESC_HDR_SEL1_MDEUA |
2209                                      DESC_HDR_MODE1_MDEU_INIT |
2210                                      DESC_HDR_MODE1_MDEU_PAD |
2211                                      DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2212         },
2213         {       .type = CRYPTO_ALG_TYPE_AEAD,
2214                 .alg.aead = {
2215                         .base = {
2216                                 .cra_name = "authenc(hmac(sha384),cbc(aes))",
2217                                 .cra_driver_name = "authenc-hmac-sha384-"
2218                                                    "cbc-aes-talitos",
2219                                 .cra_blocksize = AES_BLOCK_SIZE,
2220                                 .cra_flags = CRYPTO_ALG_ASYNC,
2221                         },
2222                         .ivsize = AES_BLOCK_SIZE,
2223                         .maxauthsize = SHA384_DIGEST_SIZE,
2224                 },
2225                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2226                                      DESC_HDR_SEL0_AESU |
2227                                      DESC_HDR_MODE0_AESU_CBC |
2228                                      DESC_HDR_SEL1_MDEUB |
2229                                      DESC_HDR_MODE1_MDEU_INIT |
2230                                      DESC_HDR_MODE1_MDEU_PAD |
2231                                      DESC_HDR_MODE1_MDEUB_SHA384_HMAC,
2232         },
2233         {       .type = CRYPTO_ALG_TYPE_AEAD,
2234                 .alg.aead = {
2235                         .base = {
2236                                 .cra_name = "authenc(hmac(sha384),"
2237                                             "cbc(des3_ede))",
2238                                 .cra_driver_name = "authenc-hmac-sha384-"
2239                                                    "cbc-3des-talitos",
2240                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2241                                 .cra_flags = CRYPTO_ALG_ASYNC,
2242                         },
2243                         .ivsize = DES3_EDE_BLOCK_SIZE,
2244                         .maxauthsize = SHA384_DIGEST_SIZE,
2245                 },
2246                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2247                                      DESC_HDR_SEL0_DEU |
2248                                      DESC_HDR_MODE0_DEU_CBC |
2249                                      DESC_HDR_MODE0_DEU_3DES |
2250                                      DESC_HDR_SEL1_MDEUB |
2251                                      DESC_HDR_MODE1_MDEU_INIT |
2252                                      DESC_HDR_MODE1_MDEU_PAD |
2253                                      DESC_HDR_MODE1_MDEUB_SHA384_HMAC,
2254         },
2255         {       .type = CRYPTO_ALG_TYPE_AEAD,
2256                 .alg.aead = {
2257                         .base = {
2258                                 .cra_name = "authenc(hmac(sha512),cbc(aes))",
2259                                 .cra_driver_name = "authenc-hmac-sha512-"
2260                                                    "cbc-aes-talitos",
2261                                 .cra_blocksize = AES_BLOCK_SIZE,
2262                                 .cra_flags = CRYPTO_ALG_ASYNC,
2263                         },
2264                         .ivsize = AES_BLOCK_SIZE,
2265                         .maxauthsize = SHA512_DIGEST_SIZE,
2266                 },
2267                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2268                                      DESC_HDR_SEL0_AESU |
2269                                      DESC_HDR_MODE0_AESU_CBC |
2270                                      DESC_HDR_SEL1_MDEUB |
2271                                      DESC_HDR_MODE1_MDEU_INIT |
2272                                      DESC_HDR_MODE1_MDEU_PAD |
2273                                      DESC_HDR_MODE1_MDEUB_SHA512_HMAC,
2274         },
2275         {       .type = CRYPTO_ALG_TYPE_AEAD,
2276                 .alg.aead = {
2277                         .base = {
2278                                 .cra_name = "authenc(hmac(sha512),"
2279                                             "cbc(des3_ede))",
2280                                 .cra_driver_name = "authenc-hmac-sha512-"
2281                                                    "cbc-3des-talitos",
2282                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2283                                 .cra_flags = CRYPTO_ALG_ASYNC,
2284                         },
2285                         .ivsize = DES3_EDE_BLOCK_SIZE,
2286                         .maxauthsize = SHA512_DIGEST_SIZE,
2287                 },
2288                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2289                                      DESC_HDR_SEL0_DEU |
2290                                      DESC_HDR_MODE0_DEU_CBC |
2291                                      DESC_HDR_MODE0_DEU_3DES |
2292                                      DESC_HDR_SEL1_MDEUB |
2293                                      DESC_HDR_MODE1_MDEU_INIT |
2294                                      DESC_HDR_MODE1_MDEU_PAD |
2295                                      DESC_HDR_MODE1_MDEUB_SHA512_HMAC,
2296         },
2297         {       .type = CRYPTO_ALG_TYPE_AEAD,
2298                 .alg.aead = {
2299                         .base = {
2300                                 .cra_name = "authenc(hmac(md5),cbc(aes))",
2301                                 .cra_driver_name = "authenc-hmac-md5-"
2302                                                    "cbc-aes-talitos",
2303                                 .cra_blocksize = AES_BLOCK_SIZE,
2304                                 .cra_flags = CRYPTO_ALG_ASYNC,
2305                         },
2306                         .ivsize = AES_BLOCK_SIZE,
2307                         .maxauthsize = MD5_DIGEST_SIZE,
2308                 },
2309                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2310                                      DESC_HDR_SEL0_AESU |
2311                                      DESC_HDR_MODE0_AESU_CBC |
2312                                      DESC_HDR_SEL1_MDEUA |
2313                                      DESC_HDR_MODE1_MDEU_INIT |
2314                                      DESC_HDR_MODE1_MDEU_PAD |
2315                                      DESC_HDR_MODE1_MDEU_MD5_HMAC,
2316         },
2317         {       .type = CRYPTO_ALG_TYPE_AEAD,
2318                 .alg.aead = {
2319                         .base = {
2320                                 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
2321                                 .cra_driver_name = "authenc-hmac-md5-"
2322                                                    "cbc-3des-talitos",
2323                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2324                                 .cra_flags = CRYPTO_ALG_ASYNC,
2325                         },
2326                         .ivsize = DES3_EDE_BLOCK_SIZE,
2327                         .maxauthsize = MD5_DIGEST_SIZE,
2328                 },
2329                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2330                                      DESC_HDR_SEL0_DEU |
2331                                      DESC_HDR_MODE0_DEU_CBC |
2332                                      DESC_HDR_MODE0_DEU_3DES |
2333                                      DESC_HDR_SEL1_MDEUA |
2334                                      DESC_HDR_MODE1_MDEU_INIT |
2335                                      DESC_HDR_MODE1_MDEU_PAD |
2336                                      DESC_HDR_MODE1_MDEU_MD5_HMAC,
2337         },
2338         /* ABLKCIPHER algorithms. */
2339         {       .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2340                 .alg.crypto = {
2341                         .cra_name = "ecb(aes)",
2342                         .cra_driver_name = "ecb-aes-talitos",
2343                         .cra_blocksize = AES_BLOCK_SIZE,
2344                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2345                                      CRYPTO_ALG_ASYNC,
2346                         .cra_ablkcipher = {
2347                                 .min_keysize = AES_MIN_KEY_SIZE,
2348                                 .max_keysize = AES_MAX_KEY_SIZE,
2349                                 .ivsize = AES_BLOCK_SIZE,
2350                         }
2351                 },
2352                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2353                                      DESC_HDR_SEL0_AESU,
2354         },
2355         {       .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2356                 .alg.crypto = {
2357                         .cra_name = "cbc(aes)",
2358                         .cra_driver_name = "cbc-aes-talitos",
2359                         .cra_blocksize = AES_BLOCK_SIZE,
2360                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2361                                      CRYPTO_ALG_ASYNC,
2362                         .cra_ablkcipher = {
2363                                 .min_keysize = AES_MIN_KEY_SIZE,
2364                                 .max_keysize = AES_MAX_KEY_SIZE,
2365                                 .ivsize = AES_BLOCK_SIZE,
2366                         }
2367                 },
2368                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2369                                      DESC_HDR_SEL0_AESU |
2370                                      DESC_HDR_MODE0_AESU_CBC,
2371         },
2372         {       .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2373                 .alg.crypto = {
2374                         .cra_name = "ctr(aes)",
2375                         .cra_driver_name = "ctr-aes-talitos",
2376                         .cra_blocksize = AES_BLOCK_SIZE,
2377                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2378                                      CRYPTO_ALG_ASYNC,
2379                         .cra_ablkcipher = {
2380                                 .min_keysize = AES_MIN_KEY_SIZE,
2381                                 .max_keysize = AES_MAX_KEY_SIZE,
2382                                 .ivsize = AES_BLOCK_SIZE,
2383                         }
2384                 },
2385                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2386                                      DESC_HDR_SEL0_AESU |
2387                                      DESC_HDR_MODE0_AESU_CTR,
2388         },
2389         {       .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2390                 .alg.crypto = {
2391                         .cra_name = "ecb(des)",
2392                         .cra_driver_name = "ecb-des-talitos",
2393                         .cra_blocksize = DES_BLOCK_SIZE,
2394                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2395                                      CRYPTO_ALG_ASYNC,
2396                         .cra_ablkcipher = {
2397                                 .min_keysize = DES_KEY_SIZE,
2398                                 .max_keysize = DES_KEY_SIZE,
2399                                 .ivsize = DES_BLOCK_SIZE,
2400                         }
2401                 },
2402                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2403                                      DESC_HDR_SEL0_DEU,
2404         },
2405         {       .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2406                 .alg.crypto = {
2407                         .cra_name = "cbc(des)",
2408                         .cra_driver_name = "cbc-des-talitos",
2409                         .cra_blocksize = DES_BLOCK_SIZE,
2410                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2411                                      CRYPTO_ALG_ASYNC,
2412                         .cra_ablkcipher = {
2413                                 .min_keysize = DES_KEY_SIZE,
2414                                 .max_keysize = DES_KEY_SIZE,
2415                                 .ivsize = DES_BLOCK_SIZE,
2416                         }
2417                 },
2418                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2419                                      DESC_HDR_SEL0_DEU |
2420                                      DESC_HDR_MODE0_DEU_CBC,
2421         },
2422         {       .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2423                 .alg.crypto = {
2424                         .cra_name = "ecb(des3_ede)",
2425                         .cra_driver_name = "ecb-3des-talitos",
2426                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2427                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2428                                      CRYPTO_ALG_ASYNC,
2429                         .cra_ablkcipher = {
2430                                 .min_keysize = DES3_EDE_KEY_SIZE,
2431                                 .max_keysize = DES3_EDE_KEY_SIZE,
2432                                 .ivsize = DES3_EDE_BLOCK_SIZE,
2433                         }
2434                 },
2435                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2436                                      DESC_HDR_SEL0_DEU |
2437                                      DESC_HDR_MODE0_DEU_3DES,
2438         },
2439         {       .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2440                 .alg.crypto = {
2441                         .cra_name = "cbc(des3_ede)",
2442                         .cra_driver_name = "cbc-3des-talitos",
2443                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2444                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2445                                      CRYPTO_ALG_ASYNC,
2446                         .cra_ablkcipher = {
2447                                 .min_keysize = DES3_EDE_KEY_SIZE,
2448                                 .max_keysize = DES3_EDE_KEY_SIZE,
2449                                 .ivsize = DES3_EDE_BLOCK_SIZE,
2450                         }
2451                 },
2452                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2453                                      DESC_HDR_SEL0_DEU |
2454                                      DESC_HDR_MODE0_DEU_CBC |
2455                                      DESC_HDR_MODE0_DEU_3DES,
2456         },
2457         /* AHASH algorithms. */
2458         {       .type = CRYPTO_ALG_TYPE_AHASH,
2459                 .alg.hash = {
2460                         .halg.digestsize = MD5_DIGEST_SIZE,
2461                         .halg.base = {
2462                                 .cra_name = "md5",
2463                                 .cra_driver_name = "md5-talitos",
2464                                 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
2465                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2466                                              CRYPTO_ALG_ASYNC,
2467                         }
2468                 },
2469                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2470                                      DESC_HDR_SEL0_MDEUA |
2471                                      DESC_HDR_MODE0_MDEU_MD5,
2472         },
2473         {       .type = CRYPTO_ALG_TYPE_AHASH,
2474                 .alg.hash = {
2475                         .halg.digestsize = SHA1_DIGEST_SIZE,
2476                         .halg.base = {
2477                                 .cra_name = "sha1",
2478                                 .cra_driver_name = "sha1-talitos",
2479                                 .cra_blocksize = SHA1_BLOCK_SIZE,
2480                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2481                                              CRYPTO_ALG_ASYNC,
2482                         }
2483                 },
2484                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2485                                      DESC_HDR_SEL0_MDEUA |
2486                                      DESC_HDR_MODE0_MDEU_SHA1,
2487         },
2488         {       .type = CRYPTO_ALG_TYPE_AHASH,
2489                 .alg.hash = {
2490                         .halg.digestsize = SHA224_DIGEST_SIZE,
2491                         .halg.base = {
2492                                 .cra_name = "sha224",
2493                                 .cra_driver_name = "sha224-talitos",
2494                                 .cra_blocksize = SHA224_BLOCK_SIZE,
2495                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2496                                              CRYPTO_ALG_ASYNC,
2497                         }
2498                 },
2499                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2500                                      DESC_HDR_SEL0_MDEUA |
2501                                      DESC_HDR_MODE0_MDEU_SHA224,
2502         },
2503         {       .type = CRYPTO_ALG_TYPE_AHASH,
2504                 .alg.hash = {
2505                         .halg.digestsize = SHA256_DIGEST_SIZE,
2506                         .halg.base = {
2507                                 .cra_name = "sha256",
2508                                 .cra_driver_name = "sha256-talitos",
2509                                 .cra_blocksize = SHA256_BLOCK_SIZE,
2510                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2511                                              CRYPTO_ALG_ASYNC,
2512                         }
2513                 },
2514                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2515                                      DESC_HDR_SEL0_MDEUA |
2516                                      DESC_HDR_MODE0_MDEU_SHA256,
2517         },
2518         {       .type = CRYPTO_ALG_TYPE_AHASH,
2519                 .alg.hash = {
2520                         .halg.digestsize = SHA384_DIGEST_SIZE,
2521                         .halg.base = {
2522                                 .cra_name = "sha384",
2523                                 .cra_driver_name = "sha384-talitos",
2524                                 .cra_blocksize = SHA384_BLOCK_SIZE,
2525                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2526                                              CRYPTO_ALG_ASYNC,
2527                         }
2528                 },
2529                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2530                                      DESC_HDR_SEL0_MDEUB |
2531                                      DESC_HDR_MODE0_MDEUB_SHA384,
2532         },
2533         {       .type = CRYPTO_ALG_TYPE_AHASH,
2534                 .alg.hash = {
2535                         .halg.digestsize = SHA512_DIGEST_SIZE,
2536                         .halg.base = {
2537                                 .cra_name = "sha512",
2538                                 .cra_driver_name = "sha512-talitos",
2539                                 .cra_blocksize = SHA512_BLOCK_SIZE,
2540                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2541                                              CRYPTO_ALG_ASYNC,
2542                         }
2543                 },
2544                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2545                                      DESC_HDR_SEL0_MDEUB |
2546                                      DESC_HDR_MODE0_MDEUB_SHA512,
2547         },
2548         {       .type = CRYPTO_ALG_TYPE_AHASH,
2549                 .alg.hash = {
2550                         .halg.digestsize = MD5_DIGEST_SIZE,
2551                         .halg.base = {
2552                                 .cra_name = "hmac(md5)",
2553                                 .cra_driver_name = "hmac-md5-talitos",
2554                                 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
2555                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2556                                              CRYPTO_ALG_ASYNC,
2557                         }
2558                 },
2559                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2560                                      DESC_HDR_SEL0_MDEUA |
2561                                      DESC_HDR_MODE0_MDEU_MD5,
2562         },
2563         {       .type = CRYPTO_ALG_TYPE_AHASH,
2564                 .alg.hash = {
2565                         .halg.digestsize = SHA1_DIGEST_SIZE,
2566                         .halg.base = {
2567                                 .cra_name = "hmac(sha1)",
2568                                 .cra_driver_name = "hmac-sha1-talitos",
2569                                 .cra_blocksize = SHA1_BLOCK_SIZE,
2570                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2571                                              CRYPTO_ALG_ASYNC,
2572                         }
2573                 },
2574                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2575                                      DESC_HDR_SEL0_MDEUA |
2576                                      DESC_HDR_MODE0_MDEU_SHA1,
2577         },
2578         {       .type = CRYPTO_ALG_TYPE_AHASH,
2579                 .alg.hash = {
2580                         .halg.digestsize = SHA224_DIGEST_SIZE,
2581                         .halg.base = {
2582                                 .cra_name = "hmac(sha224)",
2583                                 .cra_driver_name = "hmac-sha224-talitos",
2584                                 .cra_blocksize = SHA224_BLOCK_SIZE,
2585                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2586                                              CRYPTO_ALG_ASYNC,
2587                         }
2588                 },
2589                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2590                                      DESC_HDR_SEL0_MDEUA |
2591                                      DESC_HDR_MODE0_MDEU_SHA224,
2592         },
2593         {       .type = CRYPTO_ALG_TYPE_AHASH,
2594                 .alg.hash = {
2595                         .halg.digestsize = SHA256_DIGEST_SIZE,
2596                         .halg.base = {
2597                                 .cra_name = "hmac(sha256)",
2598                                 .cra_driver_name = "hmac-sha256-talitos",
2599                                 .cra_blocksize = SHA256_BLOCK_SIZE,
2600                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2601                                              CRYPTO_ALG_ASYNC,
2602                         }
2603                 },
2604                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2605                                      DESC_HDR_SEL0_MDEUA |
2606                                      DESC_HDR_MODE0_MDEU_SHA256,
2607         },
2608         {       .type = CRYPTO_ALG_TYPE_AHASH,
2609                 .alg.hash = {
2610                         .halg.digestsize = SHA384_DIGEST_SIZE,
2611                         .halg.base = {
2612                                 .cra_name = "hmac(sha384)",
2613                                 .cra_driver_name = "hmac-sha384-talitos",
2614                                 .cra_blocksize = SHA384_BLOCK_SIZE,
2615                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2616                                              CRYPTO_ALG_ASYNC,
2617                         }
2618                 },
2619                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2620                                      DESC_HDR_SEL0_MDEUB |
2621                                      DESC_HDR_MODE0_MDEUB_SHA384,
2622         },
2623         {       .type = CRYPTO_ALG_TYPE_AHASH,
2624                 .alg.hash = {
2625                         .halg.digestsize = SHA512_DIGEST_SIZE,
2626                         .halg.base = {
2627                                 .cra_name = "hmac(sha512)",
2628                                 .cra_driver_name = "hmac-sha512-talitos",
2629                                 .cra_blocksize = SHA512_BLOCK_SIZE,
2630                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2631                                              CRYPTO_ALG_ASYNC,
2632                         }
2633                 },
2634                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2635                                      DESC_HDR_SEL0_MDEUB |
2636                                      DESC_HDR_MODE0_MDEUB_SHA512,
2637         }
2638 };
2639
2640 struct talitos_crypto_alg {
2641         struct list_head entry;
2642         struct device *dev;
2643         struct talitos_alg_template algt;
2644 };
2645
2646 static int talitos_init_common(struct talitos_ctx *ctx,
2647                                struct talitos_crypto_alg *talitos_alg)
2648 {
2649         struct talitos_private *priv;
2650
2651         /* update context with ptr to dev */
2652         ctx->dev = talitos_alg->dev;
2653
2654         /* assign SEC channel to tfm in round-robin fashion */
2655         priv = dev_get_drvdata(ctx->dev);
2656         ctx->ch = atomic_inc_return(&priv->last_chan) &
2657                   (priv->num_channels - 1);
2658
2659         /* copy descriptor header template value */
2660         ctx->desc_hdr_template = talitos_alg->algt.desc_hdr_template;
2661
2662         /* select done notification */
2663         ctx->desc_hdr_template |= DESC_HDR_DONE_NOTIFY;
2664
2665         return 0;
2666 }
2667
2668 static int talitos_cra_init(struct crypto_tfm *tfm)
2669 {
2670         struct crypto_alg *alg = tfm->__crt_alg;
2671         struct talitos_crypto_alg *talitos_alg;
2672         struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
2673
2674         if ((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) == CRYPTO_ALG_TYPE_AHASH)
2675                 talitos_alg = container_of(__crypto_ahash_alg(alg),
2676                                            struct talitos_crypto_alg,
2677                                            algt.alg.hash);
2678         else
2679                 talitos_alg = container_of(alg, struct talitos_crypto_alg,
2680                                            algt.alg.crypto);
2681
2682         return talitos_init_common(ctx, talitos_alg);
2683 }
2684
2685 static int talitos_cra_init_aead(struct crypto_aead *tfm)
2686 {
2687         struct aead_alg *alg = crypto_aead_alg(tfm);
2688         struct talitos_crypto_alg *talitos_alg;
2689         struct talitos_ctx *ctx = crypto_aead_ctx(tfm);
2690
2691         talitos_alg = container_of(alg, struct talitos_crypto_alg,
2692                                    algt.alg.aead);
2693
2694         return talitos_init_common(ctx, talitos_alg);
2695 }
2696
2697 static int talitos_cra_init_ahash(struct crypto_tfm *tfm)
2698 {
2699         struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
2700
2701         talitos_cra_init(tfm);
2702
2703         ctx->keylen = 0;
2704         crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
2705                                  sizeof(struct talitos_ahash_req_ctx));
2706
2707         return 0;
2708 }
2709
2710 /*
2711  * given the alg's descriptor header template, determine whether descriptor
2712  * type and primary/secondary execution units required match the hw
2713  * capabilities description provided in the device tree node.
2714  */
2715 static int hw_supports(struct device *dev, __be32 desc_hdr_template)
2716 {
2717         struct talitos_private *priv = dev_get_drvdata(dev);
2718         int ret;
2719
2720         ret = (1 << DESC_TYPE(desc_hdr_template) & priv->desc_types) &&
2721               (1 << PRIMARY_EU(desc_hdr_template) & priv->exec_units);
2722
2723         if (SECONDARY_EU(desc_hdr_template))
2724                 ret = ret && (1 << SECONDARY_EU(desc_hdr_template)
2725                               & priv->exec_units);
2726
2727         return ret;
2728 }
2729
2730 static int talitos_remove(struct platform_device *ofdev)
2731 {
2732         struct device *dev = &ofdev->dev;
2733         struct talitos_private *priv = dev_get_drvdata(dev);
2734         struct talitos_crypto_alg *t_alg, *n;
2735         int i;
2736
2737         list_for_each_entry_safe(t_alg, n, &priv->alg_list, entry) {
2738                 switch (t_alg->algt.type) {
2739                 case CRYPTO_ALG_TYPE_ABLKCIPHER:
2740                         break;
2741                 case CRYPTO_ALG_TYPE_AEAD:
2742                         crypto_unregister_aead(&t_alg->algt.alg.aead);
2743                 case CRYPTO_ALG_TYPE_AHASH:
2744                         crypto_unregister_ahash(&t_alg->algt.alg.hash);
2745                         break;
2746                 }
2747                 list_del(&t_alg->entry);
2748                 kfree(t_alg);
2749         }
2750
2751         if (hw_supports(dev, DESC_HDR_SEL0_RNG))
2752                 talitos_unregister_rng(dev);
2753
2754         for (i = 0; priv->chan && i < priv->num_channels; i++)
2755                 kfree(priv->chan[i].fifo);
2756
2757         kfree(priv->chan);
2758
2759         for (i = 0; i < 2; i++)
2760                 if (priv->irq[i]) {
2761                         free_irq(priv->irq[i], dev);
2762                         irq_dispose_mapping(priv->irq[i]);
2763                 }
2764
2765         tasklet_kill(&priv->done_task[0]);
2766         if (priv->irq[1])
2767                 tasklet_kill(&priv->done_task[1]);
2768
2769         iounmap(priv->reg);
2770
2771         kfree(priv);
2772
2773         return 0;
2774 }
2775
2776 static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev,
2777                                                     struct talitos_alg_template
2778                                                            *template)
2779 {
2780         struct talitos_private *priv = dev_get_drvdata(dev);
2781         struct talitos_crypto_alg *t_alg;
2782         struct crypto_alg *alg;
2783
2784         t_alg = kzalloc(sizeof(struct talitos_crypto_alg), GFP_KERNEL);
2785         if (!t_alg)
2786                 return ERR_PTR(-ENOMEM);
2787
2788         t_alg->algt = *template;
2789
2790         switch (t_alg->algt.type) {
2791         case CRYPTO_ALG_TYPE_ABLKCIPHER:
2792                 alg = &t_alg->algt.alg.crypto;
2793                 alg->cra_init = talitos_cra_init;
2794                 alg->cra_type = &crypto_ablkcipher_type;
2795                 alg->cra_ablkcipher.setkey = ablkcipher_setkey;
2796                 alg->cra_ablkcipher.encrypt = ablkcipher_encrypt;
2797                 alg->cra_ablkcipher.decrypt = ablkcipher_decrypt;
2798                 alg->cra_ablkcipher.geniv = "eseqiv";
2799                 break;
2800         case CRYPTO_ALG_TYPE_AEAD:
2801                 alg = &t_alg->algt.alg.aead.base;
2802                 t_alg->algt.alg.aead.init = talitos_cra_init_aead;
2803                 t_alg->algt.alg.aead.setkey = aead_setkey;
2804                 t_alg->algt.alg.aead.encrypt = aead_encrypt;
2805                 t_alg->algt.alg.aead.decrypt = aead_decrypt;
2806                 break;
2807         case CRYPTO_ALG_TYPE_AHASH:
2808                 alg = &t_alg->algt.alg.hash.halg.base;
2809                 alg->cra_init = talitos_cra_init_ahash;
2810                 alg->cra_type = &crypto_ahash_type;
2811                 t_alg->algt.alg.hash.init = ahash_init;
2812                 t_alg->algt.alg.hash.update = ahash_update;
2813                 t_alg->algt.alg.hash.final = ahash_final;
2814                 t_alg->algt.alg.hash.finup = ahash_finup;
2815                 t_alg->algt.alg.hash.digest = ahash_digest;
2816                 t_alg->algt.alg.hash.setkey = ahash_setkey;
2817
2818                 if (!(priv->features & TALITOS_FTR_HMAC_OK) &&
2819                     !strncmp(alg->cra_name, "hmac", 4)) {
2820                         kfree(t_alg);
2821                         return ERR_PTR(-ENOTSUPP);
2822                 }
2823                 if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) &&
2824                     (!strcmp(alg->cra_name, "sha224") ||
2825                      !strcmp(alg->cra_name, "hmac(sha224)"))) {
2826                         t_alg->algt.alg.hash.init = ahash_init_sha224_swinit;
2827                         t_alg->algt.desc_hdr_template =
2828                                         DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2829                                         DESC_HDR_SEL0_MDEUA |
2830                                         DESC_HDR_MODE0_MDEU_SHA256;
2831                 }
2832                 break;
2833         default:
2834                 dev_err(dev, "unknown algorithm type %d\n", t_alg->algt.type);
2835                 kfree(t_alg);
2836                 return ERR_PTR(-EINVAL);
2837         }
2838
2839         alg->cra_module = THIS_MODULE;
2840         alg->cra_priority = TALITOS_CRA_PRIORITY;
2841         alg->cra_alignmask = 0;
2842         alg->cra_ctxsize = sizeof(struct talitos_ctx);
2843         alg->cra_flags |= CRYPTO_ALG_KERN_DRIVER_ONLY;
2844
2845         t_alg->dev = dev;
2846
2847         return t_alg;
2848 }
2849
2850 static int talitos_probe_irq(struct platform_device *ofdev)
2851 {
2852         struct device *dev = &ofdev->dev;
2853         struct device_node *np = ofdev->dev.of_node;
2854         struct talitos_private *priv = dev_get_drvdata(dev);
2855         int err;
2856         bool is_sec1 = has_ftr_sec1(priv);
2857
2858         priv->irq[0] = irq_of_parse_and_map(np, 0);
2859         if (!priv->irq[0]) {
2860                 dev_err(dev, "failed to map irq\n");
2861                 return -EINVAL;
2862         }
2863         if (is_sec1) {
2864                 err = request_irq(priv->irq[0], talitos1_interrupt_4ch, 0,
2865                                   dev_driver_string(dev), dev);
2866                 goto primary_out;
2867         }
2868
2869         priv->irq[1] = irq_of_parse_and_map(np, 1);
2870
2871         /* get the primary irq line */
2872         if (!priv->irq[1]) {
2873                 err = request_irq(priv->irq[0], talitos2_interrupt_4ch, 0,
2874                                   dev_driver_string(dev), dev);
2875                 goto primary_out;
2876         }
2877
2878         err = request_irq(priv->irq[0], talitos2_interrupt_ch0_2, 0,
2879                           dev_driver_string(dev), dev);
2880         if (err)
2881                 goto primary_out;
2882
2883         /* get the secondary irq line */
2884         err = request_irq(priv->irq[1], talitos2_interrupt_ch1_3, 0,
2885                           dev_driver_string(dev), dev);
2886         if (err) {
2887                 dev_err(dev, "failed to request secondary irq\n");
2888                 irq_dispose_mapping(priv->irq[1]);
2889                 priv->irq[1] = 0;
2890         }
2891
2892         return err;
2893
2894 primary_out:
2895         if (err) {
2896                 dev_err(dev, "failed to request primary irq\n");
2897                 irq_dispose_mapping(priv->irq[0]);
2898                 priv->irq[0] = 0;
2899         }
2900
2901         return err;
2902 }
2903
2904 static int talitos_probe(struct platform_device *ofdev)
2905 {
2906         struct device *dev = &ofdev->dev;
2907         struct device_node *np = ofdev->dev.of_node;
2908         struct talitos_private *priv;
2909         const unsigned int *prop;
2910         int i, err;
2911         int stride;
2912
2913         priv = kzalloc(sizeof(struct talitos_private), GFP_KERNEL);
2914         if (!priv)
2915                 return -ENOMEM;
2916
2917         INIT_LIST_HEAD(&priv->alg_list);
2918
2919         dev_set_drvdata(dev, priv);
2920
2921         priv->ofdev = ofdev;
2922
2923         spin_lock_init(&priv->reg_lock);
2924
2925         priv->reg = of_iomap(np, 0);
2926         if (!priv->reg) {
2927                 dev_err(dev, "failed to of_iomap\n");
2928                 err = -ENOMEM;
2929                 goto err_out;
2930         }
2931
2932         /* get SEC version capabilities from device tree */
2933         prop = of_get_property(np, "fsl,num-channels", NULL);
2934         if (prop)
2935                 priv->num_channels = *prop;
2936
2937         prop = of_get_property(np, "fsl,channel-fifo-len", NULL);
2938         if (prop)
2939                 priv->chfifo_len = *prop;
2940
2941         prop = of_get_property(np, "fsl,exec-units-mask", NULL);
2942         if (prop)
2943                 priv->exec_units = *prop;
2944
2945         prop = of_get_property(np, "fsl,descriptor-types-mask", NULL);
2946         if (prop)
2947                 priv->desc_types = *prop;
2948
2949         if (!is_power_of_2(priv->num_channels) || !priv->chfifo_len ||
2950             !priv->exec_units || !priv->desc_types) {
2951                 dev_err(dev, "invalid property data in device tree node\n");
2952                 err = -EINVAL;
2953                 goto err_out;
2954         }
2955
2956         if (of_device_is_compatible(np, "fsl,sec3.0"))
2957                 priv->features |= TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT;
2958
2959         if (of_device_is_compatible(np, "fsl,sec2.1"))
2960                 priv->features |= TALITOS_FTR_HW_AUTH_CHECK |
2961                                   TALITOS_FTR_SHA224_HWINIT |
2962                                   TALITOS_FTR_HMAC_OK;
2963
2964         if (of_device_is_compatible(np, "fsl,sec1.0"))
2965                 priv->features |= TALITOS_FTR_SEC1;
2966
2967         if (of_device_is_compatible(np, "fsl,sec1.2")) {
2968                 priv->reg_deu = priv->reg + TALITOS12_DEU;
2969                 priv->reg_aesu = priv->reg + TALITOS12_AESU;
2970                 priv->reg_mdeu = priv->reg + TALITOS12_MDEU;
2971                 stride = TALITOS1_CH_STRIDE;
2972         } else if (of_device_is_compatible(np, "fsl,sec1.0")) {
2973                 priv->reg_deu = priv->reg + TALITOS10_DEU;
2974                 priv->reg_aesu = priv->reg + TALITOS10_AESU;
2975                 priv->reg_mdeu = priv->reg + TALITOS10_MDEU;
2976                 priv->reg_afeu = priv->reg + TALITOS10_AFEU;
2977                 priv->reg_rngu = priv->reg + TALITOS10_RNGU;
2978                 priv->reg_pkeu = priv->reg + TALITOS10_PKEU;
2979                 stride = TALITOS1_CH_STRIDE;
2980         } else {
2981                 priv->reg_deu = priv->reg + TALITOS2_DEU;
2982                 priv->reg_aesu = priv->reg + TALITOS2_AESU;
2983                 priv->reg_mdeu = priv->reg + TALITOS2_MDEU;
2984                 priv->reg_afeu = priv->reg + TALITOS2_AFEU;
2985                 priv->reg_rngu = priv->reg + TALITOS2_RNGU;
2986                 priv->reg_pkeu = priv->reg + TALITOS2_PKEU;
2987                 priv->reg_keu = priv->reg + TALITOS2_KEU;
2988                 priv->reg_crcu = priv->reg + TALITOS2_CRCU;
2989                 stride = TALITOS2_CH_STRIDE;
2990         }
2991
2992         err = talitos_probe_irq(ofdev);
2993         if (err)
2994                 goto err_out;
2995
2996         if (of_device_is_compatible(np, "fsl,sec1.0")) {
2997                 tasklet_init(&priv->done_task[0], talitos1_done_4ch,
2998                              (unsigned long)dev);
2999         } else {
3000                 if (!priv->irq[1]) {
3001                         tasklet_init(&priv->done_task[0], talitos2_done_4ch,
3002                                      (unsigned long)dev);
3003                 } else {
3004                         tasklet_init(&priv->done_task[0], talitos2_done_ch0_2,
3005                                      (unsigned long)dev);
3006                         tasklet_init(&priv->done_task[1], talitos2_done_ch1_3,
3007                                      (unsigned long)dev);
3008                 }
3009         }
3010
3011         priv->chan = kzalloc(sizeof(struct talitos_channel) *
3012                              priv->num_channels, GFP_KERNEL);
3013         if (!priv->chan) {
3014                 dev_err(dev, "failed to allocate channel management space\n");
3015                 err = -ENOMEM;
3016                 goto err_out;
3017         }
3018
3019         priv->fifo_len = roundup_pow_of_two(priv->chfifo_len);
3020
3021         for (i = 0; i < priv->num_channels; i++) {
3022                 priv->chan[i].reg = priv->reg + stride * (i + 1);
3023                 if (!priv->irq[1] || !(i & 1))
3024                         priv->chan[i].reg += TALITOS_CH_BASE_OFFSET;
3025
3026                 spin_lock_init(&priv->chan[i].head_lock);
3027                 spin_lock_init(&priv->chan[i].tail_lock);
3028
3029                 priv->chan[i].fifo = kzalloc(sizeof(struct talitos_request) *
3030                                              priv->fifo_len, GFP_KERNEL);
3031                 if (!priv->chan[i].fifo) {
3032                         dev_err(dev, "failed to allocate request fifo %d\n", i);
3033                         err = -ENOMEM;
3034                         goto err_out;
3035                 }
3036
3037                 atomic_set(&priv->chan[i].submit_count,
3038                            -(priv->chfifo_len - 1));
3039         }
3040
3041         dma_set_mask(dev, DMA_BIT_MASK(36));
3042
3043         /* reset and initialize the h/w */
3044         err = init_device(dev);
3045         if (err) {
3046                 dev_err(dev, "failed to initialize device\n");
3047                 goto err_out;
3048         }
3049
3050         /* register the RNG, if available */
3051         if (hw_supports(dev, DESC_HDR_SEL0_RNG)) {
3052                 err = talitos_register_rng(dev);
3053                 if (err) {
3054                         dev_err(dev, "failed to register hwrng: %d\n", err);
3055                         goto err_out;
3056                 } else
3057                         dev_info(dev, "hwrng\n");
3058         }
3059
3060         /* register crypto algorithms the device supports */
3061         for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
3062                 if (hw_supports(dev, driver_algs[i].desc_hdr_template)) {
3063                         struct talitos_crypto_alg *t_alg;
3064                         struct crypto_alg *alg = NULL;
3065
3066                         t_alg = talitos_alg_alloc(dev, &driver_algs[i]);
3067                         if (IS_ERR(t_alg)) {
3068                                 err = PTR_ERR(t_alg);
3069                                 if (err == -ENOTSUPP)
3070                                         continue;
3071                                 goto err_out;
3072                         }
3073
3074                         switch (t_alg->algt.type) {
3075                         case CRYPTO_ALG_TYPE_ABLKCIPHER:
3076                                 err = crypto_register_alg(
3077                                                 &t_alg->algt.alg.crypto);
3078                                 alg = &t_alg->algt.alg.crypto;
3079                                 break;
3080
3081                         case CRYPTO_ALG_TYPE_AEAD:
3082                                 err = crypto_register_aead(
3083                                         &t_alg->algt.alg.aead);
3084                                 alg = &t_alg->algt.alg.aead.base;
3085                                 break;
3086
3087                         case CRYPTO_ALG_TYPE_AHASH:
3088                                 err = crypto_register_ahash(
3089                                                 &t_alg->algt.alg.hash);
3090                                 alg = &t_alg->algt.alg.hash.halg.base;
3091                                 break;
3092                         }
3093                         if (err) {
3094                                 dev_err(dev, "%s alg registration failed\n",
3095                                         alg->cra_driver_name);
3096                                 kfree(t_alg);
3097                         } else
3098                                 list_add_tail(&t_alg->entry, &priv->alg_list);
3099                 }
3100         }
3101         if (!list_empty(&priv->alg_list))
3102                 dev_info(dev, "%s algorithms registered in /proc/crypto\n",
3103                          (char *)of_get_property(np, "compatible", NULL));
3104
3105         return 0;
3106
3107 err_out:
3108         talitos_remove(ofdev);
3109
3110         return err;
3111 }
3112
3113 static const struct of_device_id talitos_match[] = {
3114 #ifdef CONFIG_CRYPTO_DEV_TALITOS1
3115         {
3116                 .compatible = "fsl,sec1.0",
3117         },
3118 #endif
3119 #ifdef CONFIG_CRYPTO_DEV_TALITOS2
3120         {
3121                 .compatible = "fsl,sec2.0",
3122         },
3123 #endif
3124         {},
3125 };
3126 MODULE_DEVICE_TABLE(of, talitos_match);
3127
3128 static struct platform_driver talitos_driver = {
3129         .driver = {
3130                 .name = "talitos",
3131                 .of_match_table = talitos_match,
3132         },
3133         .probe = talitos_probe,
3134         .remove = talitos_remove,
3135 };
3136
3137 module_platform_driver(talitos_driver);
3138
3139 MODULE_LICENSE("GPL");
3140 MODULE_AUTHOR("Kim Phillips <kim.phillips@freescale.com>");
3141 MODULE_DESCRIPTION("Freescale integrated security engine (SEC) driver");