drivers/crypto/img-hash.c

   1 /*
   2  * Copyright (c) 2014 Imagination Technologies
   3  * Authors:  Will Thomas, James Hartley
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License version 2 as published
   7  * by the Free Software Foundation.
   8  *
   9  *      Interface structure taken from omap-sham driver
  10  */
  11
  12 #include <linux/clk.h>
  13 #include <linux/dmaengine.h>
  14 #include <linux/interrupt.h>
  15 #include <linux/io.h>
  16 #include <linux/kernel.h>
  17 #include <linux/module.h>
  18 #include <linux/of_device.h>
  19 #include <linux/platform_device.h>
  20 #include <linux/scatterlist.h>
  21
  22 #include <crypto/internal/hash.h>
  23 #include <crypto/md5.h>
  24 #include <crypto/sha.h>
  25
  26 #define CR_RESET                        0
  27 #define CR_RESET_SET                    1
  28 #define CR_RESET_UNSET                  0
  29
  30 #define CR_MESSAGE_LENGTH_H             0x4
  31 #define CR_MESSAGE_LENGTH_L             0x8
  32
  33 #define CR_CONTROL                      0xc
  34 #define CR_CONTROL_BYTE_ORDER_3210      0
  35 #define CR_CONTROL_BYTE_ORDER_0123      1
  36 #define CR_CONTROL_BYTE_ORDER_2310      2
  37 #define CR_CONTROL_BYTE_ORDER_1032      3
  38 #define CR_CONTROL_BYTE_ORDER_SHIFT     8
  39 #define CR_CONTROL_ALGO_MD5     0
  40 #define CR_CONTROL_ALGO_SHA1    1
  41 #define CR_CONTROL_ALGO_SHA224  2
  42 #define CR_CONTROL_ALGO_SHA256  3
  43
  44 #define CR_INTSTAT                      0x10
  45 #define CR_INTENAB                      0x14
  46 #define CR_INTCLEAR                     0x18
  47 #define CR_INT_RESULTS_AVAILABLE        BIT(0)
  48 #define CR_INT_NEW_RESULTS_SET          BIT(1)
  49 #define CR_INT_RESULT_READ_ERR          BIT(2)
  50 #define CR_INT_MESSAGE_WRITE_ERROR      BIT(3)
  51 #define CR_INT_STATUS                   BIT(8)
  52
  53 #define CR_RESULT_QUEUE         0x1c
  54 #define CR_RSD0                         0x40
  55 #define CR_CORE_REV                     0x50
  56 #define CR_CORE_DES1            0x60
  57 #define CR_CORE_DES2            0x70
  58
  59 #define DRIVER_FLAGS_BUSY               BIT(0)
  60 #define DRIVER_FLAGS_FINAL              BIT(1)
  61 #define DRIVER_FLAGS_DMA_ACTIVE         BIT(2)
  62 #define DRIVER_FLAGS_OUTPUT_READY       BIT(3)
  63 #define DRIVER_FLAGS_INIT               BIT(4)
  64 #define DRIVER_FLAGS_CPU                BIT(5)
  65 #define DRIVER_FLAGS_DMA_READY          BIT(6)
  66 #define DRIVER_FLAGS_ERROR              BIT(7)
  67 #define DRIVER_FLAGS_SG                 BIT(8)
  68 #define DRIVER_FLAGS_SHA1               BIT(18)
  69 #define DRIVER_FLAGS_SHA224             BIT(19)
  70 #define DRIVER_FLAGS_SHA256             BIT(20)
  71 #define DRIVER_FLAGS_MD5                BIT(21)
  72
  73 #define IMG_HASH_QUEUE_LENGTH           20
  74 #define IMG_HASH_DMA_THRESHOLD          64
  75
  76 #ifdef __LITTLE_ENDIAN
  77 #define IMG_HASH_BYTE_ORDER             CR_CONTROL_BYTE_ORDER_3210
  78 #else
  79 #define IMG_HASH_BYTE_ORDER             CR_CONTROL_BYTE_ORDER_0123
  80 #endif
  81
  82 struct img_hash_dev;
  83
  84 struct img_hash_request_ctx {
  85         struct img_hash_dev     *hdev;
  86         u8 digest[SHA256_DIGEST_SIZE] __aligned(sizeof(u32));
  87         unsigned long           flags;
  88         size_t                  digsize;
  89
  90         dma_addr_t              dma_addr;
  91         size_t                  dma_ct;
  92
  93         /* sg root */
  94         struct scatterlist      *sgfirst;
  95         /* walk state */
  96         struct scatterlist      *sg;
  97         size_t                  nents;
  98         size_t                  offset;
  99         unsigned int            total;
 100         size_t                  sent;
 101
 102         unsigned long           op;
 103
 104         size_t                  bufcnt;
 105         struct ahash_request    fallback_req;
 106
 107         /* Zero length buffer must remain last member of struct */
 108         u8 buffer[0] __aligned(sizeof(u32));
 109 };
 110
 111 struct img_hash_ctx {
 112         struct img_hash_dev     *hdev;
 113         unsigned long           flags;
 114         struct crypto_ahash     *fallback;
 115 };
 116
 117 struct img_hash_dev {
 118         struct list_head        list;
 119         struct device           *dev;
 120         struct clk              *hash_clk;
 121         struct clk              *sys_clk;
 122         void __iomem            *io_base;
 123
 124         phys_addr_t             bus_addr;
 125         void __iomem            *cpu_addr;
 126
 127         spinlock_t              lock;
 128         int                     err;
 129         struct tasklet_struct   done_task;
 130         struct tasklet_struct   dma_task;
 131
 132         unsigned long           flags;
 133         struct crypto_queue     queue;
 134         struct ahash_request    *req;
 135
 136         struct dma_chan         *dma_lch;
 137 };
 138
 139 struct img_hash_drv {
 140         struct list_head dev_list;
 141         spinlock_t lock;
 142 };
 143
 144 static struct img_hash_drv img_hash = {
 145         .dev_list = LIST_HEAD_INIT(img_hash.dev_list),
 146         .lock = __SPIN_LOCK_UNLOCKED(img_hash.lock),
 147 };
 148
 149 static inline u32 img_hash_read(struct img_hash_dev *hdev, u32 offset)
 150 {
 151         return readl_relaxed(hdev->io_base + offset);
 152 }
 153
 154 static inline void img_hash_write(struct img_hash_dev *hdev,
 155                                   u32 offset, u32 value)
 156 {
 157         writel_relaxed(value, hdev->io_base + offset);
 158 }
 159
 160 static inline u32 img_hash_read_result_queue(struct img_hash_dev *hdev)
 161 {
 162         return be32_to_cpu(img_hash_read(hdev, CR_RESULT_QUEUE));
 163 }
 164
 165 static void img_hash_start(struct img_hash_dev *hdev, bool dma)
 166 {
 167         struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);
 168         u32 cr = IMG_HASH_BYTE_ORDER << CR_CONTROL_BYTE_ORDER_SHIFT;
 169
 170         if (ctx->flags & DRIVER_FLAGS_MD5)
 171                 cr |= CR_CONTROL_ALGO_MD5;
 172         else if (ctx->flags & DRIVER_FLAGS_SHA1)
 173                 cr |= CR_CONTROL_ALGO_SHA1;
 174         else if (ctx->flags & DRIVER_FLAGS_SHA224)
 175                 cr |= CR_CONTROL_ALGO_SHA224;
 176         else if (ctx->flags & DRIVER_FLAGS_SHA256)
 177                 cr |= CR_CONTROL_ALGO_SHA256;
 178         dev_dbg(hdev->dev, "Starting hash process\n");
 179         img_hash_write(hdev, CR_CONTROL, cr);
 180
 181         /*
 182          * The hardware block requires two cycles between writing the control
 183          * register and writing the first word of data in non DMA mode, to
 184          * ensure the first data write is not grouped in burst with the control
 185          * register write a read is issued to 'flush' the bus.
 186          */
 187         if (!dma)
 188                 img_hash_read(hdev, CR_CONTROL);
 189 }
 190
 191 static int img_hash_xmit_cpu(struct img_hash_dev *hdev, const u8 *buf,
 192                              size_t length, int final)
 193 {
 194         u32 count, len32;
 195         const u32 *buffer = (const u32 *)buf;
 196
 197         dev_dbg(hdev->dev, "xmit_cpu:  length: %zu bytes\n", length);
 198
 199         if (final)
 200                 hdev->flags |= DRIVER_FLAGS_FINAL;
 201
 202         len32 = DIV_ROUND_UP(length, sizeof(u32));
 203
 204         for (count = 0; count < len32; count++)
 205                 writel_relaxed(buffer[count], hdev->cpu_addr);
 206
 207         return -EINPROGRESS;
 208 }
 209
 210 static void img_hash_dma_callback(void *data)
 211 {
 212         struct img_hash_dev *hdev = (struct img_hash_dev *)data;
 213         struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);
 214
 215         if (ctx->bufcnt) {
 216                 img_hash_xmit_cpu(hdev, ctx->buffer, ctx->bufcnt, 0);
 217                 ctx->bufcnt = 0;
 218         }
 219         if (ctx->sg)
 220                 tasklet_schedule(&hdev->dma_task);
 221 }
 222
 223 static int img_hash_xmit_dma(struct img_hash_dev *hdev, struct scatterlist *sg)
 224 {
 225         struct dma_async_tx_descriptor *desc;
 226         struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);
 227
 228         ctx->dma_ct = dma_map_sg(hdev->dev, sg, 1, DMA_MEM_TO_DEV);
 229         if (ctx->dma_ct == 0) {
 230                 dev_err(hdev->dev, "Invalid DMA sg\n");
 231                 hdev->err = -EINVAL;
 232                 return -EINVAL;
 233         }
 234
 235         desc = dmaengine_prep_slave_sg(hdev->dma_lch,
 236                                        sg,
 237                                        ctx->dma_ct,
 238                                        DMA_MEM_TO_DEV,
 239                                        DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 240         if (!desc) {
 241                 dev_err(hdev->dev, "Null DMA descriptor\n");
 242                 hdev->err = -EINVAL;
 243                 dma_unmap_sg(hdev->dev, sg, 1, DMA_MEM_TO_DEV);
 244                 return -EINVAL;
 245         }
 246         desc->callback = img_hash_dma_callback;
 247         desc->callback_param = hdev;
 248         dmaengine_submit(desc);
 249         dma_async_issue_pending(hdev->dma_lch);
 250
 251         return 0;
 252 }
 253
 254 static int img_hash_write_via_cpu(struct img_hash_dev *hdev)
 255 {
 256         struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);
 257
 258         ctx->bufcnt = sg_copy_to_buffer(hdev->req->src, sg_nents(ctx->sg),
 259                                         ctx->buffer, hdev->req->nbytes);
 260
 261         ctx->total = hdev->req->nbytes;
 262         ctx->bufcnt = 0;
 263
 264         hdev->flags |= (DRIVER_FLAGS_CPU | DRIVER_FLAGS_FINAL);
 265
 266         img_hash_start(hdev, false);
 267
 268         return img_hash_xmit_cpu(hdev, ctx->buffer, ctx->total, 1);
 269 }
 270
 271 static int img_hash_finish(struct ahash_request *req)
 272 {
 273         struct img_hash_request_ctx *ctx = ahash_request_ctx(req);
 274
 275         if (!req->result)
 276                 return -EINVAL;
 277
 278         memcpy(req->result, ctx->digest, ctx->digsize);
 279
 280         return 0;
 281 }
 282
 283 static void img_hash_copy_hash(struct ahash_request *req)
 284 {
 285         struct img_hash_request_ctx *ctx = ahash_request_ctx(req);
 286         u32 *hash = (u32 *)ctx->digest;
 287         int i;
 288
 289         for (i = (ctx->digsize / sizeof(u32)) - 1; i >= 0; i--)
 290                 hash[i] = img_hash_read_result_queue(ctx->hdev);
 291 }
 292
 293 static void img_hash_finish_req(struct ahash_request *req, int err)
 294 {
 295         struct img_hash_request_ctx *ctx = ahash_request_ctx(req);
 296         struct img_hash_dev *hdev =  ctx->hdev;
 297
 298         if (!err) {
 299                 img_hash_copy_hash(req);
 300                 if (DRIVER_FLAGS_FINAL & hdev->flags)
 301                         err = img_hash_finish(req);
 302         } else {
 303                 dev_warn(hdev->dev, "Hash failed with error %d\n", err);
 304                 ctx->flags |= DRIVER_FLAGS_ERROR;
 305         }
 306
 307         hdev->flags &= ~(DRIVER_FLAGS_DMA_READY | DRIVER_FLAGS_OUTPUT_READY |
 308                 DRIVER_FLAGS_CPU | DRIVER_FLAGS_BUSY | DRIVER_FLAGS_FINAL);
 309
 310         if (req->base.complete)
 311                 req->base.complete(&req->base, err);
 312 }
 313
 314 static int img_hash_write_via_dma(struct img_hash_dev *hdev)
 315 {
 316         struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);
 317
 318         img_hash_start(hdev, true);
 319
 320         dev_dbg(hdev->dev, "xmit dma size: %d\n", ctx->total);
 321
 322         if (!ctx->total)
 323                 hdev->flags |= DRIVER_FLAGS_FINAL;
 324
 325         hdev->flags |= DRIVER_FLAGS_DMA_ACTIVE | DRIVER_FLAGS_FINAL;
 326
 327         tasklet_schedule(&hdev->dma_task);
 328
 329         return -EINPROGRESS;
 330 }
 331
 332 static int img_hash_dma_init(struct img_hash_dev *hdev)
 333 {
 334         struct dma_slave_config dma_conf;
 335         int err = -EINVAL;
 336
 337         hdev->dma_lch = dma_request_slave_channel(hdev->dev, "tx");
 338         if (!hdev->dma_lch) {
 339                 dev_err(hdev->dev, "Couldn't acquire a slave DMA channel.\n");
 340                 return -EBUSY;
 341         }
 342         dma_conf.direction = DMA_MEM_TO_DEV;
 343         dma_conf.dst_addr = hdev->bus_addr;
 344         dma_conf.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
 345         dma_conf.dst_maxburst = 16;
 346         dma_conf.device_fc = false;
 347
 348         err = dmaengine_slave_config(hdev->dma_lch,  &dma_conf);
 349         if (err) {
 350                 dev_err(hdev->dev, "Couldn't configure DMA slave.\n");
 351                 dma_release_channel(hdev->dma_lch);
 352                 return err;
 353         }
 354
 355         return 0;
 356 }
 357
 358 static void img_hash_dma_task(unsigned long d)
 359 {
 360         struct img_hash_dev *hdev = (struct img_hash_dev *)d;
 361         struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);
 362         u8 *addr;
 363         size_t nbytes, bleft, wsend, len, tbc;
 364         struct scatterlist tsg;
 365
 366         if (!hdev->req || !ctx->sg)
 367                 return;
 368
 369         addr = sg_virt(ctx->sg);
 370         nbytes = ctx->sg->length - ctx->offset;
 371
 372         /*
 373          * The hash accelerator does not support a data valid mask. This means
 374          * that if each dma (i.e. per page) is not a multiple of 4 bytes, the
 375          * padding bytes in the last word written by that dma would erroneously
 376          * be included in the hash. To avoid this we round down the transfer,
 377          * and add the excess to the start of the next dma. It does not matter
 378          * that the final dma may not be a multiple of 4 bytes as the hashing
 379          * block is programmed to accept the correct number of bytes.
 380          */
 381
 382         bleft = nbytes % 4;
 383         wsend = (nbytes / 4);
 384
 385         if (wsend) {
 386                 sg_init_one(&tsg, addr + ctx->offset, wsend * 4);
 387                 if (img_hash_xmit_dma(hdev, &tsg)) {
 388                         dev_err(hdev->dev, "DMA failed, falling back to CPU");
 389                         ctx->flags |= DRIVER_FLAGS_CPU;
 390                         hdev->err = 0;
 391                         img_hash_xmit_cpu(hdev, addr + ctx->offset,
 392                                           wsend * 4, 0);
 393                         ctx->sent += wsend * 4;
 394                         wsend = 0;
 395                 } else {
 396                         ctx->sent += wsend * 4;
 397                 }
 398         }
 399
 400         if (bleft) {
 401                 ctx->bufcnt = sg_pcopy_to_buffer(ctx->sgfirst, ctx->nents,
 402                                                  ctx->buffer, bleft, ctx->sent);
 403                 tbc = 0;
 404                 ctx->sg = sg_next(ctx->sg);
 405                 while (ctx->sg && (ctx->bufcnt < 4)) {
 406                         len = ctx->sg->length;
 407                         if (likely(len > (4 - ctx->bufcnt)))
 408                                 len = 4 - ctx->bufcnt;
 409                         tbc = sg_pcopy_to_buffer(ctx->sgfirst, ctx->nents,
 410                                                  ctx->buffer + ctx->bufcnt, len,
 411                                         ctx->sent + ctx->bufcnt);
 412                         ctx->bufcnt += tbc;
 413                         if (tbc >= ctx->sg->length) {
 414                                 ctx->sg = sg_next(ctx->sg);
 415                                 tbc = 0;
 416                         }
 417                 }
 418
 419                 ctx->sent += ctx->bufcnt;
 420                 ctx->offset = tbc;
 421
 422                 if (!wsend)
 423                         img_hash_dma_callback(hdev);
 424         } else {
 425                 ctx->offset = 0;
 426                 ctx->sg = sg_next(ctx->sg);
 427         }
 428 }
 429
 430 static int img_hash_write_via_dma_stop(struct img_hash_dev *hdev)
 431 {
 432         struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);
 433
 434         if (ctx->flags & DRIVER_FLAGS_SG)
 435                 dma_unmap_sg(hdev->dev, ctx->sg, ctx->dma_ct, DMA_TO_DEVICE);
 436
 437         return 0;
 438 }
 439
 440 static int img_hash_process_data(struct img_hash_dev *hdev)
 441 {
 442         struct ahash_request *req = hdev->req;
 443         struct img_hash_request_ctx *ctx = ahash_request_ctx(req);
 444         int err = 0;
 445
 446         ctx->bufcnt = 0;
 447
 448         if (req->nbytes >= IMG_HASH_DMA_THRESHOLD) {
 449                 dev_dbg(hdev->dev, "process data request(%d bytes) using DMA\n",
 450                         req->nbytes);
 451                 err = img_hash_write_via_dma(hdev);
 452         } else {
 453                 dev_dbg(hdev->dev, "process data request(%d bytes) using CPU\n",
 454                         req->nbytes);
 455                 err = img_hash_write_via_cpu(hdev);
 456         }
 457         return err;
 458 }
 459
 460 static int img_hash_hw_init(struct img_hash_dev *hdev)
 461 {
 462         unsigned long long nbits;
 463         u32 u, l;
 464
 465         img_hash_write(hdev, CR_RESET, CR_RESET_SET);
 466         img_hash_write(hdev, CR_RESET, CR_RESET_UNSET);
 467         img_hash_write(hdev, CR_INTENAB, CR_INT_NEW_RESULTS_SET);
 468
 469         nbits = (u64)hdev->req->nbytes << 3;
 470         u = nbits >> 32;
 471         l = nbits;
 472         img_hash_write(hdev, CR_MESSAGE_LENGTH_H, u);
 473         img_hash_write(hdev, CR_MESSAGE_LENGTH_L, l);
 474
 475         if (!(DRIVER_FLAGS_INIT & hdev->flags)) {
 476                 hdev->flags |= DRIVER_FLAGS_INIT;
 477                 hdev->err = 0;
 478         }
 479         dev_dbg(hdev->dev, "hw initialized, nbits: %llx\n", nbits);
 480         return 0;
 481 }
 482
 483 static int img_hash_init(struct ahash_request *req)
 484 {
 485         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 486         struct img_hash_request_ctx *rctx = ahash_request_ctx(req);
 487         struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 488
 489         ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback);
 490         rctx->fallback_req.base.flags = req->base.flags
 491                 & CRYPTO_TFM_REQ_MAY_SLEEP;
 492
 493         return crypto_ahash_init(&rctx->fallback_req);
 494 }
 495
 496 static int img_hash_handle_queue(struct img_hash_dev *hdev,
 497                                  struct ahash_request *req)
 498 {
 499         struct crypto_async_request *async_req, *backlog;
 500         struct img_hash_request_ctx *ctx;
 501         unsigned long flags;
 502         int err = 0, res = 0;
 503
 504         spin_lock_irqsave(&hdev->lock, flags);
 505
 506         if (req)
 507                 res = ahash_enqueue_request(&hdev->queue, req);
 508
 509         if (DRIVER_FLAGS_BUSY & hdev->flags) {
 510                 spin_unlock_irqrestore(&hdev->lock, flags);
 511                 return res;
 512         }
 513
 514         backlog = crypto_get_backlog(&hdev->queue);
 515         async_req = crypto_dequeue_request(&hdev->queue);
 516         if (async_req)
 517                 hdev->flags |= DRIVER_FLAGS_BUSY;
 518
 519         spin_unlock_irqrestore(&hdev->lock, flags);
 520
 521         if (!async_req)
 522                 return res;
 523
 524         if (backlog)
 525                 backlog->complete(backlog, -EINPROGRESS);
 526
 527         req = ahash_request_cast(async_req);
 528         hdev->req = req;
 529
 530         ctx = ahash_request_ctx(req);
 531
 532         dev_info(hdev->dev, "processing req, op: %lu, bytes: %d\n",
 533                  ctx->op, req->nbytes);
 534
 535         err = img_hash_hw_init(hdev);
 536
 537         if (!err)
 538                 err = img_hash_process_data(hdev);
 539
 540         if (err != -EINPROGRESS) {
 541                 /* done_task will not finish so do it here */
 542                 img_hash_finish_req(req, err);
 543         }
 544         return res;
 545 }
 546
 547 static int img_hash_update(struct ahash_request *req)
 548 {
 549         struct img_hash_request_ctx *rctx = ahash_request_ctx(req);
 550         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 551         struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 552
 553         ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback);
 554         rctx->fallback_req.base.flags = req->base.flags
 555                 & CRYPTO_TFM_REQ_MAY_SLEEP;
 556         rctx->fallback_req.nbytes = req->nbytes;
 557         rctx->fallback_req.src = req->src;
 558
 559         return crypto_ahash_update(&rctx->fallback_req);
 560 }
 561
 562 static int img_hash_final(struct ahash_request *req)
 563 {
 564         struct img_hash_request_ctx *rctx = ahash_request_ctx(req);
 565         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 566         struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 567
 568         ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback);
 569         rctx->fallback_req.base.flags = req->base.flags
 570                 & CRYPTO_TFM_REQ_MAY_SLEEP;
 571         rctx->fallback_req.result = req->result;
 572
 573         return crypto_ahash_final(&rctx->fallback_req);
 574 }
 575
 576 static int img_hash_finup(struct ahash_request *req)
 577 {
 578         struct img_hash_request_ctx *rctx = ahash_request_ctx(req);
 579         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 580         struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 581
 582         ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback);
 583         rctx->fallback_req.base.flags = req->base.flags
 584                 & CRYPTO_TFM_REQ_MAY_SLEEP;
 585         rctx->fallback_req.nbytes = req->nbytes;
 586         rctx->fallback_req.src = req->src;
 587         rctx->fallback_req.result = req->result;
 588
 589         return crypto_ahash_finup(&rctx->fallback_req);
 590 }
 591
 592 static int img_hash_digest(struct ahash_request *req)
 593 {
 594         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 595         struct img_hash_ctx *tctx = crypto_ahash_ctx(tfm);
 596         struct img_hash_request_ctx *ctx = ahash_request_ctx(req);
 597         struct img_hash_dev *hdev = NULL;
 598         struct img_hash_dev *tmp;
 599         int err;
 600
 601         spin_lock(&img_hash.lock);
 602         if (!tctx->hdev) {
 603                 list_for_each_entry(tmp, &img_hash.dev_list, list) {
 604                         hdev = tmp;
 605                         break;
 606                 }
 607                 tctx->hdev = hdev;
 608
 609         } else {
 610                 hdev = tctx->hdev;
 611         }
 612
 613         spin_unlock(&img_hash.lock);
 614         ctx->hdev = hdev;
 615         ctx->flags = 0;
 616         ctx->digsize = crypto_ahash_digestsize(tfm);
 617
 618         switch (ctx->digsize) {
 619         case SHA1_DIGEST_SIZE:
 620                 ctx->flags |= DRIVER_FLAGS_SHA1;
 621                 break;
 622         case SHA256_DIGEST_SIZE:
 623                 ctx->flags |= DRIVER_FLAGS_SHA256;
 624                 break;
 625         case SHA224_DIGEST_SIZE:
 626                 ctx->flags |= DRIVER_FLAGS_SHA224;
 627                 break;
 628         case MD5_DIGEST_SIZE:
 629                 ctx->flags |= DRIVER_FLAGS_MD5;
 630                 break;
 631         default:
 632                 return -EINVAL;
 633         }
 634
 635         ctx->bufcnt = 0;
 636         ctx->offset = 0;
 637         ctx->sent = 0;
 638         ctx->total = req->nbytes;
 639         ctx->sg = req->src;
 640         ctx->sgfirst = req->src;
 641         ctx->nents = sg_nents(ctx->sg);
 642
 643         err = img_hash_handle_queue(tctx->hdev, req);
 644
 645         return err;
 646 }
 647
 648 static int img_hash_cra_init(struct crypto_tfm *tfm)
 649 {
 650         struct img_hash_ctx *ctx = crypto_tfm_ctx(tfm);
 651         const char *alg_name = crypto_tfm_alg_name(tfm);
 652         int err = -ENOMEM;
 653
 654         ctx->fallback = crypto_alloc_ahash(alg_name, 0,
 655                                            CRYPTO_ALG_NEED_FALLBACK);
 656         if (IS_ERR(ctx->fallback)) {
 657                 pr_err("img_hash: Could not load fallback driver.\n");
 658                 err = PTR_ERR(ctx->fallback);
 659                 goto err;
 660         }
 661         crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
 662                                  sizeof(struct img_hash_request_ctx) +
 663                                  IMG_HASH_DMA_THRESHOLD);
 664
 665         return 0;
 666
 667 err:
 668         return err;
 669 }
 670
 671 static void img_hash_cra_exit(struct crypto_tfm *tfm)
 672 {
 673         struct img_hash_ctx *tctx = crypto_tfm_ctx(tfm);
 674
 675         crypto_free_ahash(tctx->fallback);
 676 }
 677
 678 static irqreturn_t img_irq_handler(int irq, void *dev_id)
 679 {
 680         struct img_hash_dev *hdev = dev_id;
 681         u32 reg;
 682
 683         reg = img_hash_read(hdev, CR_INTSTAT);
 684         img_hash_write(hdev, CR_INTCLEAR, reg);
 685
 686         if (reg & CR_INT_NEW_RESULTS_SET) {
 687                 dev_dbg(hdev->dev, "IRQ CR_INT_NEW_RESULTS_SET\n");
 688                 if (DRIVER_FLAGS_BUSY & hdev->flags) {
 689                         hdev->flags |= DRIVER_FLAGS_OUTPUT_READY;
 690                         if (!(DRIVER_FLAGS_CPU & hdev->flags))
 691                                 hdev->flags |= DRIVER_FLAGS_DMA_READY;
 692                         tasklet_schedule(&hdev->done_task);
 693                 } else {
 694                         dev_warn(hdev->dev,
 695                                  "HASH interrupt when no active requests.\n");
 696                 }
 697         } else if (reg & CR_INT_RESULTS_AVAILABLE) {
 698                 dev_warn(hdev->dev,
 699                          "IRQ triggered before the hash had completed\n");
 700         } else if (reg & CR_INT_RESULT_READ_ERR) {
 701                 dev_warn(hdev->dev,
 702                          "Attempt to read from an empty result queue\n");
 703         } else if (reg & CR_INT_MESSAGE_WRITE_ERROR) {
 704                 dev_warn(hdev->dev,
 705                          "Data written before the hardware was configured\n");
 706         }
 707         return IRQ_HANDLED;
 708 }
 709
 710 static struct ahash_alg img_algs[] = {
 711         {
 712                 .init = img_hash_init,
 713                 .update = img_hash_update,
 714                 .final = img_hash_final,
 715                 .finup = img_hash_finup,
 716                 .digest = img_hash_digest,
 717                 .halg = {
 718                         .digestsize = MD5_DIGEST_SIZE,
 719                         .base = {
 720                                 .cra_name = "md5",
 721                                 .cra_driver_name = "img-md5",
 722                                 .cra_priority = 300,
 723                                 .cra_flags =
 724                                 CRYPTO_ALG_ASYNC |
 725                                 CRYPTO_ALG_NEED_FALLBACK,
 726                                 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
 727                                 .cra_ctxsize = sizeof(struct img_hash_ctx),
 728                                 .cra_init = img_hash_cra_init,
 729                                 .cra_exit = img_hash_cra_exit,
 730                                 .cra_module = THIS_MODULE,
 731                         }
 732                 }
 733         },
 734         {
 735                 .init = img_hash_init,
 736                 .update = img_hash_update,
 737                 .final = img_hash_final,
 738                 .finup = img_hash_finup,
 739                 .digest = img_hash_digest,
 740                 .halg = {
 741                         .digestsize = SHA1_DIGEST_SIZE,
 742                         .base = {
 743                                 .cra_name = "sha1",
 744                                 .cra_driver_name = "img-sha1",
 745                                 .cra_priority = 300,
 746                                 .cra_flags =
 747                                 CRYPTO_ALG_ASYNC |
 748                                 CRYPTO_ALG_NEED_FALLBACK,
 749                                 .cra_blocksize = SHA1_BLOCK_SIZE,
 750                                 .cra_ctxsize = sizeof(struct img_hash_ctx),
 751                                 .cra_init = img_hash_cra_init,
 752                                 .cra_exit = img_hash_cra_exit,
 753                                 .cra_module = THIS_MODULE,
 754                         }
 755                 }
 756         },
 757         {
 758                 .init = img_hash_init,
 759                 .update = img_hash_update,
 760                 .final = img_hash_final,
 761                 .finup = img_hash_finup,
 762                 .digest = img_hash_digest,
 763                 .halg = {
 764                         .digestsize = SHA224_DIGEST_SIZE,
 765                         .base = {
 766                                 .cra_name = "sha224",
 767                                 .cra_driver_name = "img-sha224",
 768                                 .cra_priority = 300,
 769                                 .cra_flags =
 770                                 CRYPTO_ALG_ASYNC |
 771                                 CRYPTO_ALG_NEED_FALLBACK,
 772                                 .cra_blocksize = SHA224_BLOCK_SIZE,
 773                                 .cra_ctxsize = sizeof(struct img_hash_ctx),
 774                                 .cra_init = img_hash_cra_init,
 775                                 .cra_exit = img_hash_cra_exit,
 776                                 .cra_module = THIS_MODULE,
 777                         }
 778                 }
 779         },
 780         {
 781                 .init = img_hash_init,
 782                 .update = img_hash_update,
 783                 .final = img_hash_final,
 784                 .finup = img_hash_finup,
 785                 .digest = img_hash_digest,
 786                 .halg = {
 787                         .digestsize = SHA256_DIGEST_SIZE,
 788                         .base = {
 789                                 .cra_name = "sha256",
 790                                 .cra_driver_name = "img-sha256",
 791                                 .cra_priority = 300,
 792                                 .cra_flags =
 793                                 CRYPTO_ALG_ASYNC |
 794                                 CRYPTO_ALG_NEED_FALLBACK,
 795                                 .cra_blocksize = SHA256_BLOCK_SIZE,
 796                                 .cra_ctxsize = sizeof(struct img_hash_ctx),
 797                                 .cra_init = img_hash_cra_init,
 798                                 .cra_exit = img_hash_cra_exit,
 799                                 .cra_module = THIS_MODULE,
 800                         }
 801                 }
 802         }
 803 };
 804
 805 static int img_register_algs(struct img_hash_dev *hdev)
 806 {
 807         int i, err;
 808
 809         for (i = 0; i < ARRAY_SIZE(img_algs); i++) {
 810                 err = crypto_register_ahash(&img_algs[i]);
 811                 if (err)
 812                         goto err_reg;
 813         }
 814         return 0;
 815
 816 err_reg:
 817         for (; i--; )
 818                 crypto_unregister_ahash(&img_algs[i]);
 819
 820         return err;
 821 }
 822
 823 static int img_unregister_algs(struct img_hash_dev *hdev)
 824 {
 825         int i;
 826
 827         for (i = 0; i < ARRAY_SIZE(img_algs); i++)
 828                 crypto_unregister_ahash(&img_algs[i]);
 829         return 0;
 830 }
 831
 832 static void img_hash_done_task(unsigned long data)
 833 {
 834         struct img_hash_dev *hdev = (struct img_hash_dev *)data;
 835         int err = 0;
 836
 837         if (hdev->err == -EINVAL) {
 838                 err = hdev->err;
 839                 goto finish;
 840         }
 841
 842         if (!(DRIVER_FLAGS_BUSY & hdev->flags)) {
 843                 img_hash_handle_queue(hdev, NULL);
 844                 return;
 845         }
 846
 847         if (DRIVER_FLAGS_CPU & hdev->flags) {
 848                 if (DRIVER_FLAGS_OUTPUT_READY & hdev->flags) {
 849                         hdev->flags &= ~DRIVER_FLAGS_OUTPUT_READY;
 850                         goto finish;
 851                 }
 852         } else if (DRIVER_FLAGS_DMA_READY & hdev->flags) {
 853                 if (DRIVER_FLAGS_DMA_ACTIVE & hdev->flags) {
 854                         hdev->flags &= ~DRIVER_FLAGS_DMA_ACTIVE;
 855                         img_hash_write_via_dma_stop(hdev);
 856                         if (hdev->err) {
 857                                 err = hdev->err;
 858                                 goto finish;
 859                         }
 860                 }
 861                 if (DRIVER_FLAGS_OUTPUT_READY & hdev->flags) {
 862                         hdev->flags &= ~(DRIVER_FLAGS_DMA_READY |
 863                                         DRIVER_FLAGS_OUTPUT_READY);
 864                         goto finish;
 865                 }
 866         }
 867         return;
 868
 869 finish:
 870         img_hash_finish_req(hdev->req, err);
 871 }
 872
 873 static const struct of_device_id img_hash_match[] = {
 874         { .compatible = "img,hash-accelerator" },
 875         {}
 876 };
 877 MODULE_DEVICE_TABLE(of, img_hash_match);
 878
 879 static int img_hash_probe(struct platform_device *pdev)
 880 {
 881         struct img_hash_dev *hdev;
 882         struct device *dev = &pdev->dev;
 883         struct resource *hash_res;
 884         int     irq;
 885         int err;
 886
 887         hdev = devm_kzalloc(dev, sizeof(*hdev), GFP_KERNEL);
 888         if (hdev == NULL)
 889                 return -ENOMEM;
 890
 891         spin_lock_init(&hdev->lock);
 892
 893         hdev->dev = dev;
 894
 895         platform_set_drvdata(pdev, hdev);
 896
 897         INIT_LIST_HEAD(&hdev->list);
 898
 899         tasklet_init(&hdev->done_task, img_hash_done_task, (unsigned long)hdev);
 900         tasklet_init(&hdev->dma_task, img_hash_dma_task, (unsigned long)hdev);
 901
 902         crypto_init_queue(&hdev->queue, IMG_HASH_QUEUE_LENGTH);
 903
 904         /* Register bank */
 905         hash_res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 906
 907         hdev->io_base = devm_ioremap_resource(dev, hash_res);
 908         if (IS_ERR(hdev->io_base)) {
 909                 err = PTR_ERR(hdev->io_base);
 910                 dev_err(dev, "can't ioremap, returned %d\n", err);
 911
 912                 goto res_err;
 913         }
 914
 915         /* Write port (DMA or CPU) */
 916         hash_res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
 917         hdev->cpu_addr = devm_ioremap_resource(dev, hash_res);
 918         if (IS_ERR(hdev->cpu_addr)) {
 919                 dev_err(dev, "can't ioremap write port\n");
 920                 err = PTR_ERR(hdev->cpu_addr);
 921                 goto res_err;
 922         }
 923         hdev->bus_addr = hash_res->start;
 924
 925         irq = platform_get_irq(pdev, 0);
 926         if (irq < 0) {
 927                 dev_err(dev, "no IRQ resource info\n");
 928                 err = irq;
 929                 goto res_err;
 930         }
 931
 932         err = devm_request_irq(dev, irq, img_irq_handler, 0,
 933                                dev_name(dev), hdev);
 934         if (err) {
 935                 dev_err(dev, "unable to request irq\n");
 936                 goto res_err;
 937         }
 938         dev_dbg(dev, "using IRQ channel %d\n", irq);
 939
 940         hdev->hash_clk = devm_clk_get(&pdev->dev, "hash");
 941         if (IS_ERR(hdev->hash_clk)) {
 942                 dev_err(dev, "clock initialization failed.\n");
 943                 err = PTR_ERR(hdev->hash_clk);
 944                 goto res_err;
 945         }
 946
 947         hdev->sys_clk = devm_clk_get(&pdev->dev, "sys");
 948         if (IS_ERR(hdev->sys_clk)) {
 949                 dev_err(dev, "clock initialization failed.\n");
 950                 err = PTR_ERR(hdev->sys_clk);
 951                 goto res_err;
 952         }
 953
 954         err = clk_prepare_enable(hdev->hash_clk);
 955         if (err)
 956                 goto res_err;
 957
 958         err = clk_prepare_enable(hdev->sys_clk);
 959         if (err)
 960                 goto clk_err;
 961
 962         err = img_hash_dma_init(hdev);
 963         if (err)
 964                 goto dma_err;
 965
 966         dev_dbg(dev, "using %s for DMA transfers\n",
 967                 dma_chan_name(hdev->dma_lch));
 968
 969         spin_lock(&img_hash.lock);
 970         list_add_tail(&hdev->list, &img_hash.dev_list);
 971         spin_unlock(&img_hash.lock);
 972
 973         err = img_register_algs(hdev);
 974         if (err)
 975                 goto err_algs;
 976         dev_dbg(dev, "Img MD5/SHA1/SHA224/SHA256 Hardware accelerator initialized\n");
 977
 978         return 0;
 979
 980 err_algs:
 981         spin_lock(&img_hash.lock);
 982         list_del(&hdev->list);
 983         spin_unlock(&img_hash.lock);
 984         dma_release_channel(hdev->dma_lch);
 985 dma_err:
 986         clk_disable_unprepare(hdev->sys_clk);
 987 clk_err:
 988         clk_disable_unprepare(hdev->hash_clk);
 989 res_err:
 990         tasklet_kill(&hdev->done_task);
 991         tasklet_kill(&hdev->dma_task);
 992
 993         return err;
 994 }
 995
 996 static int img_hash_remove(struct platform_device *pdev)
 997 {
 998         static struct img_hash_dev *hdev;
 999
1000         hdev = platform_get_drvdata(pdev);
1001         spin_lock(&img_hash.lock);
1002         list_del(&hdev->list);
1003         spin_unlock(&img_hash.lock);
1004
1005         img_unregister_algs(hdev);
1006
1007         tasklet_kill(&hdev->done_task);
1008         tasklet_kill(&hdev->dma_task);
1009
1010         dma_release_channel(hdev->dma_lch);
1011
1012         clk_disable_unprepare(hdev->hash_clk);
1013         clk_disable_unprepare(hdev->sys_clk);
1014
1015         return 0;
1016 }
1017
1018 static struct platform_driver img_hash_driver = {
1019         .probe          = img_hash_probe,
1020         .remove         = img_hash_remove,
1021         .driver         = {
1022                 .name   = "img-hash-accelerator",
1023                 .of_match_table = of_match_ptr(img_hash_match),
1024         }
1025 };
1026 module_platform_driver(img_hash_driver);
1027
1028 MODULE_LICENSE("GPL v2");
1029 MODULE_DESCRIPTION("Imgtec SHA1/224/256 & MD5 hw accelerator driver");
1030 MODULE_AUTHOR("Will Thomas.");
1031 MODULE_AUTHOR("James Hartley <james.hartley@imgtec.com>");