drivers/dma/dmatest.c

   1 /*
   2  * DMA Engine test module
   3  *
   4  * Copyright (C) 2007 Atmel Corporation
   5  * Copyright (C) 2013 Intel Corporation
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License version 2 as
   9  * published by the Free Software Foundation.
  10  */
  11 #include <linux/delay.h>
  12 #include <linux/dma-mapping.h>
  13 #include <linux/dmaengine.h>
  14 #include <linux/freezer.h>
  15 #include <linux/init.h>
  16 #include <linux/kthread.h>
  17 #include <linux/module.h>
  18 #include <linux/moduleparam.h>
  19 #include <linux/random.h>
  20 #include <linux/slab.h>
  21 #include <linux/wait.h>
  22 #include <linux/ctype.h>
  23 #include <linux/debugfs.h>
  24 #include <linux/uaccess.h>
  25 #include <linux/seq_file.h>
  26
  27 static unsigned int test_buf_size = 16384;
  28 module_param(test_buf_size, uint, S_IRUGO | S_IWUSR);
  29 MODULE_PARM_DESC(test_buf_size, "Size of the memcpy test buffer");
  30
  31 static char test_channel[20];
  32 module_param_string(channel, test_channel, sizeof(test_channel),
  33                 S_IRUGO | S_IWUSR);
  34 MODULE_PARM_DESC(channel, "Bus ID of the channel to test (default: any)");
  35
  36 static char test_device[20];
  37 module_param_string(device, test_device, sizeof(test_device),
  38                 S_IRUGO | S_IWUSR);
  39 MODULE_PARM_DESC(device, "Bus ID of the DMA Engine to test (default: any)");
  40
  41 static unsigned int threads_per_chan = 1;
  42 module_param(threads_per_chan, uint, S_IRUGO | S_IWUSR);
  43 MODULE_PARM_DESC(threads_per_chan,
  44                 "Number of threads to start per channel (default: 1)");
  45
  46 static unsigned int max_channels;
  47 module_param(max_channels, uint, S_IRUGO | S_IWUSR);
  48 MODULE_PARM_DESC(max_channels,
  49                 "Maximum number of channels to use (default: all)");
  50
  51 static unsigned int iterations;
  52 module_param(iterations, uint, S_IRUGO | S_IWUSR);
  53 MODULE_PARM_DESC(iterations,
  54                 "Iterations before stopping test (default: infinite)");
  55
  56 static unsigned int xor_sources = 3;
  57 module_param(xor_sources, uint, S_IRUGO | S_IWUSR);
  58 MODULE_PARM_DESC(xor_sources,
  59                 "Number of xor source buffers (default: 3)");
  60
  61 static unsigned int pq_sources = 3;
  62 module_param(pq_sources, uint, S_IRUGO | S_IWUSR);
  63 MODULE_PARM_DESC(pq_sources,
  64                 "Number of p+q source buffers (default: 3)");
  65
  66 static int timeout = 3000;
  67 module_param(timeout, uint, S_IRUGO | S_IWUSR);
  68 MODULE_PARM_DESC(timeout, "Transfer Timeout in msec (default: 3000), "
  69                  "Pass -1 for infinite timeout");
  70
  71 /* Maximum amount of mismatched bytes in buffer to print */
  72 #define MAX_ERROR_COUNT         32
  73
  74 /*
  75  * Initialization patterns. All bytes in the source buffer has bit 7
  76  * set, all bytes in the destination buffer has bit 7 cleared.
  77  *
  78  * Bit 6 is set for all bytes which are to be copied by the DMA
  79  * engine. Bit 5 is set for all bytes which are to be overwritten by
  80  * the DMA engine.
  81  *
  82  * The remaining bits are the inverse of a counter which increments by
  83  * one for each byte address.
  84  */
  85 #define PATTERN_SRC             0x80
  86 #define PATTERN_DST             0x00
  87 #define PATTERN_COPY            0x40
  88 #define PATTERN_OVERWRITE       0x20
  89 #define PATTERN_COUNT_MASK      0x1f
  90
  91 enum dmatest_error_type {
  92         DMATEST_ET_OK,
  93         DMATEST_ET_MAP_SRC,
  94         DMATEST_ET_MAP_DST,
  95         DMATEST_ET_PREP,
  96         DMATEST_ET_SUBMIT,
  97         DMATEST_ET_TIMEOUT,
  98         DMATEST_ET_DMA_ERROR,
  99         DMATEST_ET_DMA_IN_PROGRESS,
 100         DMATEST_ET_VERIFY,
 101         DMATEST_ET_VERIFY_BUF,
 102 };
 103
 104 struct dmatest_verify_buffer {
 105         unsigned int    index;
 106         u8              expected;
 107         u8              actual;
 108 };
 109
 110 struct dmatest_verify_result {
 111         unsigned int                    error_count;
 112         struct dmatest_verify_buffer    data[MAX_ERROR_COUNT];
 113         u8                              pattern;
 114         bool                            is_srcbuf;
 115 };
 116
 117 struct dmatest_thread_result {
 118         struct list_head        node;
 119         unsigned int            n;
 120         unsigned int            src_off;
 121         unsigned int            dst_off;
 122         unsigned int            len;
 123         enum dmatest_error_type type;
 124         union {
 125                 unsigned long                   data;
 126                 dma_cookie_t                    cookie;
 127                 enum dma_status                 status;
 128                 int                             error;
 129                 struct dmatest_verify_result    *vr;
 130         };
 131 };
 132
 133 struct dmatest_result {
 134         struct list_head        node;
 135         char                    *name;
 136         struct list_head        results;
 137 };
 138
 139 struct dmatest_info;
 140
 141 struct dmatest_thread {
 142         struct list_head        node;
 143         struct dmatest_info     *info;
 144         struct task_struct      *task;
 145         struct dma_chan         *chan;
 146         u8                      **srcs;
 147         u8                      **dsts;
 148         enum dma_transaction_type type;
 149         bool                    done;
 150 };
 151
 152 struct dmatest_chan {
 153         struct list_head        node;
 154         struct dma_chan         *chan;
 155         struct list_head        threads;
 156 };
 157
 158 /**
 159  * struct dmatest_params - test parameters.
 160  * @buf_size:           size of the memcpy test buffer
 161  * @channel:            bus ID of the channel to test
 162  * @device:             bus ID of the DMA Engine to test
 163  * @threads_per_chan:   number of threads to start per channel
 164  * @max_channels:       maximum number of channels to use
 165  * @iterations:         iterations before stopping test
 166  * @xor_sources:        number of xor source buffers
 167  * @pq_sources:         number of p+q source buffers
 168  * @timeout:            transfer timeout in msec, -1 for infinite timeout
 169  */
 170 struct dmatest_params {
 171         unsigned int    buf_size;
 172         char            channel[20];
 173         char            device[20];
 174         unsigned int    threads_per_chan;
 175         unsigned int    max_channels;
 176         unsigned int    iterations;
 177         unsigned int    xor_sources;
 178         unsigned int    pq_sources;
 179         int             timeout;
 180 };
 181
 182 /**
 183  * struct dmatest_info - test information.
 184  * @params:             test parameters
 185  * @lock:               access protection to the fields of this structure
 186  */
 187 struct dmatest_info {
 188         /* Test parameters */
 189         struct dmatest_params   params;
 190
 191         /* Internal state */
 192         struct list_head        channels;
 193         unsigned int            nr_channels;
 194         struct mutex            lock;
 195
 196         /* debugfs related stuff */
 197         struct dentry           *root;
 198
 199         /* Test results */
 200         struct list_head        results;
 201         struct mutex            results_lock;
 202 };
 203
 204 static struct dmatest_info test_info;
 205
 206 static bool dmatest_match_channel(struct dmatest_params *params,
 207                 struct dma_chan *chan)
 208 {
 209         if (params->channel[0] == '\0')
 210                 return true;
 211         return strcmp(dma_chan_name(chan), params->channel) == 0;
 212 }
 213
 214 static bool dmatest_match_device(struct dmatest_params *params,
 215                 struct dma_device *device)
 216 {
 217         if (params->device[0] == '\0')
 218                 return true;
 219         return strcmp(dev_name(device->dev), params->device) == 0;
 220 }
 221
 222 static unsigned long dmatest_random(void)
 223 {
 224         unsigned long buf;
 225
 226         get_random_bytes(&buf, sizeof(buf));
 227         return buf;
 228 }
 229
 230 static void dmatest_init_srcs(u8 **bufs, unsigned int start, unsigned int len,
 231                 unsigned int buf_size)
 232 {
 233         unsigned int i;
 234         u8 *buf;
 235
 236         for (; (buf = *bufs); bufs++) {
 237                 for (i = 0; i < start; i++)
 238                         buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK);
 239                 for ( ; i < start + len; i++)
 240                         buf[i] = PATTERN_SRC | PATTERN_COPY
 241                                 | (~i & PATTERN_COUNT_MASK);
 242                 for ( ; i < buf_size; i++)
 243                         buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK);
 244                 buf++;
 245         }
 246 }
 247
 248 static void dmatest_init_dsts(u8 **bufs, unsigned int start, unsigned int len,
 249                 unsigned int buf_size)
 250 {
 251         unsigned int i;
 252         u8 *buf;
 253
 254         for (; (buf = *bufs); bufs++) {
 255                 for (i = 0; i < start; i++)
 256                         buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK);
 257                 for ( ; i < start + len; i++)
 258                         buf[i] = PATTERN_DST | PATTERN_OVERWRITE
 259                                 | (~i & PATTERN_COUNT_MASK);
 260                 for ( ; i < buf_size; i++)
 261                         buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK);
 262         }
 263 }
 264
 265 static unsigned int dmatest_verify(struct dmatest_verify_result *vr, u8 **bufs,
 266                 unsigned int start, unsigned int end, unsigned int counter,
 267                 u8 pattern, bool is_srcbuf)
 268 {
 269         unsigned int i;
 270         unsigned int error_count = 0;
 271         u8 actual;
 272         u8 expected;
 273         u8 *buf;
 274         unsigned int counter_orig = counter;
 275         struct dmatest_verify_buffer *vb;
 276
 277         for (; (buf = *bufs); bufs++) {
 278                 counter = counter_orig;
 279                 for (i = start; i < end; i++) {
 280                         actual = buf[i];
 281                         expected = pattern | (~counter & PATTERN_COUNT_MASK);
 282                         if (actual != expected) {
 283                                 if (error_count < MAX_ERROR_COUNT && vr) {
 284                                         vb = &vr->data[error_count];
 285                                         vb->index = i;
 286                                         vb->expected = expected;
 287                                         vb->actual = actual;
 288                                 }
 289                                 error_count++;
 290                         }
 291                         counter++;
 292                 }
 293         }
 294
 295         if (error_count > MAX_ERROR_COUNT)
 296                 pr_warning("%s: %u errors suppressed\n",
 297                         current->comm, error_count - MAX_ERROR_COUNT);
 298
 299         return error_count;
 300 }
 301
 302 /* poor man's completion - we want to use wait_event_freezable() on it */
 303 struct dmatest_done {
 304         bool                    done;
 305         wait_queue_head_t       *wait;
 306 };
 307
 308 static void dmatest_callback(void *arg)
 309 {
 310         struct dmatest_done *done = arg;
 311
 312         done->done = true;
 313         wake_up_all(done->wait);
 314 }
 315
 316 static inline void unmap_src(struct device *dev, dma_addr_t *addr, size_t len,
 317                              unsigned int count)
 318 {
 319         while (count--)
 320                 dma_unmap_single(dev, addr[count], len, DMA_TO_DEVICE);
 321 }
 322
 323 static inline void unmap_dst(struct device *dev, dma_addr_t *addr, size_t len,
 324                              unsigned int count)
 325 {
 326         while (count--)
 327                 dma_unmap_single(dev, addr[count], len, DMA_BIDIRECTIONAL);
 328 }
 329
 330 static unsigned int min_odd(unsigned int x, unsigned int y)
 331 {
 332         unsigned int val = min(x, y);
 333
 334         return val % 2 ? val : val - 1;
 335 }
 336
 337 static char *verify_result_get_one(struct dmatest_verify_result *vr,
 338                 unsigned int i)
 339 {
 340         struct dmatest_verify_buffer *vb = &vr->data[i];
 341         u8 diff = vb->actual ^ vr->pattern;
 342         static char buf[512];
 343         char *msg;
 344
 345         if (vr->is_srcbuf)
 346                 msg = "srcbuf overwritten!";
 347         else if ((vr->pattern & PATTERN_COPY)
 348                         && (diff & (PATTERN_COPY | PATTERN_OVERWRITE)))
 349                 msg = "dstbuf not copied!";
 350         else if (diff & PATTERN_SRC)
 351                 msg = "dstbuf was copied!";
 352         else
 353                 msg = "dstbuf mismatch!";
 354
 355         snprintf(buf, sizeof(buf) - 1, "%s [0x%x] Expected %02x, got %02x", msg,
 356                  vb->index, vb->expected, vb->actual);
 357
 358         return buf;
 359 }
 360
 361 static char *thread_result_get(const char *name,
 362                 struct dmatest_thread_result *tr)
 363 {
 364         static const char * const messages[] = {
 365                 [DMATEST_ET_OK]                 = "No errors",
 366                 [DMATEST_ET_MAP_SRC]            = "src mapping error",
 367                 [DMATEST_ET_MAP_DST]            = "dst mapping error",
 368                 [DMATEST_ET_PREP]               = "prep error",
 369                 [DMATEST_ET_SUBMIT]             = "submit error",
 370                 [DMATEST_ET_TIMEOUT]            = "test timed out",
 371                 [DMATEST_ET_DMA_ERROR]          =
 372                         "got completion callback (DMA_ERROR)",
 373                 [DMATEST_ET_DMA_IN_PROGRESS]    =
 374                         "got completion callback (DMA_IN_PROGRESS)",
 375                 [DMATEST_ET_VERIFY]             = "errors",
 376                 [DMATEST_ET_VERIFY_BUF]         = "verify errors",
 377         };
 378         static char buf[512];
 379
 380         snprintf(buf, sizeof(buf) - 1,
 381                  "%s: #%u: %s with src_off=0x%x ""dst_off=0x%x len=0x%x (%lu)",
 382                  name, tr->n, messages[tr->type], tr->src_off, tr->dst_off,
 383                  tr->len, tr->data);
 384
 385         return buf;
 386 }
 387
 388 static int thread_result_add(struct dmatest_info *info,
 389                 struct dmatest_result *r, enum dmatest_error_type type,
 390                 unsigned int n, unsigned int src_off, unsigned int dst_off,
 391                 unsigned int len, unsigned long data)
 392 {
 393         struct dmatest_thread_result *tr;
 394
 395         tr = kzalloc(sizeof(*tr), GFP_KERNEL);
 396         if (!tr)
 397                 return -ENOMEM;
 398
 399         tr->type = type;
 400         tr->n = n;
 401         tr->src_off = src_off;
 402         tr->dst_off = dst_off;
 403         tr->len = len;
 404         tr->data = data;
 405
 406         mutex_lock(&info->results_lock);
 407         list_add_tail(&tr->node, &r->results);
 408         mutex_unlock(&info->results_lock);
 409
 410         if (tr->type == DMATEST_ET_OK)
 411                 pr_debug("%s\n", thread_result_get(r->name, tr));
 412         else
 413                 pr_warn("%s\n", thread_result_get(r->name, tr));
 414
 415         return 0;
 416 }
 417
 418 static unsigned int verify_result_add(struct dmatest_info *info,
 419                 struct dmatest_result *r, unsigned int n,
 420                 unsigned int src_off, unsigned int dst_off, unsigned int len,
 421                 u8 **bufs, int whence, unsigned int counter, u8 pattern,
 422                 bool is_srcbuf)
 423 {
 424         struct dmatest_verify_result *vr;
 425         unsigned int error_count;
 426         unsigned int buf_off = is_srcbuf ? src_off : dst_off;
 427         unsigned int start, end;
 428
 429         if (whence < 0) {
 430                 start = 0;
 431                 end = buf_off;
 432         } else if (whence > 0) {
 433                 start = buf_off + len;
 434                 end = info->params.buf_size;
 435         } else {
 436                 start = buf_off;
 437                 end = buf_off + len;
 438         }
 439
 440         vr = kmalloc(sizeof(*vr), GFP_KERNEL);
 441         if (!vr) {
 442                 pr_warn("dmatest: No memory to store verify result\n");
 443                 return dmatest_verify(NULL, bufs, start, end, counter, pattern,
 444                                       is_srcbuf);
 445         }
 446
 447         vr->pattern = pattern;
 448         vr->is_srcbuf = is_srcbuf;
 449
 450         error_count = dmatest_verify(vr, bufs, start, end, counter, pattern,
 451                                      is_srcbuf);
 452         if (error_count) {
 453                 vr->error_count = error_count;
 454                 thread_result_add(info, r, DMATEST_ET_VERIFY_BUF, n, src_off,
 455                                   dst_off, len, (unsigned long)vr);
 456                 return error_count;
 457         }
 458
 459         kfree(vr);
 460         return 0;
 461 }
 462
 463 static void result_free(struct dmatest_info *info, const char *name)
 464 {
 465         struct dmatest_result *r, *_r;
 466
 467         mutex_lock(&info->results_lock);
 468         list_for_each_entry_safe(r, _r, &info->results, node) {
 469                 struct dmatest_thread_result *tr, *_tr;
 470
 471                 if (name && strcmp(r->name, name))
 472                         continue;
 473
 474                 list_for_each_entry_safe(tr, _tr, &r->results, node) {
 475                         if (tr->type == DMATEST_ET_VERIFY_BUF)
 476                                 kfree(tr->vr);
 477                         list_del(&tr->node);
 478                         kfree(tr);
 479                 }
 480
 481                 kfree(r->name);
 482                 list_del(&r->node);
 483                 kfree(r);
 484         }
 485
 486         mutex_unlock(&info->results_lock);
 487 }
 488
 489 static struct dmatest_result *result_init(struct dmatest_info *info,
 490                 const char *name)
 491 {
 492         struct dmatest_result *r;
 493
 494         r = kzalloc(sizeof(*r), GFP_KERNEL);
 495         if (r) {
 496                 r->name = kstrdup(name, GFP_KERNEL);
 497                 INIT_LIST_HEAD(&r->results);
 498                 mutex_lock(&info->results_lock);
 499                 list_add_tail(&r->node, &info->results);
 500                 mutex_unlock(&info->results_lock);
 501         }
 502         return r;
 503 }
 504
 505 /*
 506  * This function repeatedly tests DMA transfers of various lengths and
 507  * offsets for a given operation type until it is told to exit by
 508  * kthread_stop(). There may be multiple threads running this function
 509  * in parallel for a single channel, and there may be multiple channels
 510  * being tested in parallel.
 511  *
 512  * Before each test, the source and destination buffer is initialized
 513  * with a known pattern. This pattern is different depending on
 514  * whether it's in an area which is supposed to be copied or
 515  * overwritten, and different in the source and destination buffers.
 516  * So if the DMA engine doesn't copy exactly what we tell it to copy,
 517  * we'll notice.
 518  */
 519 static int dmatest_func(void *data)
 520 {
 521         DECLARE_WAIT_QUEUE_HEAD_ONSTACK(done_wait);
 522         struct dmatest_thread   *thread = data;
 523         struct dmatest_done     done = { .wait = &done_wait };
 524         struct dmatest_info     *info;
 525         struct dmatest_params   *params;
 526         struct dma_chan         *chan;
 527         struct dma_device       *dev;
 528         const char              *thread_name;
 529         unsigned int            src_off, dst_off, len;
 530         unsigned int            error_count;
 531         unsigned int            failed_tests = 0;
 532         unsigned int            total_tests = 0;
 533         dma_cookie_t            cookie;
 534         enum dma_status         status;
 535         enum dma_ctrl_flags     flags;
 536         u8                      *pq_coefs = NULL;
 537         int                     ret;
 538         int                     src_cnt;
 539         int                     dst_cnt;
 540         int                     i;
 541         struct dmatest_result   *result;
 542
 543         thread_name = current->comm;
 544         set_freezable();
 545
 546         ret = -ENOMEM;
 547
 548         smp_rmb();
 549         info = thread->info;
 550         params = &info->params;
 551         chan = thread->chan;
 552         dev = chan->device;
 553         if (thread->type == DMA_MEMCPY)
 554                 src_cnt = dst_cnt = 1;
 555         else if (thread->type == DMA_XOR) {
 556                 /* force odd to ensure dst = src */
 557                 src_cnt = min_odd(params->xor_sources | 1, dev->max_xor);
 558                 dst_cnt = 1;
 559         } else if (thread->type == DMA_PQ) {
 560                 /* force odd to ensure dst = src */
 561                 src_cnt = min_odd(params->pq_sources | 1, dma_maxpq(dev, 0));
 562                 dst_cnt = 2;
 563
 564                 pq_coefs = kmalloc(params->pq_sources+1, GFP_KERNEL);
 565                 if (!pq_coefs)
 566                         goto err_thread_type;
 567
 568                 for (i = 0; i < src_cnt; i++)
 569                         pq_coefs[i] = 1;
 570         } else
 571                 goto err_thread_type;
 572
 573         result = result_init(info, thread_name);
 574         if (!result)
 575                 goto err_srcs;
 576
 577         thread->srcs = kcalloc(src_cnt+1, sizeof(u8 *), GFP_KERNEL);
 578         if (!thread->srcs)
 579                 goto err_srcs;
 580         for (i = 0; i < src_cnt; i++) {
 581                 thread->srcs[i] = kmalloc(params->buf_size, GFP_KERNEL);
 582                 if (!thread->srcs[i])
 583                         goto err_srcbuf;
 584         }
 585         thread->srcs[i] = NULL;
 586
 587         thread->dsts = kcalloc(dst_cnt+1, sizeof(u8 *), GFP_KERNEL);
 588         if (!thread->dsts)
 589                 goto err_dsts;
 590         for (i = 0; i < dst_cnt; i++) {
 591                 thread->dsts[i] = kmalloc(params->buf_size, GFP_KERNEL);
 592                 if (!thread->dsts[i])
 593                         goto err_dstbuf;
 594         }
 595         thread->dsts[i] = NULL;
 596
 597         set_user_nice(current, 10);
 598
 599         /*
 600          * src buffers are freed by the DMAEngine code with dma_unmap_single()
 601          * dst buffers are freed by ourselves below
 602          */
 603         flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT
 604               | DMA_COMPL_SKIP_DEST_UNMAP | DMA_COMPL_SRC_UNMAP_SINGLE;
 605
 606         while (!kthread_should_stop()
 607                && !(params->iterations && total_tests >= params->iterations)) {
 608                 struct dma_async_tx_descriptor *tx = NULL;
 609                 dma_addr_t dma_srcs[src_cnt];
 610                 dma_addr_t dma_dsts[dst_cnt];
 611                 u8 align = 0;
 612
 613                 total_tests++;
 614
 615                 /* honor alignment restrictions */
 616                 if (thread->type == DMA_MEMCPY)
 617                         align = dev->copy_align;
 618                 else if (thread->type == DMA_XOR)
 619                         align = dev->xor_align;
 620                 else if (thread->type == DMA_PQ)
 621                         align = dev->pq_align;
 622
 623                 if (1 << align > params->buf_size) {
 624                         pr_err("%u-byte buffer too small for %d-byte alignment\n",
 625                                params->buf_size, 1 << align);
 626                         break;
 627                 }
 628
 629                 len = dmatest_random() % params->buf_size + 1;
 630                 len = (len >> align) << align;
 631                 if (!len)
 632                         len = 1 << align;
 633                 src_off = dmatest_random() % (params->buf_size - len + 1);
 634                 dst_off = dmatest_random() % (params->buf_size - len + 1);
 635
 636                 src_off = (src_off >> align) << align;
 637                 dst_off = (dst_off >> align) << align;
 638
 639                 dmatest_init_srcs(thread->srcs, src_off, len, params->buf_size);
 640                 dmatest_init_dsts(thread->dsts, dst_off, len, params->buf_size);
 641
 642                 for (i = 0; i < src_cnt; i++) {
 643                         u8 *buf = thread->srcs[i] + src_off;
 644
 645                         dma_srcs[i] = dma_map_single(dev->dev, buf, len,
 646                                                      DMA_TO_DEVICE);
 647                         ret = dma_mapping_error(dev->dev, dma_srcs[i]);
 648                         if (ret) {
 649                                 unmap_src(dev->dev, dma_srcs, len, i);
 650                                 thread_result_add(info, result,
 651                                                   DMATEST_ET_MAP_SRC,
 652                                                   total_tests, src_off, dst_off,
 653                                                   len, ret);
 654                                 failed_tests++;
 655                                 continue;
 656                         }
 657                 }
 658                 /* map with DMA_BIDIRECTIONAL to force writeback/invalidate */
 659                 for (i = 0; i < dst_cnt; i++) {
 660                         dma_dsts[i] = dma_map_single(dev->dev, thread->dsts[i],
 661                                                      params->buf_size,
 662                                                      DMA_BIDIRECTIONAL);
 663                         ret = dma_mapping_error(dev->dev, dma_dsts[i]);
 664                         if (ret) {
 665                                 unmap_src(dev->dev, dma_srcs, len, src_cnt);
 666                                 unmap_dst(dev->dev, dma_dsts, params->buf_size,
 667                                           i);
 668                                 thread_result_add(info, result,
 669                                                   DMATEST_ET_MAP_DST,
 670                                                   total_tests, src_off, dst_off,
 671                                                   len, ret);
 672                                 failed_tests++;
 673                                 continue;
 674                         }
 675                 }
 676
 677                 if (thread->type == DMA_MEMCPY)
 678                         tx = dev->device_prep_dma_memcpy(chan,
 679                                                          dma_dsts[0] + dst_off,
 680                                                          dma_srcs[0], len,
 681                                                          flags);
 682                 else if (thread->type == DMA_XOR)
 683                         tx = dev->device_prep_dma_xor(chan,
 684                                                       dma_dsts[0] + dst_off,
 685                                                       dma_srcs, src_cnt,
 686                                                       len, flags);
 687                 else if (thread->type == DMA_PQ) {
 688                         dma_addr_t dma_pq[dst_cnt];
 689
 690                         for (i = 0; i < dst_cnt; i++)
 691                                 dma_pq[i] = dma_dsts[i] + dst_off;
 692                         tx = dev->device_prep_dma_pq(chan, dma_pq, dma_srcs,
 693                                                      src_cnt, pq_coefs,
 694                                                      len, flags);
 695                 }
 696
 697                 if (!tx) {
 698                         unmap_src(dev->dev, dma_srcs, len, src_cnt);
 699                         unmap_dst(dev->dev, dma_dsts, params->buf_size,
 700                                   dst_cnt);
 701                         thread_result_add(info, result, DMATEST_ET_PREP,
 702                                           total_tests, src_off, dst_off,
 703                                           len, 0);
 704                         msleep(100);
 705                         failed_tests++;
 706                         continue;
 707                 }
 708
 709                 done.done = false;
 710                 tx->callback = dmatest_callback;
 711                 tx->callback_param = &done;
 712                 cookie = tx->tx_submit(tx);
 713
 714                 if (dma_submit_error(cookie)) {
 715                         thread_result_add(info, result, DMATEST_ET_SUBMIT,
 716                                           total_tests, src_off, dst_off,
 717                                           len, cookie);
 718                         msleep(100);
 719                         failed_tests++;
 720                         continue;
 721                 }
 722                 dma_async_issue_pending(chan);
 723
 724                 wait_event_freezable_timeout(done_wait, done.done,
 725                                              msecs_to_jiffies(params->timeout));
 726
 727                 status = dma_async_is_tx_complete(chan, cookie, NULL, NULL);
 728
 729                 if (!done.done) {
 730                         /*
 731                          * We're leaving the timed out dma operation with
 732                          * dangling pointer to done_wait.  To make this
 733                          * correct, we'll need to allocate wait_done for
 734                          * each test iteration and perform "who's gonna
 735                          * free it this time?" dancing.  For now, just
 736                          * leave it dangling.
 737                          */
 738                         thread_result_add(info, result, DMATEST_ET_TIMEOUT,
 739                                           total_tests, src_off, dst_off,
 740                                           len, 0);
 741                         failed_tests++;
 742                         continue;
 743                 } else if (status != DMA_SUCCESS) {
 744                         enum dmatest_error_type type = (status == DMA_ERROR) ?
 745                                 DMATEST_ET_DMA_ERROR : DMATEST_ET_DMA_IN_PROGRESS;
 746                         thread_result_add(info, result, type,
 747                                           total_tests, src_off, dst_off,
 748                                           len, status);
 749                         failed_tests++;
 750                         continue;
 751                 }
 752
 753                 /* Unmap by myself (see DMA_COMPL_SKIP_DEST_UNMAP above) */
 754                 unmap_dst(dev->dev, dma_dsts, params->buf_size, dst_cnt);
 755
 756                 error_count = 0;
 757
 758                 pr_debug("%s: verifying source buffer...\n", thread_name);
 759                 error_count += verify_result_add(info, result, total_tests,
 760                                 src_off, dst_off, len, thread->srcs, -1,
 761                                 0, PATTERN_SRC, true);
 762                 error_count += verify_result_add(info, result, total_tests,
 763                                 src_off, dst_off, len, thread->srcs, 0,
 764                                 src_off, PATTERN_SRC | PATTERN_COPY, true);
 765                 error_count += verify_result_add(info, result, total_tests,
 766                                 src_off, dst_off, len, thread->srcs, 1,
 767                                 src_off + len, PATTERN_SRC, true);
 768
 769                 pr_debug("%s: verifying dest buffer...\n", thread_name);
 770                 error_count += verify_result_add(info, result, total_tests,
 771                                 src_off, dst_off, len, thread->dsts, -1,
 772                                 0, PATTERN_DST, false);
 773                 error_count += verify_result_add(info, result, total_tests,
 774                                 src_off, dst_off, len, thread->dsts, 0,
 775                                 src_off, PATTERN_SRC | PATTERN_COPY, false);
 776                 error_count += verify_result_add(info, result, total_tests,
 777                                 src_off, dst_off, len, thread->dsts, 1,
 778                                 dst_off + len, PATTERN_DST, false);
 779
 780                 if (error_count) {
 781                         thread_result_add(info, result, DMATEST_ET_VERIFY,
 782                                           total_tests, src_off, dst_off,
 783                                           len, error_count);
 784                         failed_tests++;
 785                 } else {
 786                         thread_result_add(info, result, DMATEST_ET_OK,
 787                                           total_tests, src_off, dst_off,
 788                                           len, 0);
 789                 }
 790         }
 791
 792         ret = 0;
 793         for (i = 0; thread->dsts[i]; i++)
 794                 kfree(thread->dsts[i]);
 795 err_dstbuf:
 796         kfree(thread->dsts);
 797 err_dsts:
 798         for (i = 0; thread->srcs[i]; i++)
 799                 kfree(thread->srcs[i]);
 800 err_srcbuf:
 801         kfree(thread->srcs);
 802 err_srcs:
 803         kfree(pq_coefs);
 804 err_thread_type:
 805         pr_notice("%s: terminating after %u tests, %u failures (status %d)\n",
 806                         thread_name, total_tests, failed_tests, ret);
 807
 808         /* terminate all transfers on specified channels */
 809         if (ret)
 810                 dmaengine_terminate_all(chan);
 811
 812         thread->done = true;
 813
 814         if (params->iterations > 0)
 815                 while (!kthread_should_stop()) {
 816                         DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wait_dmatest_exit);
 817                         interruptible_sleep_on(&wait_dmatest_exit);
 818                 }
 819
 820         return ret;
 821 }
 822
 823 static void dmatest_cleanup_channel(struct dmatest_chan *dtc)
 824 {
 825         struct dmatest_thread   *thread;
 826         struct dmatest_thread   *_thread;
 827         int                     ret;
 828
 829         list_for_each_entry_safe(thread, _thread, &dtc->threads, node) {
 830                 ret = kthread_stop(thread->task);
 831                 pr_debug("dmatest: thread %s exited with status %d\n",
 832                                 thread->task->comm, ret);
 833                 list_del(&thread->node);
 834                 kfree(thread);
 835         }
 836
 837         /* terminate all transfers on specified channels */
 838         dmaengine_terminate_all(dtc->chan);
 839
 840         kfree(dtc);
 841 }
 842
 843 static int dmatest_add_threads(struct dmatest_info *info,
 844                 struct dmatest_chan *dtc, enum dma_transaction_type type)
 845 {
 846         struct dmatest_params *params = &info->params;
 847         struct dmatest_thread *thread;
 848         struct dma_chan *chan = dtc->chan;
 849         char *op;
 850         unsigned int i;
 851
 852         if (type == DMA_MEMCPY)
 853                 op = "copy";
 854         else if (type == DMA_XOR)
 855                 op = "xor";
 856         else if (type == DMA_PQ)
 857                 op = "pq";
 858         else
 859                 return -EINVAL;
 860
 861         for (i = 0; i < params->threads_per_chan; i++) {
 862                 thread = kzalloc(sizeof(struct dmatest_thread), GFP_KERNEL);
 863                 if (!thread) {
 864                         pr_warning("dmatest: No memory for %s-%s%u\n",
 865                                    dma_chan_name(chan), op, i);
 866
 867                         break;
 868                 }
 869                 thread->info = info;
 870                 thread->chan = dtc->chan;
 871                 thread->type = type;
 872                 smp_wmb();
 873                 thread->task = kthread_run(dmatest_func, thread, "%s-%s%u",
 874                                 dma_chan_name(chan), op, i);
 875                 if (IS_ERR(thread->task)) {
 876                         pr_warning("dmatest: Failed to run thread %s-%s%u\n",
 877                                         dma_chan_name(chan), op, i);
 878                         kfree(thread);
 879                         break;
 880                 }
 881
 882                 /* srcbuf and dstbuf are allocated by the thread itself */
 883
 884                 list_add_tail(&thread->node, &dtc->threads);
 885         }
 886
 887         return i;
 888 }
 889
 890 static int dmatest_add_channel(struct dmatest_info *info,
 891                 struct dma_chan *chan)
 892 {
 893         struct dmatest_chan     *dtc;
 894         struct dma_device       *dma_dev = chan->device;
 895         unsigned int            thread_count = 0;
 896         int cnt;
 897
 898         dtc = kmalloc(sizeof(struct dmatest_chan), GFP_KERNEL);
 899         if (!dtc) {
 900                 pr_warning("dmatest: No memory for %s\n", dma_chan_name(chan));
 901                 return -ENOMEM;
 902         }
 903
 904         dtc->chan = chan;
 905         INIT_LIST_HEAD(&dtc->threads);
 906
 907         if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) {
 908                 cnt = dmatest_add_threads(info, dtc, DMA_MEMCPY);
 909                 thread_count += cnt > 0 ? cnt : 0;
 910         }
 911         if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
 912                 cnt = dmatest_add_threads(info, dtc, DMA_XOR);
 913                 thread_count += cnt > 0 ? cnt : 0;
 914         }
 915         if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) {
 916                 cnt = dmatest_add_threads(info, dtc, DMA_PQ);
 917                 thread_count += cnt > 0 ? cnt : 0;
 918         }
 919
 920         pr_info("dmatest: Started %u threads using %s\n",
 921                 thread_count, dma_chan_name(chan));
 922
 923         list_add_tail(&dtc->node, &info->channels);
 924         info->nr_channels++;
 925
 926         return 0;
 927 }
 928
 929 static bool filter(struct dma_chan *chan, void *param)
 930 {
 931         struct dmatest_params *params = param;
 932
 933         if (!dmatest_match_channel(params, chan) ||
 934             !dmatest_match_device(params, chan->device))
 935                 return false;
 936         else
 937                 return true;
 938 }
 939
 940 static int __run_threaded_test(struct dmatest_info *info)
 941 {
 942         dma_cap_mask_t mask;
 943         struct dma_chan *chan;
 944         struct dmatest_params *params = &info->params;
 945         int err = 0;
 946
 947         dma_cap_zero(mask);
 948         dma_cap_set(DMA_MEMCPY, mask);
 949         for (;;) {
 950                 chan = dma_request_channel(mask, filter, params);
 951                 if (chan) {
 952                         err = dmatest_add_channel(info, chan);
 953                         if (err) {
 954                                 dma_release_channel(chan);
 955                                 break; /* add_channel failed, punt */
 956                         }
 957                 } else
 958                         break; /* no more channels available */
 959                 if (params->max_channels &&
 960                     info->nr_channels >= params->max_channels)
 961                         break; /* we have all we need */
 962         }
 963         return err;
 964 }
 965
 966 #ifndef MODULE
 967 static int run_threaded_test(struct dmatest_info *info)
 968 {
 969         int ret;
 970
 971         mutex_lock(&info->lock);
 972         ret = __run_threaded_test(info);
 973         mutex_unlock(&info->lock);
 974         return ret;
 975 }
 976 #endif
 977
 978 static void __stop_threaded_test(struct dmatest_info *info)
 979 {
 980         struct dmatest_chan *dtc, *_dtc;
 981         struct dma_chan *chan;
 982
 983         list_for_each_entry_safe(dtc, _dtc, &info->channels, node) {
 984                 list_del(&dtc->node);
 985                 chan = dtc->chan;
 986                 dmatest_cleanup_channel(dtc);
 987                 pr_debug("dmatest: dropped channel %s\n", dma_chan_name(chan));
 988                 dma_release_channel(chan);
 989         }
 990
 991         info->nr_channels = 0;
 992 }
 993
 994 static void stop_threaded_test(struct dmatest_info *info)
 995 {
 996         mutex_lock(&info->lock);
 997         __stop_threaded_test(info);
 998         mutex_unlock(&info->lock);
 999 }
1000
1001 static int __restart_threaded_test(struct dmatest_info *info, bool run)
1002 {
1003         struct dmatest_params *params = &info->params;
1004
1005         /* Stop any running test first */
1006         __stop_threaded_test(info);
1007
1008         if (run == false)
1009                 return 0;
1010
1011         /* Clear results from previous run */
1012         result_free(info, NULL);
1013
1014         /* Copy test parameters */
1015         params->buf_size = test_buf_size;
1016         strlcpy(params->channel, strim(test_channel), sizeof(params->channel));
1017         strlcpy(params->device, strim(test_device), sizeof(params->device));
1018         params->threads_per_chan = threads_per_chan;
1019         params->max_channels = max_channels;
1020         params->iterations = iterations;
1021         params->xor_sources = xor_sources;
1022         params->pq_sources = pq_sources;
1023         params->timeout = timeout;
1024
1025         /* Run test with new parameters */
1026         return __run_threaded_test(info);
1027 }
1028
1029 static bool __is_threaded_test_run(struct dmatest_info *info)
1030 {
1031         struct dmatest_chan *dtc;
1032
1033         list_for_each_entry(dtc, &info->channels, node) {
1034                 struct dmatest_thread *thread;
1035
1036                 list_for_each_entry(thread, &dtc->threads, node) {
1037                         if (!thread->done)
1038                                 return true;
1039                 }
1040         }
1041
1042         return false;
1043 }
1044
1045 static ssize_t dtf_read_run(struct file *file, char __user *user_buf,
1046                 size_t count, loff_t *ppos)
1047 {
1048         struct dmatest_info *info = file->private_data;
1049         char buf[3];
1050
1051         mutex_lock(&info->lock);
1052
1053         if (__is_threaded_test_run(info)) {
1054                 buf[0] = 'Y';
1055         } else {
1056                 __stop_threaded_test(info);
1057                 buf[0] = 'N';
1058         }
1059
1060         mutex_unlock(&info->lock);
1061         buf[1] = '\n';
1062         buf[2] = 0x00;
1063         return simple_read_from_buffer(user_buf, count, ppos, buf, 2);
1064 }
1065
1066 static ssize_t dtf_write_run(struct file *file, const char __user *user_buf,
1067                 size_t count, loff_t *ppos)
1068 {
1069         struct dmatest_info *info = file->private_data;
1070         char buf[16];
1071         bool bv;
1072         int ret = 0;
1073
1074         if (copy_from_user(buf, user_buf, min(count, (sizeof(buf) - 1))))
1075                 return -EFAULT;
1076
1077         if (strtobool(buf, &bv) == 0) {
1078                 mutex_lock(&info->lock);
1079
1080                 if (__is_threaded_test_run(info))
1081                         ret = -EBUSY;
1082                 else
1083                         ret = __restart_threaded_test(info, bv);
1084
1085                 mutex_unlock(&info->lock);
1086         }
1087
1088         return ret ? ret : count;
1089 }
1090
1091 static const struct file_operations dtf_run_fops = {
1092         .read   = dtf_read_run,
1093         .write  = dtf_write_run,
1094         .open   = simple_open,
1095         .llseek = default_llseek,
1096 };
1097
1098 static int dtf_results_show(struct seq_file *sf, void *data)
1099 {
1100         struct dmatest_info *info = sf->private;
1101         struct dmatest_result *result;
1102         struct dmatest_thread_result *tr;
1103         unsigned int i;
1104
1105         mutex_lock(&info->results_lock);
1106         list_for_each_entry(result, &info->results, node) {
1107                 list_for_each_entry(tr, &result->results, node) {
1108                         seq_printf(sf, "%s\n",
1109                                 thread_result_get(result->name, tr));
1110                         if (tr->type == DMATEST_ET_VERIFY_BUF) {
1111                                 for (i = 0; i < tr->vr->error_count; i++) {
1112                                         seq_printf(sf, "\t%s\n",
1113                                                 verify_result_get_one(tr->vr, i));
1114                                 }
1115                         }
1116                 }
1117         }
1118
1119         mutex_unlock(&info->results_lock);
1120         return 0;
1121 }
1122
1123 static int dtf_results_open(struct inode *inode, struct file *file)
1124 {
1125         return single_open(file, dtf_results_show, inode->i_private);
1126 }
1127
1128 static const struct file_operations dtf_results_fops = {
1129         .open           = dtf_results_open,
1130         .read           = seq_read,
1131         .llseek         = seq_lseek,
1132         .release        = single_release,
1133 };
1134
1135 static int dmatest_register_dbgfs(struct dmatest_info *info)
1136 {
1137         struct dentry *d;
1138
1139         d = debugfs_create_dir("dmatest", NULL);
1140         if (IS_ERR(d))
1141                 return PTR_ERR(d);
1142         if (!d)
1143                 goto err_root;
1144
1145         info->root = d;
1146
1147         /* Run or stop threaded test */
1148         debugfs_create_file("run", S_IWUSR | S_IRUGO, info->root, info,
1149                             &dtf_run_fops);
1150
1151         /* Results of test in progress */
1152         debugfs_create_file("results", S_IRUGO, info->root, info,
1153                             &dtf_results_fops);
1154
1155         return 0;
1156
1157 err_root:
1158         pr_err("dmatest: Failed to initialize debugfs\n");
1159         return -ENOMEM;
1160 }
1161
1162 static int __init dmatest_init(void)
1163 {
1164         struct dmatest_info *info = &test_info;
1165         int ret;
1166
1167         memset(info, 0, sizeof(*info));
1168
1169         mutex_init(&info->lock);
1170         INIT_LIST_HEAD(&info->channels);
1171
1172         mutex_init(&info->results_lock);
1173         INIT_LIST_HEAD(&info->results);
1174
1175         ret = dmatest_register_dbgfs(info);
1176         if (ret)
1177                 return ret;
1178
1179 #ifdef MODULE
1180         return 0;
1181 #else
1182         return run_threaded_test(info);
1183 #endif
1184 }
1185 /* when compiled-in wait for drivers to load first */
1186 late_initcall(dmatest_init);
1187
1188 static void __exit dmatest_exit(void)
1189 {
1190         struct dmatest_info *info = &test_info;
1191
1192         debugfs_remove_recursive(info->root);
1193         stop_threaded_test(info);
1194         result_free(info, NULL);
1195 }
1196 module_exit(dmatest_exit);
1197
1198 MODULE_AUTHOR("Haavard Skinnemoen (Atmel)");
1199 MODULE_LICENSE("GPL v2");