drivers/hwtracing/coresight/coresight-tmc-etf.c

   1 /*
   2  * Copyright(C) 2016 Linaro Limited. All rights reserved.
   3  * Author: Mathieu Poirier <mathieu.poirier@linaro.org>
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 as published by
   7  * the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  *
  14  * You should have received a copy of the GNU General Public License along with
  15  * this program.  If not, see <http://www.gnu.org/licenses/>.
  16  */
  17
  18 #include <linux/circ_buf.h>
  19 #include <linux/coresight.h>
  20 #include <linux/perf_event.h>
  21 #include <linux/slab.h>
  22 #include "coresight-priv.h"
  23 #include "coresight-tmc.h"
  24
  25 static void tmc_etb_enable_hw(struct tmc_drvdata *drvdata)
  26 {
  27         CS_UNLOCK(drvdata->base);
  28
  29         /* Wait for TMCSReady bit to be set */
  30         tmc_wait_for_tmcready(drvdata);
  31
  32         writel_relaxed(TMC_MODE_CIRCULAR_BUFFER, drvdata->base + TMC_MODE);
  33         writel_relaxed(TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI |
  34                        TMC_FFCR_FON_FLIN | TMC_FFCR_FON_TRIG_EVT |
  35                        TMC_FFCR_TRIGON_TRIGIN,
  36                        drvdata->base + TMC_FFCR);
  37
  38         writel_relaxed(drvdata->trigger_cntr, drvdata->base + TMC_TRG);
  39         tmc_enable_hw(drvdata);
  40
  41         CS_LOCK(drvdata->base);
  42 }
  43
  44 static void tmc_etb_dump_hw(struct tmc_drvdata *drvdata)
  45 {
  46         char *bufp;
  47         u32 read_data;
  48         int i;
  49
  50         bufp = drvdata->buf;
  51         drvdata->len = 0;
  52         while (1) {
  53                 for (i = 0; i < drvdata->memwidth; i++) {
  54                         read_data = readl_relaxed(drvdata->base + TMC_RRD);
  55                         if (read_data == 0xFFFFFFFF)
  56                                 return;
  57                         memcpy(bufp, &read_data, 4);
  58                         bufp += 4;
  59                         drvdata->len += 4;
  60                 }
  61         }
  62 }
  63
  64 static void tmc_etb_disable_hw(struct tmc_drvdata *drvdata)
  65 {
  66         CS_UNLOCK(drvdata->base);
  67
  68         tmc_flush_and_stop(drvdata);
  69         /*
  70          * When operating in sysFS mode the content of the buffer needs to be
  71          * read before the TMC is disabled.
  72          */
  73         if (drvdata->mode == CS_MODE_SYSFS)
  74                 tmc_etb_dump_hw(drvdata);
  75         tmc_disable_hw(drvdata);
  76
  77         CS_LOCK(drvdata->base);
  78 }
  79
  80 static void tmc_etf_enable_hw(struct tmc_drvdata *drvdata)
  81 {
  82         CS_UNLOCK(drvdata->base);
  83
  84         /* Wait for TMCSReady bit to be set */
  85         tmc_wait_for_tmcready(drvdata);
  86
  87         writel_relaxed(TMC_MODE_HARDWARE_FIFO, drvdata->base + TMC_MODE);
  88         writel_relaxed(TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI,
  89                        drvdata->base + TMC_FFCR);
  90         writel_relaxed(0x0, drvdata->base + TMC_BUFWM);
  91         tmc_enable_hw(drvdata);
  92
  93         CS_LOCK(drvdata->base);
  94 }
  95
  96 static void tmc_etf_disable_hw(struct tmc_drvdata *drvdata)
  97 {
  98         CS_UNLOCK(drvdata->base);
  99
 100         tmc_flush_and_stop(drvdata);
 101         tmc_disable_hw(drvdata);
 102
 103         CS_LOCK(drvdata->base);
 104 }
 105
 106 static int tmc_enable_etf_sink_sysfs(struct coresight_device *csdev)
 107 {
 108         int ret = 0;
 109         bool used = false;
 110         char *buf = NULL;
 111         unsigned long flags;
 112         struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 113
 114         /*
 115          * If we don't have a buffer release the lock and allocate memory.
 116          * Otherwise keep the lock and move along.
 117          */
 118         spin_lock_irqsave(&drvdata->spinlock, flags);
 119         if (!drvdata->buf) {
 120                 spin_unlock_irqrestore(&drvdata->spinlock, flags);
 121
 122                 /* Allocating the memory here while outside of the spinlock */
 123                 buf = kzalloc(drvdata->size, GFP_KERNEL);
 124                 if (!buf)
 125                         return -ENOMEM;
 126
 127                 /* Let's try again */
 128                 spin_lock_irqsave(&drvdata->spinlock, flags);
 129         }
 130
 131         if (drvdata->reading) {
 132                 ret = -EBUSY;
 133                 goto out;
 134         }
 135
 136         /*
 137          * In sysFS mode we can have multiple writers per sink.  Since this
 138          * sink is already enabled no memory is needed and the HW need not be
 139          * touched.
 140          */
 141         if (drvdata->mode == CS_MODE_SYSFS)
 142                 goto out;
 143
 144         /*
 145          * If drvdata::buf isn't NULL, memory was allocated for a previous
 146          * trace run but wasn't read.  If so simply zero-out the memory.
 147          * Otherwise use the memory allocated above.
 148          *
 149          * The memory is freed when users read the buffer using the
 150          * /dev/xyz.{etf|etb} interface.  See tmc_read_unprepare_etf() for
 151          * details.
 152          */
 153         if (drvdata->buf) {
 154                 memset(drvdata->buf, 0, drvdata->size);
 155         } else {
 156                 used = true;
 157                 drvdata->buf = buf;
 158         }
 159
 160         drvdata->mode = CS_MODE_SYSFS;
 161         tmc_etb_enable_hw(drvdata);
 162 out:
 163         spin_unlock_irqrestore(&drvdata->spinlock, flags);
 164
 165         /* Free memory outside the spinlock if need be */
 166         if (!used)
 167                 kfree(buf);
 168
 169         if (!ret)
 170                 dev_info(drvdata->dev, "TMC-ETB/ETF enabled\n");
 171
 172         return ret;
 173 }
 174
 175 static int tmc_enable_etf_sink_perf(struct coresight_device *csdev)
 176 {
 177         int ret = 0;
 178         unsigned long flags;
 179         struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 180
 181         spin_lock_irqsave(&drvdata->spinlock, flags);
 182         if (drvdata->reading) {
 183                 ret = -EINVAL;
 184                 goto out;
 185         }
 186
 187         /*
 188          * In Perf mode there can be only one writer per sink.  There
 189          * is also no need to continue if the ETB/ETR is already operated
 190          * from sysFS.
 191          */
 192         if (drvdata->mode != CS_MODE_DISABLED) {
 193                 ret = -EINVAL;
 194                 goto out;
 195         }
 196
 197         drvdata->mode = CS_MODE_PERF;
 198         tmc_etb_enable_hw(drvdata);
 199 out:
 200         spin_unlock_irqrestore(&drvdata->spinlock, flags);
 201
 202         return ret;
 203 }
 204
 205 static int tmc_enable_etf_sink(struct coresight_device *csdev, u32 mode)
 206 {
 207         switch (mode) {
 208         case CS_MODE_SYSFS:
 209                 return tmc_enable_etf_sink_sysfs(csdev);
 210         case CS_MODE_PERF:
 211                 return tmc_enable_etf_sink_perf(csdev);
 212         }
 213
 214         /* We shouldn't be here */
 215         return -EINVAL;
 216 }
 217
 218 static void tmc_disable_etf_sink(struct coresight_device *csdev)
 219 {
 220         unsigned long flags;
 221         struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 222
 223         spin_lock_irqsave(&drvdata->spinlock, flags);
 224         if (drvdata->reading) {
 225                 spin_unlock_irqrestore(&drvdata->spinlock, flags);
 226                 return;
 227         }
 228
 229         /* Disable the TMC only if it needs to */
 230         if (drvdata->mode != CS_MODE_DISABLED) {
 231                 tmc_etb_disable_hw(drvdata);
 232                 drvdata->mode = CS_MODE_DISABLED;
 233         }
 234
 235         spin_unlock_irqrestore(&drvdata->spinlock, flags);
 236
 237         dev_info(drvdata->dev, "TMC-ETB/ETF disabled\n");
 238 }
 239
 240 static int tmc_enable_etf_link(struct coresight_device *csdev,
 241                                int inport, int outport)
 242 {
 243         unsigned long flags;
 244         struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 245
 246         spin_lock_irqsave(&drvdata->spinlock, flags);
 247         if (drvdata->reading) {
 248                 spin_unlock_irqrestore(&drvdata->spinlock, flags);
 249                 return -EBUSY;
 250         }
 251
 252         tmc_etf_enable_hw(drvdata);
 253         drvdata->mode = CS_MODE_SYSFS;
 254         spin_unlock_irqrestore(&drvdata->spinlock, flags);
 255
 256         dev_info(drvdata->dev, "TMC-ETF enabled\n");
 257         return 0;
 258 }
 259
 260 static void tmc_disable_etf_link(struct coresight_device *csdev,
 261                                  int inport, int outport)
 262 {
 263         unsigned long flags;
 264         struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 265
 266         spin_lock_irqsave(&drvdata->spinlock, flags);
 267         if (drvdata->reading) {
 268                 spin_unlock_irqrestore(&drvdata->spinlock, flags);
 269                 return;
 270         }
 271
 272         tmc_etf_disable_hw(drvdata);
 273         drvdata->mode = CS_MODE_DISABLED;
 274         spin_unlock_irqrestore(&drvdata->spinlock, flags);
 275
 276         dev_info(drvdata->dev, "TMC disabled\n");
 277 }
 278
 279 static void *tmc_alloc_etf_buffer(struct coresight_device *csdev, int cpu,
 280                                   void **pages, int nr_pages, bool overwrite)
 281 {
 282         int node;
 283         struct cs_buffers *buf;
 284
 285         if (cpu == -1)
 286                 cpu = smp_processor_id();
 287         node = cpu_to_node(cpu);
 288
 289         /* Allocate memory structure for interaction with Perf */
 290         buf = kzalloc_node(sizeof(struct cs_buffers), GFP_KERNEL, node);
 291         if (!buf)
 292                 return NULL;
 293
 294         buf->snapshot = overwrite;
 295         buf->nr_pages = nr_pages;
 296         buf->data_pages = pages;
 297
 298         return buf;
 299 }
 300
 301 static void tmc_free_etf_buffer(void *config)
 302 {
 303         struct cs_buffers *buf = config;
 304
 305         kfree(buf);
 306 }
 307
 308 static int tmc_set_etf_buffer(struct coresight_device *csdev,
 309                               struct perf_output_handle *handle,
 310                               void *sink_config)
 311 {
 312         int ret = 0;
 313         unsigned long head;
 314         struct cs_buffers *buf = sink_config;
 315
 316         /* wrap head around to the amount of space we have */
 317         head = handle->head & ((buf->nr_pages << PAGE_SHIFT) - 1);
 318
 319         /* find the page to write to */
 320         buf->cur = head / PAGE_SIZE;
 321
 322         /* and offset within that page */
 323         buf->offset = head % PAGE_SIZE;
 324
 325         local_set(&buf->data_size, 0);
 326
 327         return ret;
 328 }
 329
 330 static unsigned long tmc_reset_etf_buffer(struct coresight_device *csdev,
 331                                           struct perf_output_handle *handle,
 332                                           void *sink_config)
 333 {
 334         long size = 0;
 335         struct cs_buffers *buf = sink_config;
 336
 337         if (buf) {
 338                 /*
 339                  * In snapshot mode ->data_size holds the new address of the
 340                  * ring buffer's head.  The size itself is the whole address
 341                  * range since we want the latest information.
 342                  */
 343                 if (buf->snapshot)
 344                         handle->head = local_xchg(&buf->data_size,
 345                                                   buf->nr_pages << PAGE_SHIFT);
 346                 /*
 347                  * Tell the tracer PMU how much we got in this run and if
 348                  * something went wrong along the way.  Nobody else can use
 349                  * this cs_buffers instance until we are done.  As such
 350                  * resetting parameters here and squaring off with the ring
 351                  * buffer API in the tracer PMU is fine.
 352                  */
 353                 size = local_xchg(&buf->data_size, 0);
 354         }
 355
 356         return size;
 357 }
 358
 359 static void tmc_update_etf_buffer(struct coresight_device *csdev,
 360                                   struct perf_output_handle *handle,
 361                                   void *sink_config)
 362 {
 363         int i, cur;
 364         u32 *buf_ptr;
 365         u32 read_ptr, write_ptr;
 366         u32 status, to_read;
 367         unsigned long offset;
 368         struct cs_buffers *buf = sink_config;
 369         struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 370
 371         if (!buf)
 372                 return;
 373
 374         /* This shouldn't happen */
 375         if (WARN_ON_ONCE(drvdata->mode != CS_MODE_PERF))
 376                 return;
 377
 378         CS_UNLOCK(drvdata->base);
 379
 380         tmc_flush_and_stop(drvdata);
 381
 382         read_ptr = readl_relaxed(drvdata->base + TMC_RRP);
 383         write_ptr = readl_relaxed(drvdata->base + TMC_RWP);
 384
 385         /*
 386          * Get a hold of the status register and see if a wrap around
 387          * has occurred.  If so adjust things accordingly.
 388          */
 389         status = readl_relaxed(drvdata->base + TMC_STS);
 390         if (status & TMC_STS_FULL) {
 391                 perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);
 392                 to_read = drvdata->size;
 393         } else {
 394                 to_read = CIRC_CNT(write_ptr, read_ptr, drvdata->size);
 395         }
 396
 397         /*
 398          * The TMC RAM buffer may be bigger than the space available in the
 399          * perf ring buffer (handle->size).  If so advance the RRP so that we
 400          * get the latest trace data.
 401          */
 402         if (to_read > handle->size) {
 403                 u32 mask = 0;
 404
 405                 /*
 406                  * The value written to RRP must be byte-address aligned to
 407                  * the width of the trace memory databus _and_ to a frame
 408                  * boundary (16 byte), whichever is the biggest. For example,
 409                  * for 32-bit, 64-bit and 128-bit wide trace memory, the four
 410                  * LSBs must be 0s. For 256-bit wide trace memory, the five
 411                  * LSBs must be 0s.
 412                  */
 413                 switch (drvdata->memwidth) {
 414                 case TMC_MEM_INTF_WIDTH_32BITS:
 415                 case TMC_MEM_INTF_WIDTH_64BITS:
 416                 case TMC_MEM_INTF_WIDTH_128BITS:
 417                         mask = GENMASK(31, 5);
 418                         break;
 419                 case TMC_MEM_INTF_WIDTH_256BITS:
 420                         mask = GENMASK(31, 6);
 421                         break;
 422                 }
 423
 424                 /*
 425                  * Make sure the new size is aligned in accordance with the
 426                  * requirement explained above.
 427                  */
 428                 to_read = handle->size & mask;
 429                 /* Move the RAM read pointer up */
 430                 read_ptr = (write_ptr + drvdata->size) - to_read;
 431                 /* Make sure we are still within our limits */
 432                 if (read_ptr > (drvdata->size - 1))
 433                         read_ptr -= drvdata->size;
 434                 /* Tell the HW */
 435                 writel_relaxed(read_ptr, drvdata->base + TMC_RRP);
 436                 perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);
 437         }
 438
 439         cur = buf->cur;
 440         offset = buf->offset;
 441
 442         /* for every byte to read */
 443         for (i = 0; i < to_read; i += 4) {
 444                 buf_ptr = buf->data_pages[cur] + offset;
 445                 *buf_ptr = readl_relaxed(drvdata->base + TMC_RRD);
 446
 447                 offset += 4;
 448                 if (offset >= PAGE_SIZE) {
 449                         offset = 0;
 450                         cur++;
 451                         /* wrap around at the end of the buffer */
 452                         cur &= buf->nr_pages - 1;
 453                 }
 454         }
 455
 456         /*
 457          * In snapshot mode all we have to do is communicate to
 458          * perf_aux_output_end() the address of the current head.  In full
 459          * trace mode the same function expects a size to move rb->aux_head
 460          * forward.
 461          */
 462         if (buf->snapshot)
 463                 local_set(&buf->data_size, (cur * PAGE_SIZE) + offset);
 464         else
 465                 local_add(to_read, &buf->data_size);
 466
 467         CS_LOCK(drvdata->base);
 468 }
 469
 470 static const struct coresight_ops_sink tmc_etf_sink_ops = {
 471         .enable         = tmc_enable_etf_sink,
 472         .disable        = tmc_disable_etf_sink,
 473         .alloc_buffer   = tmc_alloc_etf_buffer,
 474         .free_buffer    = tmc_free_etf_buffer,
 475         .set_buffer     = tmc_set_etf_buffer,
 476         .reset_buffer   = tmc_reset_etf_buffer,
 477         .update_buffer  = tmc_update_etf_buffer,
 478 };
 479
 480 static const struct coresight_ops_link tmc_etf_link_ops = {
 481         .enable         = tmc_enable_etf_link,
 482         .disable        = tmc_disable_etf_link,
 483 };
 484
 485 const struct coresight_ops tmc_etb_cs_ops = {
 486         .sink_ops       = &tmc_etf_sink_ops,
 487 };
 488
 489 const struct coresight_ops tmc_etf_cs_ops = {
 490         .sink_ops       = &tmc_etf_sink_ops,
 491         .link_ops       = &tmc_etf_link_ops,
 492 };
 493
 494 int tmc_read_prepare_etb(struct tmc_drvdata *drvdata)
 495 {
 496         enum tmc_mode mode;
 497         int ret = 0;
 498         unsigned long flags;
 499
 500         /* config types are set a boot time and never change */
 501         if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETB &&
 502                          drvdata->config_type != TMC_CONFIG_TYPE_ETF))
 503                 return -EINVAL;
 504
 505         spin_lock_irqsave(&drvdata->spinlock, flags);
 506
 507         if (drvdata->reading) {
 508                 ret = -EBUSY;
 509                 goto out;
 510         }
 511
 512         /* There is no point in reading a TMC in HW FIFO mode */
 513         mode = readl_relaxed(drvdata->base + TMC_MODE);
 514         if (mode != TMC_MODE_CIRCULAR_BUFFER) {
 515                 ret = -EINVAL;
 516                 goto out;
 517         }
 518
 519         /* Don't interfere if operated from Perf */
 520         if (drvdata->mode == CS_MODE_PERF) {
 521                 ret = -EINVAL;
 522                 goto out;
 523         }
 524
 525         /* If drvdata::buf is NULL the trace data has been read already */
 526         if (drvdata->buf == NULL) {
 527                 ret = -EINVAL;
 528                 goto out;
 529         }
 530
 531         /* Disable the TMC if need be */
 532         if (drvdata->mode == CS_MODE_SYSFS)
 533                 tmc_etb_disable_hw(drvdata);
 534
 535         drvdata->reading = true;
 536 out:
 537         spin_unlock_irqrestore(&drvdata->spinlock, flags);
 538
 539         return ret;
 540 }
 541
 542 int tmc_read_unprepare_etb(struct tmc_drvdata *drvdata)
 543 {
 544         char *buf = NULL;
 545         enum tmc_mode mode;
 546         unsigned long flags;
 547
 548         /* config types are set a boot time and never change */
 549         if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETB &&
 550                          drvdata->config_type != TMC_CONFIG_TYPE_ETF))
 551                 return -EINVAL;
 552
 553         spin_lock_irqsave(&drvdata->spinlock, flags);
 554
 555         /* There is no point in reading a TMC in HW FIFO mode */
 556         mode = readl_relaxed(drvdata->base + TMC_MODE);
 557         if (mode != TMC_MODE_CIRCULAR_BUFFER) {
 558                 spin_unlock_irqrestore(&drvdata->spinlock, flags);
 559                 return -EINVAL;
 560         }
 561
 562         /* Re-enable the TMC if need be */
 563         if (drvdata->mode == CS_MODE_SYSFS) {
 564                 /*
 565                  * The trace run will continue with the same allocated trace
 566                  * buffer. As such zero-out the buffer so that we don't end
 567                  * up with stale data.
 568                  *
 569                  * Since the tracer is still enabled drvdata::buf
 570                  * can't be NULL.
 571                  */
 572                 memset(drvdata->buf, 0, drvdata->size);
 573                 tmc_etb_enable_hw(drvdata);
 574         } else {
 575                 /*
 576                  * The ETB/ETF is not tracing and the buffer was just read.
 577                  * As such prepare to free the trace buffer.
 578                  */
 579                 buf = drvdata->buf;
 580                 drvdata->buf = NULL;
 581         }
 582
 583         drvdata->reading = false;
 584         spin_unlock_irqrestore(&drvdata->spinlock, flags);
 585
 586         /*
 587          * Free allocated memory outside of the spinlock.  There is no need
 588          * to assert the validity of 'buf' since calling kfree(NULL) is safe.
 589          */
 590         kfree(buf);
 591
 592         return 0;
 593 }