arch/powerpc/kernel/perf_event_fsl_emb.c

   1 /*
   2  * Performance event support - Freescale Embedded Performance Monitor
   3  *
   4  * Copyright 2008-2009 Paul Mackerras, IBM Corporation.
   5  * Copyright 2010 Freescale Semiconductor, Inc.
   6  *
   7  * This program is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU General Public License
   9  * as published by the Free Software Foundation; either version
  10  * 2 of the License, or (at your option) any later version.
  11  */
  12 #include <linux/kernel.h>
  13 #include <linux/sched.h>
  14 #include <linux/perf_event.h>
  15 #include <linux/percpu.h>
  16 #include <linux/hardirq.h>
  17 #include <asm/reg_fsl_emb.h>
  18 #include <asm/pmc.h>
  19 #include <asm/machdep.h>
  20 #include <asm/firmware.h>
  21 #include <asm/ptrace.h>
  22
  23 struct cpu_hw_events {
  24         int n_events;
  25         int disabled;
  26         u8  pmcs_enabled;
  27         struct perf_event *event[MAX_HWEVENTS];
  28 };
  29 static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events);
  30
  31 static struct fsl_emb_pmu *ppmu;
  32
  33 /* Number of perf_events counting hardware events */
  34 static atomic_t num_events;
  35 /* Used to avoid races in calling reserve/release_pmc_hardware */
  36 static DEFINE_MUTEX(pmc_reserve_mutex);
  37
  38 /*
  39  * If interrupts were soft-disabled when a PMU interrupt occurs, treat
  40  * it as an NMI.
  41  */
  42 static inline int perf_intr_is_nmi(struct pt_regs *regs)
  43 {
  44 #ifdef __powerpc64__
  45         return !regs->softe;
  46 #else
  47         return 0;
  48 #endif
  49 }
  50
  51 static void perf_event_interrupt(struct pt_regs *regs);
  52
  53 /*
  54  * Read one performance monitor counter (PMC).
  55  */
  56 static unsigned long read_pmc(int idx)
  57 {
  58         unsigned long val;
  59
  60         switch (idx) {
  61         case 0:
  62                 val = mfpmr(PMRN_PMC0);
  63                 break;
  64         case 1:
  65                 val = mfpmr(PMRN_PMC1);
  66                 break;
  67         case 2:
  68                 val = mfpmr(PMRN_PMC2);
  69                 break;
  70         case 3:
  71                 val = mfpmr(PMRN_PMC3);
  72                 break;
  73         default:
  74                 printk(KERN_ERR "oops trying to read PMC%d\n", idx);
  75                 val = 0;
  76         }
  77         return val;
  78 }
  79
  80 /*
  81  * Write one PMC.
  82  */
  83 static void write_pmc(int idx, unsigned long val)
  84 {
  85         switch (idx) {
  86         case 0:
  87                 mtpmr(PMRN_PMC0, val);
  88                 break;
  89         case 1:
  90                 mtpmr(PMRN_PMC1, val);
  91                 break;
  92         case 2:
  93                 mtpmr(PMRN_PMC2, val);
  94                 break;
  95         case 3:
  96                 mtpmr(PMRN_PMC3, val);
  97                 break;
  98         default:
  99                 printk(KERN_ERR "oops trying to write PMC%d\n", idx);
 100         }
 101
 102         isync();
 103 }
 104
 105 /*
 106  * Write one local control A register
 107  */
 108 static void write_pmlca(int idx, unsigned long val)
 109 {
 110         switch (idx) {
 111         case 0:
 112                 mtpmr(PMRN_PMLCA0, val);
 113                 break;
 114         case 1:
 115                 mtpmr(PMRN_PMLCA1, val);
 116                 break;
 117         case 2:
 118                 mtpmr(PMRN_PMLCA2, val);
 119                 break;
 120         case 3:
 121                 mtpmr(PMRN_PMLCA3, val);
 122                 break;
 123         default:
 124                 printk(KERN_ERR "oops trying to write PMLCA%d\n", idx);
 125         }
 126
 127         isync();
 128 }
 129
 130 /*
 131  * Write one local control B register
 132  */
 133 static void write_pmlcb(int idx, unsigned long val)
 134 {
 135         switch (idx) {
 136         case 0:
 137                 mtpmr(PMRN_PMLCB0, val);
 138                 break;
 139         case 1:
 140                 mtpmr(PMRN_PMLCB1, val);
 141                 break;
 142         case 2:
 143                 mtpmr(PMRN_PMLCB2, val);
 144                 break;
 145         case 3:
 146                 mtpmr(PMRN_PMLCB3, val);
 147                 break;
 148         default:
 149                 printk(KERN_ERR "oops trying to write PMLCB%d\n", idx);
 150         }
 151
 152         isync();
 153 }
 154
 155 static void fsl_emb_pmu_read(struct perf_event *event)
 156 {
 157         s64 val, delta, prev;
 158
 159         /*
 160          * Performance monitor interrupts come even when interrupts
 161          * are soft-disabled, as long as interrupts are hard-enabled.
 162          * Therefore we treat them like NMIs.
 163          */
 164         do {
 165                 prev = local64_read(&event->hw.prev_count);
 166                 barrier();
 167                 val = read_pmc(event->hw.idx);
 168         } while (local64_cmpxchg(&event->hw.prev_count, prev, val) != prev);
 169
 170         /* The counters are only 32 bits wide */
 171         delta = (val - prev) & 0xfffffffful;
 172         local64_add(delta, &event->count);
 173         local64_sub(delta, &event->hw.period_left);
 174 }
 175
 176 /*
 177  * Disable all events to prevent PMU interrupts and to allow
 178  * events to be added or removed.
 179  */
 180 static void fsl_emb_pmu_pmu_disable(struct pmu *pmu)
 181 {
 182         struct cpu_hw_events *cpuhw;
 183         unsigned long flags;
 184
 185         local_irq_save(flags);
 186         cpuhw = &__get_cpu_var(cpu_hw_events);
 187
 188         if (!cpuhw->disabled) {
 189                 cpuhw->disabled = 1;
 190
 191                 /*
 192                  * Check if we ever enabled the PMU on this cpu.
 193                  */
 194                 if (!cpuhw->pmcs_enabled) {
 195                         ppc_enable_pmcs();
 196                         cpuhw->pmcs_enabled = 1;
 197                 }
 198
 199                 if (atomic_read(&num_events)) {
 200                         /*
 201                          * Set the 'freeze all counters' bit, and disable
 202                          * interrupts.  The barrier is to make sure the
 203                          * mtpmr has been executed and the PMU has frozen
 204                          * the events before we return.
 205                          */
 206
 207                         mtpmr(PMRN_PMGC0, PMGC0_FAC);
 208                         isync();
 209                 }
 210         }
 211         local_irq_restore(flags);
 212 }
 213
 214 /*
 215  * Re-enable all events if disable == 0.
 216  * If we were previously disabled and events were added, then
 217  * put the new config on the PMU.
 218  */
 219 static void fsl_emb_pmu_pmu_enable(struct pmu *pmu)
 220 {
 221         struct cpu_hw_events *cpuhw;
 222         unsigned long flags;
 223
 224         local_irq_save(flags);
 225         cpuhw = &__get_cpu_var(cpu_hw_events);
 226         if (!cpuhw->disabled)
 227                 goto out;
 228
 229         cpuhw->disabled = 0;
 230         ppc_set_pmu_inuse(cpuhw->n_events != 0);
 231
 232         if (cpuhw->n_events > 0) {
 233                 mtpmr(PMRN_PMGC0, PMGC0_PMIE | PMGC0_FCECE);
 234                 isync();
 235         }
 236
 237  out:
 238         local_irq_restore(flags);
 239 }
 240
 241 static int collect_events(struct perf_event *group, int max_count,
 242                           struct perf_event *ctrs[])
 243 {
 244         int n = 0;
 245         struct perf_event *event;
 246
 247         if (!is_software_event(group)) {
 248                 if (n >= max_count)
 249                         return -1;
 250                 ctrs[n] = group;
 251                 n++;
 252         }
 253         list_for_each_entry(event, &group->sibling_list, group_entry) {
 254                 if (!is_software_event(event) &&
 255                     event->state != PERF_EVENT_STATE_OFF) {
 256                         if (n >= max_count)
 257                                 return -1;
 258                         ctrs[n] = event;
 259                         n++;
 260                 }
 261         }
 262         return n;
 263 }
 264
 265 /* context locked on entry */
 266 static int fsl_emb_pmu_enable(struct perf_event *event)
 267 {
 268         struct cpu_hw_events *cpuhw;
 269         int ret = -EAGAIN;
 270         int num_counters = ppmu->n_counter;
 271         u64 val;
 272         int i;
 273
 274         perf_pmu_disable(event->pmu);
 275         cpuhw = &get_cpu_var(cpu_hw_events);
 276
 277         if (event->hw.config & FSL_EMB_EVENT_RESTRICTED)
 278                 num_counters = ppmu->n_restricted;
 279
 280         /*
 281          * Allocate counters from top-down, so that restricted-capable
 282          * counters are kept free as long as possible.
 283          */
 284         for (i = num_counters - 1; i >= 0; i--) {
 285                 if (cpuhw->event[i])
 286                         continue;
 287
 288                 break;
 289         }
 290
 291         if (i < 0)
 292                 goto out;
 293
 294         event->hw.idx = i;
 295         cpuhw->event[i] = event;
 296         ++cpuhw->n_events;
 297
 298         val = 0;
 299         if (event->hw.sample_period) {
 300                 s64 left = local64_read(&event->hw.period_left);
 301                 if (left < 0x80000000L)
 302                         val = 0x80000000L - left;
 303         }
 304         local64_set(&event->hw.prev_count, val);
 305         write_pmc(i, val);
 306         perf_event_update_userpage(event);
 307
 308         write_pmlcb(i, event->hw.config >> 32);
 309         write_pmlca(i, event->hw.config_base);
 310
 311         ret = 0;
 312  out:
 313         put_cpu_var(cpu_hw_events);
 314         perf_pmu_enable(event->pmu);
 315         return ret;
 316 }
 317
 318 /* context locked on entry */
 319 static void fsl_emb_pmu_disable(struct perf_event *event)
 320 {
 321         struct cpu_hw_events *cpuhw;
 322         int i = event->hw.idx;
 323
 324         perf_pmu_disable(event->pmu);
 325         if (i < 0)
 326                 goto out;
 327
 328         fsl_emb_pmu_read(event);
 329
 330         cpuhw = &get_cpu_var(cpu_hw_events);
 331
 332         WARN_ON(event != cpuhw->event[event->hw.idx]);
 333
 334         write_pmlca(i, 0);
 335         write_pmlcb(i, 0);
 336         write_pmc(i, 0);
 337
 338         cpuhw->event[i] = NULL;
 339         event->hw.idx = -1;
 340
 341         /*
 342          * TODO: if at least one restricted event exists, and we
 343          * just freed up a non-restricted-capable counter, and
 344          * there is a restricted-capable counter occupied by
 345          * a non-restricted event, migrate that event to the
 346          * vacated counter.
 347          */
 348
 349         cpuhw->n_events--;
 350
 351  out:
 352         perf_pmu_enable(event->pmu);
 353         put_cpu_var(cpu_hw_events);
 354 }
 355
 356 /*
 357  * Re-enable interrupts on a event after they were throttled
 358  * because they were coming too fast.
 359  *
 360  * Context is locked on entry, but perf is not disabled.
 361  */
 362 static void fsl_emb_pmu_unthrottle(struct perf_event *event)
 363 {
 364         s64 val, left;
 365         unsigned long flags;
 366
 367         if (event->hw.idx < 0 || !event->hw.sample_period)
 368                 return;
 369         local_irq_save(flags);
 370         perf_pmu_disable(event->pmu);
 371         fsl_emb_pmu_read(event);
 372         left = event->hw.sample_period;
 373         event->hw.last_period = left;
 374         val = 0;
 375         if (left < 0x80000000L)
 376                 val = 0x80000000L - left;
 377         write_pmc(event->hw.idx, val);
 378         local64_set(&event->hw.prev_count, val);
 379         local64_set(&event->hw.period_left, left);
 380         perf_event_update_userpage(event);
 381         perf_pmu_enable(event->pmu);
 382         local_irq_restore(flags);
 383 }
 384
 385 /*
 386  * Release the PMU if this is the last perf_event.
 387  */
 388 static void hw_perf_event_destroy(struct perf_event *event)
 389 {
 390         if (!atomic_add_unless(&num_events, -1, 1)) {
 391                 mutex_lock(&pmc_reserve_mutex);
 392                 if (atomic_dec_return(&num_events) == 0)
 393                         release_pmc_hardware();
 394                 mutex_unlock(&pmc_reserve_mutex);
 395         }
 396 }
 397
 398 /*
 399  * Translate a generic cache event_id config to a raw event_id code.
 400  */
 401 static int hw_perf_cache_event(u64 config, u64 *eventp)
 402 {
 403         unsigned long type, op, result;
 404         int ev;
 405
 406         if (!ppmu->cache_events)
 407                 return -EINVAL;
 408
 409         /* unpack config */
 410         type = config & 0xff;
 411         op = (config >> 8) & 0xff;
 412         result = (config >> 16) & 0xff;
 413
 414         if (type >= PERF_COUNT_HW_CACHE_MAX ||
 415             op >= PERF_COUNT_HW_CACHE_OP_MAX ||
 416             result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
 417                 return -EINVAL;
 418
 419         ev = (*ppmu->cache_events)[type][op][result];
 420         if (ev == 0)
 421                 return -EOPNOTSUPP;
 422         if (ev == -1)
 423                 return -EINVAL;
 424         *eventp = ev;
 425         return 0;
 426 }
 427
 428 static int fsl_emb_pmu_event_init(struct perf_event *event)
 429 {
 430         u64 ev;
 431         struct perf_event *events[MAX_HWEVENTS];
 432         int n;
 433         int err;
 434         int num_restricted;
 435         int i;
 436
 437         switch (event->attr.type) {
 438         case PERF_TYPE_HARDWARE:
 439                 ev = event->attr.config;
 440                 if (ev >= ppmu->n_generic || ppmu->generic_events[ev] == 0)
 441                         return -EOPNOTSUPP;
 442                 ev = ppmu->generic_events[ev];
 443                 break;
 444
 445         case PERF_TYPE_HW_CACHE:
 446                 err = hw_perf_cache_event(event->attr.config, &ev);
 447                 if (err)
 448                         return err;
 449                 break;
 450
 451         case PERF_TYPE_RAW:
 452                 ev = event->attr.config;
 453                 break;
 454
 455         default:
 456                 return -ENOENT;
 457         }
 458
 459         event->hw.config = ppmu->xlate_event(ev);
 460         if (!(event->hw.config & FSL_EMB_EVENT_VALID))
 461                 return -EINVAL;
 462
 463         /*
 464          * If this is in a group, check if it can go on with all the
 465          * other hardware events in the group.  We assume the event
 466          * hasn't been linked into its leader's sibling list at this point.
 467          */
 468         n = 0;
 469         if (event->group_leader != event) {
 470                 n = collect_events(event->group_leader,
 471                                    ppmu->n_counter - 1, events);
 472                 if (n < 0)
 473                         return -EINVAL;
 474         }
 475
 476         if (event->hw.config & FSL_EMB_EVENT_RESTRICTED) {
 477                 num_restricted = 0;
 478                 for (i = 0; i < n; i++) {
 479                         if (events[i]->hw.config & FSL_EMB_EVENT_RESTRICTED)
 480                                 num_restricted++;
 481                 }
 482
 483                 if (num_restricted >= ppmu->n_restricted)
 484                         return -EINVAL;
 485         }
 486
 487         event->hw.idx = -1;
 488
 489         event->hw.config_base = PMLCA_CE | PMLCA_FCM1 |
 490                                 (u32)((ev << 16) & PMLCA_EVENT_MASK);
 491
 492         if (event->attr.exclude_user)
 493                 event->hw.config_base |= PMLCA_FCU;
 494         if (event->attr.exclude_kernel)
 495                 event->hw.config_base |= PMLCA_FCS;
 496         if (event->attr.exclude_idle)
 497                 return -ENOTSUPP;
 498
 499         event->hw.last_period = event->hw.sample_period;
 500         local64_set(&event->hw.period_left, event->hw.last_period);
 501
 502         /*
 503          * See if we need to reserve the PMU.
 504          * If no events are currently in use, then we have to take a
 505          * mutex to ensure that we don't race with another task doing
 506          * reserve_pmc_hardware or release_pmc_hardware.
 507          */
 508         err = 0;
 509         if (!atomic_inc_not_zero(&num_events)) {
 510                 mutex_lock(&pmc_reserve_mutex);
 511                 if (atomic_read(&num_events) == 0 &&
 512                     reserve_pmc_hardware(perf_event_interrupt))
 513                         err = -EBUSY;
 514                 else
 515                         atomic_inc(&num_events);
 516                 mutex_unlock(&pmc_reserve_mutex);
 517
 518                 mtpmr(PMRN_PMGC0, PMGC0_FAC);
 519                 isync();
 520         }
 521         event->destroy = hw_perf_event_destroy;
 522
 523         return err;
 524 }
 525
 526 static struct pmu fsl_emb_pmu = {
 527         .pmu_enable     = fsl_emb_pmu_pmu_enable,
 528         .pmu_disable    = fsl_emb_pmu_pmu_disable,
 529         .event_init     = fsl_emb_pmu_event_init,
 530         .enable         = fsl_emb_pmu_enable,
 531         .disable        = fsl_emb_pmu_disable,
 532         .read           = fsl_emb_pmu_read,
 533         .unthrottle     = fsl_emb_pmu_unthrottle,
 534 };
 535
 536 /*
 537  * A counter has overflowed; update its count and record
 538  * things if requested.  Note that interrupts are hard-disabled
 539  * here so there is no possibility of being interrupted.
 540  */
 541 static void record_and_restart(struct perf_event *event, unsigned long val,
 542                                struct pt_regs *regs, int nmi)
 543 {
 544         u64 period = event->hw.sample_period;
 545         s64 prev, delta, left;
 546         int record = 0;
 547
 548         /* we don't have to worry about interrupts here */
 549         prev = local64_read(&event->hw.prev_count);
 550         delta = (val - prev) & 0xfffffffful;
 551         local64_add(delta, &event->count);
 552
 553         /*
 554          * See if the total period for this event has expired,
 555          * and update for the next period.
 556          */
 557         val = 0;
 558         left = local64_read(&event->hw.period_left) - delta;
 559         if (period) {
 560                 if (left <= 0) {
 561                         left += period;
 562                         if (left <= 0)
 563                                 left = period;
 564                         record = 1;
 565                 }
 566                 if (left < 0x80000000LL)
 567                         val = 0x80000000LL - left;
 568         }
 569
 570         /*
 571          * Finally record data if requested.
 572          */
 573         if (record) {
 574                 struct perf_sample_data data;
 575
 576                 perf_sample_data_init(&data, 0);
 577                 data.period = event->hw.last_period;
 578
 579                 if (perf_event_overflow(event, nmi, &data, regs)) {
 580                         /*
 581                          * Interrupts are coming too fast - throttle them
 582                          * by setting the event to 0, so it will be
 583                          * at least 2^30 cycles until the next interrupt
 584                          * (assuming each event counts at most 2 counts
 585                          * per cycle).
 586                          */
 587                         val = 0;
 588                         left = ~0ULL >> 1;
 589                 }
 590         }
 591
 592         write_pmc(event->hw.idx, val);
 593         local64_set(&event->hw.prev_count, val);
 594         local64_set(&event->hw.period_left, left);
 595         perf_event_update_userpage(event);
 596 }
 597
 598 static void perf_event_interrupt(struct pt_regs *regs)
 599 {
 600         int i;
 601         struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events);
 602         struct perf_event *event;
 603         unsigned long val;
 604         int found = 0;
 605         int nmi;
 606
 607         nmi = perf_intr_is_nmi(regs);
 608         if (nmi)
 609                 nmi_enter();
 610         else
 611                 irq_enter();
 612
 613         for (i = 0; i < ppmu->n_counter; ++i) {
 614                 event = cpuhw->event[i];
 615
 616                 val = read_pmc(i);
 617                 if ((int)val < 0) {
 618                         if (event) {
 619                                 /* event has overflowed */
 620                                 found = 1;
 621                                 record_and_restart(event, val, regs, nmi);
 622                         } else {
 623                                 /*
 624                                  * Disabled counter is negative,
 625                                  * reset it just in case.
 626                                  */
 627                                 write_pmc(i, 0);
 628                         }
 629                 }
 630         }
 631
 632         /* PMM will keep counters frozen until we return from the interrupt. */
 633         mtmsr(mfmsr() | MSR_PMM);
 634         mtpmr(PMRN_PMGC0, PMGC0_PMIE | PMGC0_FCECE);
 635         isync();
 636
 637         if (nmi)
 638                 nmi_exit();
 639         else
 640                 irq_exit();
 641 }
 642
 643 void hw_perf_event_setup(int cpu)
 644 {
 645         struct cpu_hw_events *cpuhw = &per_cpu(cpu_hw_events, cpu);
 646
 647         memset(cpuhw, 0, sizeof(*cpuhw));
 648 }
 649
 650 int register_fsl_emb_pmu(struct fsl_emb_pmu *pmu)
 651 {
 652         if (ppmu)
 653                 return -EBUSY;          /* something's already registered */
 654
 655         ppmu = pmu;
 656         pr_info("%s performance monitor hardware support registered\n",
 657                 pmu->name);
 658
 659         perf_pmu_register(&fsl_emb_pmu);
 660
 661         return 0;
 662 }