kernel/perf_counter.c

   1 /*
   2  * Performance counter core code
   3  *
   4  *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
   5  *  Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
   6  *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
   7  *  Copyright  ©  2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
   8  *
   9  *  For licensing details see kernel-base/COPYING
  10  */
  11
  12 #include <linux/fs.h>
  13 #include <linux/mm.h>
  14 #include <linux/cpu.h>
  15 #include <linux/smp.h>
  16 #include <linux/file.h>
  17 #include <linux/poll.h>
  18 #include <linux/sysfs.h>
  19 #include <linux/ptrace.h>
  20 #include <linux/percpu.h>
  21 #include <linux/vmstat.h>
  22 #include <linux/hardirq.h>
  23 #include <linux/rculist.h>
  24 #include <linux/uaccess.h>
  25 #include <linux/syscalls.h>
  26 #include <linux/anon_inodes.h>
  27 #include <linux/kernel_stat.h>
  28 #include <linux/perf_counter.h>
  29 #include <linux/dcache.h>
  30
  31 #include <asm/irq_regs.h>
  32
  33 /*
  34  * Each CPU has a list of per CPU counters:
  35  */
  36 DEFINE_PER_CPU(struct perf_cpu_context, perf_cpu_context);
  37
  38 int perf_max_counters __read_mostly = 1;
  39 static int perf_reserved_percpu __read_mostly;
  40 static int perf_overcommit __read_mostly = 1;
  41
  42 static atomic_t nr_counters __read_mostly;
  43 static atomic_t nr_mmap_tracking __read_mostly;
  44 static atomic_t nr_munmap_tracking __read_mostly;
  45 static atomic_t nr_comm_tracking __read_mostly;
  46
  47 int sysctl_perf_counter_priv __read_mostly; /* do we need to be privileged */
  48 int sysctl_perf_counter_mlock __read_mostly = 128; /* 'free' kb per counter */
  49
  50 /*
  51  * Lock for (sysadmin-configurable) counter reservations:
  52  */
  53 static DEFINE_SPINLOCK(perf_resource_lock);
  54
  55 /*
  56  * Architecture provided APIs - weak aliases:
  57  */
  58 extern __weak const struct pmu *hw_perf_counter_init(struct perf_counter *counter)
  59 {
  60         return NULL;
  61 }
  62
  63 void __weak hw_perf_disable(void)               { barrier(); }
  64 void __weak hw_perf_enable(void)                { barrier(); }
  65
  66 void __weak hw_perf_counter_setup(int cpu)      { barrier(); }
  67 int __weak hw_perf_group_sched_in(struct perf_counter *group_leader,
  68                struct perf_cpu_context *cpuctx,
  69                struct perf_counter_context *ctx, int cpu)
  70 {
  71         return 0;
  72 }
  73
  74 void __weak perf_counter_print_debug(void)      { }
  75
  76 static DEFINE_PER_CPU(int, disable_count);
  77
  78 void __perf_disable(void)
  79 {
  80         __get_cpu_var(disable_count)++;
  81 }
  82
  83 bool __perf_enable(void)
  84 {
  85         return !--__get_cpu_var(disable_count);
  86 }
  87
  88 void perf_disable(void)
  89 {
  90         __perf_disable();
  91         hw_perf_disable();
  92 }
  93 EXPORT_SYMBOL_GPL(perf_disable); /* ACPI idle */
  94
  95 void perf_enable(void)
  96 {
  97         if (__perf_enable())
  98                 hw_perf_enable();
  99 }
 100 EXPORT_SYMBOL_GPL(perf_enable); /* ACPI idle */
 101
 102 static void
 103 list_add_counter(struct perf_counter *counter, struct perf_counter_context *ctx)
 104 {
 105         struct perf_counter *group_leader = counter->group_leader;
 106
 107         /*
 108          * Depending on whether it is a standalone or sibling counter,
 109          * add it straight to the context's counter list, or to the group
 110          * leader's sibling list:
 111          */
 112         if (group_leader == counter)
 113                 list_add_tail(&counter->list_entry, &ctx->counter_list);
 114         else {
 115                 list_add_tail(&counter->list_entry, &group_leader->sibling_list);
 116                 group_leader->nr_siblings++;
 117         }
 118
 119         list_add_rcu(&counter->event_entry, &ctx->event_list);
 120 }
 121
 122 static void
 123 list_del_counter(struct perf_counter *counter, struct perf_counter_context *ctx)
 124 {
 125         struct perf_counter *sibling, *tmp;
 126
 127         list_del_init(&counter->list_entry);
 128         list_del_rcu(&counter->event_entry);
 129
 130         if (counter->group_leader != counter)
 131                 counter->group_leader->nr_siblings--;
 132
 133         /*
 134          * If this was a group counter with sibling counters then
 135          * upgrade the siblings to singleton counters by adding them
 136          * to the context list directly:
 137          */
 138         list_for_each_entry_safe(sibling, tmp,
 139                                  &counter->sibling_list, list_entry) {
 140
 141                 list_move_tail(&sibling->list_entry, &ctx->counter_list);
 142                 sibling->group_leader = sibling;
 143         }
 144 }
 145
 146 static void
 147 counter_sched_out(struct perf_counter *counter,
 148                   struct perf_cpu_context *cpuctx,
 149                   struct perf_counter_context *ctx)
 150 {
 151         if (counter->state != PERF_COUNTER_STATE_ACTIVE)
 152                 return;
 153
 154         counter->state = PERF_COUNTER_STATE_INACTIVE;
 155         counter->tstamp_stopped = ctx->time;
 156         counter->pmu->disable(counter);
 157         counter->oncpu = -1;
 158
 159         if (!is_software_counter(counter))
 160                 cpuctx->active_oncpu--;
 161         ctx->nr_active--;
 162         if (counter->hw_event.exclusive || !cpuctx->active_oncpu)
 163                 cpuctx->exclusive = 0;
 164 }
 165
 166 static void
 167 group_sched_out(struct perf_counter *group_counter,
 168                 struct perf_cpu_context *cpuctx,
 169                 struct perf_counter_context *ctx)
 170 {
 171         struct perf_counter *counter;
 172
 173         if (group_counter->state != PERF_COUNTER_STATE_ACTIVE)
 174                 return;
 175
 176         counter_sched_out(group_counter, cpuctx, ctx);
 177
 178         /*
 179          * Schedule out siblings (if any):
 180          */
 181         list_for_each_entry(counter, &group_counter->sibling_list, list_entry)
 182                 counter_sched_out(counter, cpuctx, ctx);
 183
 184         if (group_counter->hw_event.exclusive)
 185                 cpuctx->exclusive = 0;
 186 }
 187
 188 /*
 189  * Cross CPU call to remove a performance counter
 190  *
 191  * We disable the counter on the hardware level first. After that we
 192  * remove it from the context list.
 193  */
 194 static void __perf_counter_remove_from_context(void *info)
 195 {
 196         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
 197         struct perf_counter *counter = info;
 198         struct perf_counter_context *ctx = counter->ctx;
 199         unsigned long flags;
 200
 201         /*
 202          * If this is a task context, we need to check whether it is
 203          * the current task context of this cpu. If not it has been
 204          * scheduled out before the smp call arrived.
 205          */
 206         if (ctx->task && cpuctx->task_ctx != ctx)
 207                 return;
 208
 209         spin_lock_irqsave(&ctx->lock, flags);
 210
 211         counter_sched_out(counter, cpuctx, ctx);
 212
 213         counter->task = NULL;
 214         ctx->nr_counters--;
 215
 216         /*
 217          * Protect the list operation against NMI by disabling the
 218          * counters on a global level. NOP for non NMI based counters.
 219          */
 220         perf_disable();
 221         list_del_counter(counter, ctx);
 222         perf_enable();
 223
 224         if (!ctx->task) {
 225                 /*
 226                  * Allow more per task counters with respect to the
 227                  * reservation:
 228                  */
 229                 cpuctx->max_pertask =
 230                         min(perf_max_counters - ctx->nr_counters,
 231                             perf_max_counters - perf_reserved_percpu);
 232         }
 233
 234         spin_unlock_irqrestore(&ctx->lock, flags);
 235 }
 236
 237
 238 /*
 239  * Remove the counter from a task's (or a CPU's) list of counters.
 240  *
 241  * Must be called with counter->mutex and ctx->mutex held.
 242  *
 243  * CPU counters are removed with a smp call. For task counters we only
 244  * call when the task is on a CPU.
 245  */
 246 static void perf_counter_remove_from_context(struct perf_counter *counter)
 247 {
 248         struct perf_counter_context *ctx = counter->ctx;
 249         struct task_struct *task = ctx->task;
 250
 251         if (!task) {
 252                 /*
 253                  * Per cpu counters are removed via an smp call and
 254                  * the removal is always sucessful.
 255                  */
 256                 smp_call_function_single(counter->cpu,
 257                                          __perf_counter_remove_from_context,
 258                                          counter, 1);
 259                 return;
 260         }
 261
 262 retry:
 263         task_oncpu_function_call(task, __perf_counter_remove_from_context,
 264                                  counter);
 265
 266         spin_lock_irq(&ctx->lock);
 267         /*
 268          * If the context is active we need to retry the smp call.
 269          */
 270         if (ctx->nr_active && !list_empty(&counter->list_entry)) {
 271                 spin_unlock_irq(&ctx->lock);
 272                 goto retry;
 273         }
 274
 275         /*
 276          * The lock prevents that this context is scheduled in so we
 277          * can remove the counter safely, if the call above did not
 278          * succeed.
 279          */
 280         if (!list_empty(&counter->list_entry)) {
 281                 ctx->nr_counters--;
 282                 list_del_counter(counter, ctx);
 283                 counter->task = NULL;
 284         }
 285         spin_unlock_irq(&ctx->lock);
 286 }
 287
 288 static inline u64 perf_clock(void)
 289 {
 290         return cpu_clock(smp_processor_id());
 291 }
 292
 293 /*
 294  * Update the record of the current time in a context.
 295  */
 296 static void update_context_time(struct perf_counter_context *ctx)
 297 {
 298         u64 now = perf_clock();
 299
 300         ctx->time += now - ctx->timestamp;
 301         ctx->timestamp = now;
 302 }
 303
 304 /*
 305  * Update the total_time_enabled and total_time_running fields for a counter.
 306  */
 307 static void update_counter_times(struct perf_counter *counter)
 308 {
 309         struct perf_counter_context *ctx = counter->ctx;
 310         u64 run_end;
 311
 312         if (counter->state < PERF_COUNTER_STATE_INACTIVE)
 313                 return;
 314
 315         counter->total_time_enabled = ctx->time - counter->tstamp_enabled;
 316
 317         if (counter->state == PERF_COUNTER_STATE_INACTIVE)
 318                 run_end = counter->tstamp_stopped;
 319         else
 320                 run_end = ctx->time;
 321
 322         counter->total_time_running = run_end - counter->tstamp_running;
 323 }
 324
 325 /*
 326  * Update total_time_enabled and total_time_running for all counters in a group.
 327  */
 328 static void update_group_times(struct perf_counter *leader)
 329 {
 330         struct perf_counter *counter;
 331
 332         update_counter_times(leader);
 333         list_for_each_entry(counter, &leader->sibling_list, list_entry)
 334                 update_counter_times(counter);
 335 }
 336
 337 /*
 338  * Cross CPU call to disable a performance counter
 339  */
 340 static void __perf_counter_disable(void *info)
 341 {
 342         struct perf_counter *counter = info;
 343         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
 344         struct perf_counter_context *ctx = counter->ctx;
 345         unsigned long flags;
 346
 347         /*
 348          * If this is a per-task counter, need to check whether this
 349          * counter's task is the current task on this cpu.
 350          */
 351         if (ctx->task && cpuctx->task_ctx != ctx)
 352                 return;
 353
 354         spin_lock_irqsave(&ctx->lock, flags);
 355
 356         /*
 357          * If the counter is on, turn it off.
 358          * If it is in error state, leave it in error state.
 359          */
 360         if (counter->state >= PERF_COUNTER_STATE_INACTIVE) {
 361                 update_context_time(ctx);
 362                 update_counter_times(counter);
 363                 if (counter == counter->group_leader)
 364                         group_sched_out(counter, cpuctx, ctx);
 365                 else
 366                         counter_sched_out(counter, cpuctx, ctx);
 367                 counter->state = PERF_COUNTER_STATE_OFF;
 368         }
 369
 370         spin_unlock_irqrestore(&ctx->lock, flags);
 371 }
 372
 373 /*
 374  * Disable a counter.
 375  */
 376 static void perf_counter_disable(struct perf_counter *counter)
 377 {
 378         struct perf_counter_context *ctx = counter->ctx;
 379         struct task_struct *task = ctx->task;
 380
 381         if (!task) {
 382                 /*
 383                  * Disable the counter on the cpu that it's on
 384                  */
 385                 smp_call_function_single(counter->cpu, __perf_counter_disable,
 386                                          counter, 1);
 387                 return;
 388         }
 389
 390  retry:
 391         task_oncpu_function_call(task, __perf_counter_disable, counter);
 392
 393         spin_lock_irq(&ctx->lock);
 394         /*
 395          * If the counter is still active, we need to retry the cross-call.
 396          */
 397         if (counter->state == PERF_COUNTER_STATE_ACTIVE) {
 398                 spin_unlock_irq(&ctx->lock);
 399                 goto retry;
 400         }
 401
 402         /*
 403          * Since we have the lock this context can't be scheduled
 404          * in, so we can change the state safely.
 405          */
 406         if (counter->state == PERF_COUNTER_STATE_INACTIVE) {
 407                 update_counter_times(counter);
 408                 counter->state = PERF_COUNTER_STATE_OFF;
 409         }
 410
 411         spin_unlock_irq(&ctx->lock);
 412 }
 413
 414 static int
 415 counter_sched_in(struct perf_counter *counter,
 416                  struct perf_cpu_context *cpuctx,
 417                  struct perf_counter_context *ctx,
 418                  int cpu)
 419 {
 420         if (counter->state <= PERF_COUNTER_STATE_OFF)
 421                 return 0;
 422
 423         counter->state = PERF_COUNTER_STATE_ACTIVE;
 424         counter->oncpu = cpu;   /* TODO: put 'cpu' into cpuctx->cpu */
 425         /*
 426          * The new state must be visible before we turn it on in the hardware:
 427          */
 428         smp_wmb();
 429
 430         if (counter->pmu->enable(counter)) {
 431                 counter->state = PERF_COUNTER_STATE_INACTIVE;
 432                 counter->oncpu = -1;
 433                 return -EAGAIN;
 434         }
 435
 436         counter->tstamp_running += ctx->time - counter->tstamp_stopped;
 437
 438         if (!is_software_counter(counter))
 439                 cpuctx->active_oncpu++;
 440         ctx->nr_active++;
 441
 442         if (counter->hw_event.exclusive)
 443                 cpuctx->exclusive = 1;
 444
 445         return 0;
 446 }
 447
 448 static int
 449 group_sched_in(struct perf_counter *group_counter,
 450                struct perf_cpu_context *cpuctx,
 451                struct perf_counter_context *ctx,
 452                int cpu)
 453 {
 454         struct perf_counter *counter, *partial_group;
 455         int ret;
 456
 457         if (group_counter->state == PERF_COUNTER_STATE_OFF)
 458                 return 0;
 459
 460         ret = hw_perf_group_sched_in(group_counter, cpuctx, ctx, cpu);
 461         if (ret)
 462                 return ret < 0 ? ret : 0;
 463
 464         group_counter->prev_state = group_counter->state;
 465         if (counter_sched_in(group_counter, cpuctx, ctx, cpu))
 466                 return -EAGAIN;
 467
 468         /*
 469          * Schedule in siblings as one group (if any):
 470          */
 471         list_for_each_entry(counter, &group_counter->sibling_list, list_entry) {
 472                 counter->prev_state = counter->state;
 473                 if (counter_sched_in(counter, cpuctx, ctx, cpu)) {
 474                         partial_group = counter;
 475                         goto group_error;
 476                 }
 477         }
 478
 479         return 0;
 480
 481 group_error:
 482         /*
 483          * Groups can be scheduled in as one unit only, so undo any
 484          * partial group before returning:
 485          */
 486         list_for_each_entry(counter, &group_counter->sibling_list, list_entry) {
 487                 if (counter == partial_group)
 488                         break;
 489                 counter_sched_out(counter, cpuctx, ctx);
 490         }
 491         counter_sched_out(group_counter, cpuctx, ctx);
 492
 493         return -EAGAIN;
 494 }
 495
 496 /*
 497  * Return 1 for a group consisting entirely of software counters,
 498  * 0 if the group contains any hardware counters.
 499  */
 500 static int is_software_only_group(struct perf_counter *leader)
 501 {
 502         struct perf_counter *counter;
 503
 504         if (!is_software_counter(leader))
 505                 return 0;
 506
 507         list_for_each_entry(counter, &leader->sibling_list, list_entry)
 508                 if (!is_software_counter(counter))
 509                         return 0;
 510
 511         return 1;
 512 }
 513
 514 /*
 515  * Work out whether we can put this counter group on the CPU now.
 516  */
 517 static int group_can_go_on(struct perf_counter *counter,
 518                            struct perf_cpu_context *cpuctx,
 519                            int can_add_hw)
 520 {
 521         /*
 522          * Groups consisting entirely of software counters can always go on.
 523          */
 524         if (is_software_only_group(counter))
 525                 return 1;
 526         /*
 527          * If an exclusive group is already on, no other hardware
 528          * counters can go on.
 529          */
 530         if (cpuctx->exclusive)
 531                 return 0;
 532         /*
 533          * If this group is exclusive and there are already
 534          * counters on the CPU, it can't go on.
 535          */
 536         if (counter->hw_event.exclusive && cpuctx->active_oncpu)
 537                 return 0;
 538         /*
 539          * Otherwise, try to add it if all previous groups were able
 540          * to go on.
 541          */
 542         return can_add_hw;
 543 }
 544
 545 static void add_counter_to_ctx(struct perf_counter *counter,
 546                                struct perf_counter_context *ctx)
 547 {
 548         list_add_counter(counter, ctx);
 549         ctx->nr_counters++;
 550         counter->prev_state = PERF_COUNTER_STATE_OFF;
 551         counter->tstamp_enabled = ctx->time;
 552         counter->tstamp_running = ctx->time;
 553         counter->tstamp_stopped = ctx->time;
 554 }
 555
 556 /*
 557  * Cross CPU call to install and enable a performance counter
 558  */
 559 static void __perf_install_in_context(void *info)
 560 {
 561         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
 562         struct perf_counter *counter = info;
 563         struct perf_counter_context *ctx = counter->ctx;
 564         struct perf_counter *leader = counter->group_leader;
 565         int cpu = smp_processor_id();
 566         unsigned long flags;
 567         int err;
 568
 569         /*
 570          * If this is a task context, we need to check whether it is
 571          * the current task context of this cpu. If not it has been
 572          * scheduled out before the smp call arrived.
 573          */
 574         if (ctx->task && cpuctx->task_ctx != ctx)
 575                 return;
 576
 577         spin_lock_irqsave(&ctx->lock, flags);
 578         update_context_time(ctx);
 579
 580         /*
 581          * Protect the list operation against NMI by disabling the
 582          * counters on a global level. NOP for non NMI based counters.
 583          */
 584         perf_disable();
 585
 586         add_counter_to_ctx(counter, ctx);
 587
 588         /*
 589          * Don't put the counter on if it is disabled or if
 590          * it is in a group and the group isn't on.
 591          */
 592         if (counter->state != PERF_COUNTER_STATE_INACTIVE ||
 593             (leader != counter && leader->state != PERF_COUNTER_STATE_ACTIVE))
 594                 goto unlock;
 595
 596         /*
 597          * An exclusive counter can't go on if there are already active
 598          * hardware counters, and no hardware counter can go on if there
 599          * is already an exclusive counter on.
 600          */
 601         if (!group_can_go_on(counter, cpuctx, 1))
 602                 err = -EEXIST;
 603         else
 604                 err = counter_sched_in(counter, cpuctx, ctx, cpu);
 605
 606         if (err) {
 607                 /*
 608                  * This counter couldn't go on.  If it is in a group
 609                  * then we have to pull the whole group off.
 610                  * If the counter group is pinned then put it in error state.
 611                  */
 612                 if (leader != counter)
 613                         group_sched_out(leader, cpuctx, ctx);
 614                 if (leader->hw_event.pinned) {
 615                         update_group_times(leader);
 616                         leader->state = PERF_COUNTER_STATE_ERROR;
 617                 }
 618         }
 619
 620         if (!err && !ctx->task && cpuctx->max_pertask)
 621                 cpuctx->max_pertask--;
 622
 623  unlock:
 624         perf_enable();
 625
 626         spin_unlock_irqrestore(&ctx->lock, flags);
 627 }
 628
 629 /*
 630  * Attach a performance counter to a context
 631  *
 632  * First we add the counter to the list with the hardware enable bit
 633  * in counter->hw_config cleared.
 634  *
 635  * If the counter is attached to a task which is on a CPU we use a smp
 636  * call to enable it in the task context. The task might have been
 637  * scheduled away, but we check this in the smp call again.
 638  *
 639  * Must be called with ctx->mutex held.
 640  */
 641 static void
 642 perf_install_in_context(struct perf_counter_context *ctx,
 643                         struct perf_counter *counter,
 644                         int cpu)
 645 {
 646         struct task_struct *task = ctx->task;
 647
 648         if (!task) {
 649                 /*
 650                  * Per cpu counters are installed via an smp call and
 651                  * the install is always sucessful.
 652                  */
 653                 smp_call_function_single(cpu, __perf_install_in_context,
 654                                          counter, 1);
 655                 return;
 656         }
 657
 658         counter->task = task;
 659 retry:
 660         task_oncpu_function_call(task, __perf_install_in_context,
 661                                  counter);
 662
 663         spin_lock_irq(&ctx->lock);
 664         /*
 665          * we need to retry the smp call.
 666          */
 667         if (ctx->is_active && list_empty(&counter->list_entry)) {
 668                 spin_unlock_irq(&ctx->lock);
 669                 goto retry;
 670         }
 671
 672         /*
 673          * The lock prevents that this context is scheduled in so we
 674          * can add the counter safely, if it the call above did not
 675          * succeed.
 676          */
 677         if (list_empty(&counter->list_entry))
 678                 add_counter_to_ctx(counter, ctx);
 679         spin_unlock_irq(&ctx->lock);
 680 }
 681
 682 /*
 683  * Cross CPU call to enable a performance counter
 684  */
 685 static void __perf_counter_enable(void *info)
 686 {
 687         struct perf_counter *counter = info;
 688         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
 689         struct perf_counter_context *ctx = counter->ctx;
 690         struct perf_counter *leader = counter->group_leader;
 691         unsigned long flags;
 692         int err;
 693
 694         /*
 695          * If this is a per-task counter, need to check whether this
 696          * counter's task is the current task on this cpu.
 697          */
 698         if (ctx->task && cpuctx->task_ctx != ctx)
 699                 return;
 700
 701         spin_lock_irqsave(&ctx->lock, flags);
 702         update_context_time(ctx);
 703
 704         counter->prev_state = counter->state;
 705         if (counter->state >= PERF_COUNTER_STATE_INACTIVE)
 706                 goto unlock;
 707         counter->state = PERF_COUNTER_STATE_INACTIVE;
 708         counter->tstamp_enabled = ctx->time - counter->total_time_enabled;
 709
 710         /*
 711          * If the counter is in a group and isn't the group leader,
 712          * then don't put it on unless the group is on.
 713          */
 714         if (leader != counter && leader->state != PERF_COUNTER_STATE_ACTIVE)
 715                 goto unlock;
 716
 717         if (!group_can_go_on(counter, cpuctx, 1)) {
 718                 err = -EEXIST;
 719         } else {
 720                 perf_disable();
 721                 if (counter == leader)
 722                         err = group_sched_in(counter, cpuctx, ctx,
 723                                              smp_processor_id());
 724                 else
 725                         err = counter_sched_in(counter, cpuctx, ctx,
 726                                                smp_processor_id());
 727                 perf_enable();
 728         }
 729
 730         if (err) {
 731                 /*
 732                  * If this counter can't go on and it's part of a
 733                  * group, then the whole group has to come off.
 734                  */
 735                 if (leader != counter)
 736                         group_sched_out(leader, cpuctx, ctx);
 737                 if (leader->hw_event.pinned) {
 738                         update_group_times(leader);
 739                         leader->state = PERF_COUNTER_STATE_ERROR;
 740                 }
 741         }
 742
 743  unlock:
 744         spin_unlock_irqrestore(&ctx->lock, flags);
 745 }
 746
 747 /*
 748  * Enable a counter.
 749  */
 750 static void perf_counter_enable(struct perf_counter *counter)
 751 {
 752         struct perf_counter_context *ctx = counter->ctx;
 753         struct task_struct *task = ctx->task;
 754
 755         if (!task) {
 756                 /*
 757                  * Enable the counter on the cpu that it's on
 758                  */
 759                 smp_call_function_single(counter->cpu, __perf_counter_enable,
 760                                          counter, 1);
 761                 return;
 762         }
 763
 764         spin_lock_irq(&ctx->lock);
 765         if (counter->state >= PERF_COUNTER_STATE_INACTIVE)
 766                 goto out;
 767
 768         /*
 769          * If the counter is in error state, clear that first.
 770          * That way, if we see the counter in error state below, we
 771          * know that it has gone back into error state, as distinct
 772          * from the task having been scheduled away before the
 773          * cross-call arrived.
 774          */
 775         if (counter->state == PERF_COUNTER_STATE_ERROR)
 776                 counter->state = PERF_COUNTER_STATE_OFF;
 777
 778  retry:
 779         spin_unlock_irq(&ctx->lock);
 780         task_oncpu_function_call(task, __perf_counter_enable, counter);
 781
 782         spin_lock_irq(&ctx->lock);
 783
 784         /*
 785          * If the context is active and the counter is still off,
 786          * we need to retry the cross-call.
 787          */
 788         if (ctx->is_active && counter->state == PERF_COUNTER_STATE_OFF)
 789                 goto retry;
 790
 791         /*
 792          * Since we have the lock this context can't be scheduled
 793          * in, so we can change the state safely.
 794          */
 795         if (counter->state == PERF_COUNTER_STATE_OFF) {
 796                 counter->state = PERF_COUNTER_STATE_INACTIVE;
 797                 counter->tstamp_enabled =
 798                         ctx->time - counter->total_time_enabled;
 799         }
 800  out:
 801         spin_unlock_irq(&ctx->lock);
 802 }
 803
 804 static int perf_counter_refresh(struct perf_counter *counter, int refresh)
 805 {
 806         /*
 807          * not supported on inherited counters
 808          */
 809         if (counter->hw_event.inherit)
 810                 return -EINVAL;
 811
 812         atomic_add(refresh, &counter->event_limit);
 813         perf_counter_enable(counter);
 814
 815         return 0;
 816 }
 817
 818 void __perf_counter_sched_out(struct perf_counter_context *ctx,
 819                               struct perf_cpu_context *cpuctx)
 820 {
 821         struct perf_counter *counter;
 822
 823         spin_lock(&ctx->lock);
 824         ctx->is_active = 0;
 825         if (likely(!ctx->nr_counters))
 826                 goto out;
 827         update_context_time(ctx);
 828
 829         perf_disable();
 830         if (ctx->nr_active) {
 831                 list_for_each_entry(counter, &ctx->counter_list, list_entry)
 832                         group_sched_out(counter, cpuctx, ctx);
 833         }
 834         perf_enable();
 835  out:
 836         spin_unlock(&ctx->lock);
 837 }
 838
 839 /*
 840  * Called from scheduler to remove the counters of the current task,
 841  * with interrupts disabled.
 842  *
 843  * We stop each counter and update the counter value in counter->count.
 844  *
 845  * This does not protect us against NMI, but disable()
 846  * sets the disabled bit in the control field of counter _before_
 847  * accessing the counter control register. If a NMI hits, then it will
 848  * not restart the counter.
 849  */
 850 void perf_counter_task_sched_out(struct task_struct *task, int cpu)
 851 {
 852         struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
 853         struct perf_counter_context *ctx = &task->perf_counter_ctx;
 854         struct pt_regs *regs;
 855
 856         if (likely(!cpuctx->task_ctx))
 857                 return;
 858
 859         update_context_time(ctx);
 860
 861         regs = task_pt_regs(task);
 862         perf_swcounter_event(PERF_COUNT_CONTEXT_SWITCHES, 1, 1, regs, 0);
 863         __perf_counter_sched_out(ctx, cpuctx);
 864
 865         cpuctx->task_ctx = NULL;
 866 }
 867
 868 static void __perf_counter_task_sched_out(struct perf_counter_context *ctx)
 869 {
 870         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
 871
 872         __perf_counter_sched_out(ctx, cpuctx);
 873         cpuctx->task_ctx = NULL;
 874 }
 875
 876 static void perf_counter_cpu_sched_out(struct perf_cpu_context *cpuctx)
 877 {
 878         __perf_counter_sched_out(&cpuctx->ctx, cpuctx);
 879 }
 880
 881 static void
 882 __perf_counter_sched_in(struct perf_counter_context *ctx,
 883                         struct perf_cpu_context *cpuctx, int cpu)
 884 {
 885         struct perf_counter *counter;
 886         int can_add_hw = 1;
 887
 888         spin_lock(&ctx->lock);
 889         ctx->is_active = 1;
 890         if (likely(!ctx->nr_counters))
 891                 goto out;
 892
 893         ctx->timestamp = perf_clock();
 894
 895         perf_disable();
 896
 897         /*
 898          * First go through the list and put on any pinned groups
 899          * in order to give them the best chance of going on.
 900          */
 901         list_for_each_entry(counter, &ctx->counter_list, list_entry) {
 902                 if (counter->state <= PERF_COUNTER_STATE_OFF ||
 903                     !counter->hw_event.pinned)
 904                         continue;
 905                 if (counter->cpu != -1 && counter->cpu != cpu)
 906                         continue;
 907
 908                 if (group_can_go_on(counter, cpuctx, 1))
 909                         group_sched_in(counter, cpuctx, ctx, cpu);
 910
 911                 /*
 912                  * If this pinned group hasn't been scheduled,
 913                  * put it in error state.
 914                  */
 915                 if (counter->state == PERF_COUNTER_STATE_INACTIVE) {
 916                         update_group_times(counter);
 917                         counter->state = PERF_COUNTER_STATE_ERROR;
 918                 }
 919         }
 920
 921         list_for_each_entry(counter, &ctx->counter_list, list_entry) {
 922                 /*
 923                  * Ignore counters in OFF or ERROR state, and
 924                  * ignore pinned counters since we did them already.
 925                  */
 926                 if (counter->state <= PERF_COUNTER_STATE_OFF ||
 927                     counter->hw_event.pinned)
 928                         continue;
 929
 930                 /*
 931                  * Listen to the 'cpu' scheduling filter constraint
 932                  * of counters:
 933                  */
 934                 if (counter->cpu != -1 && counter->cpu != cpu)
 935                         continue;
 936
 937                 if (group_can_go_on(counter, cpuctx, can_add_hw)) {
 938                         if (group_sched_in(counter, cpuctx, ctx, cpu))
 939                                 can_add_hw = 0;
 940                 }
 941         }
 942         perf_enable();
 943  out:
 944         spin_unlock(&ctx->lock);
 945 }
 946
 947 /*
 948  * Called from scheduler to add the counters of the current task
 949  * with interrupts disabled.
 950  *
 951  * We restore the counter value and then enable it.
 952  *
 953  * This does not protect us against NMI, but enable()
 954  * sets the enabled bit in the control field of counter _before_
 955  * accessing the counter control register. If a NMI hits, then it will
 956  * keep the counter running.
 957  */
 958 void perf_counter_task_sched_in(struct task_struct *task, int cpu)
 959 {
 960         struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
 961         struct perf_counter_context *ctx = &task->perf_counter_ctx;
 962
 963         __perf_counter_sched_in(ctx, cpuctx, cpu);
 964         cpuctx->task_ctx = ctx;
 965 }
 966
 967 static void perf_counter_cpu_sched_in(struct perf_cpu_context *cpuctx, int cpu)
 968 {
 969         struct perf_counter_context *ctx = &cpuctx->ctx;
 970
 971         __perf_counter_sched_in(ctx, cpuctx, cpu);
 972 }
 973
 974 int perf_counter_task_disable(void)
 975 {
 976         struct task_struct *curr = current;
 977         struct perf_counter_context *ctx = &curr->perf_counter_ctx;
 978         struct perf_counter *counter;
 979         unsigned long flags;
 980
 981         if (likely(!ctx->nr_counters))
 982                 return 0;
 983
 984         local_irq_save(flags);
 985
 986         __perf_counter_task_sched_out(ctx);
 987
 988         spin_lock(&ctx->lock);
 989
 990         /*
 991          * Disable all the counters:
 992          */
 993         perf_disable();
 994
 995         list_for_each_entry(counter, &ctx->counter_list, list_entry) {
 996                 if (counter->state != PERF_COUNTER_STATE_ERROR) {
 997                         update_group_times(counter);
 998                         counter->state = PERF_COUNTER_STATE_OFF;
 999                 }
1000         }
1001
1002         perf_enable();
1003
1004         spin_unlock_irqrestore(&ctx->lock, flags);
1005
1006         return 0;
1007 }
1008
1009 int perf_counter_task_enable(void)
1010 {
1011         struct task_struct *curr = current;
1012         struct perf_counter_context *ctx = &curr->perf_counter_ctx;
1013         struct perf_counter *counter;
1014         unsigned long flags;
1015         int cpu;
1016
1017         if (likely(!ctx->nr_counters))
1018                 return 0;
1019
1020         local_irq_save(flags);
1021         cpu = smp_processor_id();
1022
1023         __perf_counter_task_sched_out(ctx);
1024
1025         spin_lock(&ctx->lock);
1026
1027         /*
1028          * Disable all the counters:
1029          */
1030         perf_disable();
1031
1032         list_for_each_entry(counter, &ctx->counter_list, list_entry) {
1033                 if (counter->state > PERF_COUNTER_STATE_OFF)
1034                         continue;
1035                 counter->state = PERF_COUNTER_STATE_INACTIVE;
1036                 counter->tstamp_enabled =
1037                         ctx->time - counter->total_time_enabled;
1038                 counter->hw_event.disabled = 0;
1039         }
1040         perf_enable();
1041
1042         spin_unlock(&ctx->lock);
1043
1044         perf_counter_task_sched_in(curr, cpu);
1045
1046         local_irq_restore(flags);
1047
1048         return 0;
1049 }
1050
1051 /*
1052  * Round-robin a context's counters:
1053  */
1054 static void rotate_ctx(struct perf_counter_context *ctx)
1055 {
1056         struct perf_counter *counter;
1057
1058         if (!ctx->nr_counters)
1059                 return;
1060
1061         spin_lock(&ctx->lock);
1062         /*
1063          * Rotate the first entry last (works just fine for group counters too):
1064          */
1065         perf_disable();
1066         list_for_each_entry(counter, &ctx->counter_list, list_entry) {
1067                 list_move_tail(&counter->list_entry, &ctx->counter_list);
1068                 break;
1069         }
1070         perf_enable();
1071
1072         spin_unlock(&ctx->lock);
1073 }
1074
1075 void perf_counter_task_tick(struct task_struct *curr, int cpu)
1076 {
1077         struct perf_cpu_context *cpuctx;
1078         struct perf_counter_context *ctx;
1079
1080         if (!atomic_read(&nr_counters))
1081                 return;
1082
1083         cpuctx = &per_cpu(perf_cpu_context, cpu);
1084         ctx = &curr->perf_counter_ctx;
1085
1086         perf_counter_cpu_sched_out(cpuctx);
1087         __perf_counter_task_sched_out(ctx);
1088
1089         rotate_ctx(&cpuctx->ctx);
1090         rotate_ctx(ctx);
1091
1092         perf_counter_cpu_sched_in(cpuctx, cpu);
1093         perf_counter_task_sched_in(curr, cpu);
1094 }
1095
1096 /*
1097  * Cross CPU call to read the hardware counter
1098  */
1099 static void __read(void *info)
1100 {
1101         struct perf_counter *counter = info;
1102         struct perf_counter_context *ctx = counter->ctx;
1103         unsigned long flags;
1104
1105         local_irq_save(flags);
1106         if (ctx->is_active)
1107                 update_context_time(ctx);
1108         counter->pmu->read(counter);
1109         update_counter_times(counter);
1110         local_irq_restore(flags);
1111 }
1112
1113 static u64 perf_counter_read(struct perf_counter *counter)
1114 {
1115         /*
1116          * If counter is enabled and currently active on a CPU, update the
1117          * value in the counter structure:
1118          */
1119         if (counter->state == PERF_COUNTER_STATE_ACTIVE) {
1120                 smp_call_function_single(counter->oncpu,
1121                                          __read, counter, 1);
1122         } else if (counter->state == PERF_COUNTER_STATE_INACTIVE) {
1123                 update_counter_times(counter);
1124         }
1125
1126         return atomic64_read(&counter->count);
1127 }
1128
1129 static void put_context(struct perf_counter_context *ctx)
1130 {
1131         if (ctx->task)
1132                 put_task_struct(ctx->task);
1133 }
1134
1135 static struct perf_counter_context *find_get_context(pid_t pid, int cpu)
1136 {
1137         struct perf_cpu_context *cpuctx;
1138         struct perf_counter_context *ctx;
1139         struct task_struct *task;
1140
1141         /*
1142          * If cpu is not a wildcard then this is a percpu counter:
1143          */
1144         if (cpu != -1) {
1145                 /* Must be root to operate on a CPU counter: */
1146                 if (sysctl_perf_counter_priv && !capable(CAP_SYS_ADMIN))
1147                         return ERR_PTR(-EACCES);
1148
1149                 if (cpu < 0 || cpu > num_possible_cpus())
1150                         return ERR_PTR(-EINVAL);
1151
1152                 /*
1153                  * We could be clever and allow to attach a counter to an
1154                  * offline CPU and activate it when the CPU comes up, but
1155                  * that's for later.
1156                  */
1157                 if (!cpu_isset(cpu, cpu_online_map))
1158                         return ERR_PTR(-ENODEV);
1159
1160                 cpuctx = &per_cpu(perf_cpu_context, cpu);
1161                 ctx = &cpuctx->ctx;
1162
1163                 return ctx;
1164         }
1165
1166         rcu_read_lock();
1167         if (!pid)
1168                 task = current;
1169         else
1170                 task = find_task_by_vpid(pid);
1171         if (task)
1172                 get_task_struct(task);
1173         rcu_read_unlock();
1174
1175         if (!task)
1176                 return ERR_PTR(-ESRCH);
1177
1178         ctx = &task->perf_counter_ctx;
1179         ctx->task = task;
1180
1181         /* Reuse ptrace permission checks for now. */
1182         if (!ptrace_may_access(task, PTRACE_MODE_READ)) {
1183                 put_context(ctx);
1184                 return ERR_PTR(-EACCES);
1185         }
1186
1187         return ctx;
1188 }
1189
1190 static void free_counter_rcu(struct rcu_head *head)
1191 {
1192         struct perf_counter *counter;
1193
1194         counter = container_of(head, struct perf_counter, rcu_head);
1195         kfree(counter);
1196 }
1197
1198 static void perf_pending_sync(struct perf_counter *counter);
1199
1200 static void free_counter(struct perf_counter *counter)
1201 {
1202         perf_pending_sync(counter);
1203
1204         atomic_dec(&nr_counters);
1205         if (counter->hw_event.mmap)
1206                 atomic_dec(&nr_mmap_tracking);
1207         if (counter->hw_event.munmap)
1208                 atomic_dec(&nr_munmap_tracking);
1209         if (counter->hw_event.comm)
1210                 atomic_dec(&nr_comm_tracking);
1211
1212         if (counter->destroy)
1213                 counter->destroy(counter);
1214
1215         call_rcu(&counter->rcu_head, free_counter_rcu);
1216 }
1217
1218 /*
1219  * Called when the last reference to the file is gone.
1220  */
1221 static int perf_release(struct inode *inode, struct file *file)
1222 {
1223         struct perf_counter *counter = file->private_data;
1224         struct perf_counter_context *ctx = counter->ctx;
1225
1226         file->private_data = NULL;
1227
1228         mutex_lock(&ctx->mutex);
1229         mutex_lock(&counter->mutex);
1230
1231         perf_counter_remove_from_context(counter);
1232
1233         mutex_unlock(&counter->mutex);
1234         mutex_unlock(&ctx->mutex);
1235
1236         free_counter(counter);
1237         put_context(ctx);
1238
1239         return 0;
1240 }
1241
1242 /*
1243  * Read the performance counter - simple non blocking version for now
1244  */
1245 static ssize_t
1246 perf_read_hw(struct perf_counter *counter, char __user *buf, size_t count)
1247 {
1248         u64 values[3];
1249         int n;
1250
1251         /*
1252          * Return end-of-file for a read on a counter that is in
1253          * error state (i.e. because it was pinned but it couldn't be
1254          * scheduled on to the CPU at some point).
1255          */
1256         if (counter->state == PERF_COUNTER_STATE_ERROR)
1257                 return 0;
1258
1259         mutex_lock(&counter->mutex);
1260         values[0] = perf_counter_read(counter);
1261         n = 1;
1262         if (counter->hw_event.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
1263                 values[n++] = counter->total_time_enabled +
1264                         atomic64_read(&counter->child_total_time_enabled);
1265         if (counter->hw_event.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
1266                 values[n++] = counter->total_time_running +
1267                         atomic64_read(&counter->child_total_time_running);
1268         mutex_unlock(&counter->mutex);
1269
1270         if (count < n * sizeof(u64))
1271                 return -EINVAL;
1272         count = n * sizeof(u64);
1273
1274         if (copy_to_user(buf, values, count))
1275                 return -EFAULT;
1276
1277         return count;
1278 }
1279
1280 static ssize_t
1281 perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
1282 {
1283         struct perf_counter *counter = file->private_data;
1284
1285         return perf_read_hw(counter, buf, count);
1286 }
1287
1288 static unsigned int perf_poll(struct file *file, poll_table *wait)
1289 {
1290         struct perf_counter *counter = file->private_data;
1291         struct perf_mmap_data *data;
1292         unsigned int events = POLL_HUP;
1293
1294         rcu_read_lock();
1295         data = rcu_dereference(counter->data);
1296         if (data)
1297                 events = atomic_xchg(&data->poll, 0);
1298         rcu_read_unlock();
1299
1300         poll_wait(file, &counter->waitq, wait);
1301
1302         return events;
1303 }
1304
1305 static void perf_counter_reset(struct perf_counter *counter)
1306 {
1307         (void)perf_counter_read(counter);
1308         atomic64_set(&counter->count, 0);
1309         perf_counter_update_userpage(counter);
1310 }
1311
1312 static void perf_counter_for_each_sibling(struct perf_counter *counter,
1313                                           void (*func)(struct perf_counter *))
1314 {
1315         struct perf_counter_context *ctx = counter->ctx;
1316         struct perf_counter *sibling;
1317
1318         spin_lock_irq(&ctx->lock);
1319         counter = counter->group_leader;
1320
1321         func(counter);
1322         list_for_each_entry(sibling, &counter->sibling_list, list_entry)
1323                 func(sibling);
1324         spin_unlock_irq(&ctx->lock);
1325 }
1326
1327 static void perf_counter_for_each_child(struct perf_counter *counter,
1328                                         void (*func)(struct perf_counter *))
1329 {
1330         struct perf_counter *child;
1331
1332         mutex_lock(&counter->mutex);
1333         func(counter);
1334         list_for_each_entry(child, &counter->child_list, child_list)
1335                 func(child);
1336         mutex_unlock(&counter->mutex);
1337 }
1338
1339 static void perf_counter_for_each(struct perf_counter *counter,
1340                                   void (*func)(struct perf_counter *))
1341 {
1342         struct perf_counter *child;
1343
1344         mutex_lock(&counter->mutex);
1345         perf_counter_for_each_sibling(counter, func);
1346         list_for_each_entry(child, &counter->child_list, child_list)
1347                 perf_counter_for_each_sibling(child, func);
1348         mutex_unlock(&counter->mutex);
1349 }
1350
1351 static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
1352 {
1353         struct perf_counter *counter = file->private_data;
1354         void (*func)(struct perf_counter *);
1355         u32 flags = arg;
1356
1357         switch (cmd) {
1358         case PERF_COUNTER_IOC_ENABLE:
1359                 func = perf_counter_enable;
1360                 break;
1361         case PERF_COUNTER_IOC_DISABLE:
1362                 func = perf_counter_disable;
1363                 break;
1364         case PERF_COUNTER_IOC_RESET:
1365                 func = perf_counter_reset;
1366                 break;
1367
1368         case PERF_COUNTER_IOC_REFRESH:
1369                 return perf_counter_refresh(counter, arg);
1370         default:
1371                 return -ENOTTY;
1372         }
1373
1374         if (flags & PERF_IOC_FLAG_GROUP)
1375                 perf_counter_for_each(counter, func);
1376         else
1377                 perf_counter_for_each_child(counter, func);
1378
1379         return 0;
1380 }
1381
1382 /*
1383  * Callers need to ensure there can be no nesting of this function, otherwise
1384  * the seqlock logic goes bad. We can not serialize this because the arch
1385  * code calls this from NMI context.
1386  */
1387 void perf_counter_update_userpage(struct perf_counter *counter)
1388 {
1389         struct perf_mmap_data *data;
1390         struct perf_counter_mmap_page *userpg;
1391
1392         rcu_read_lock();
1393         data = rcu_dereference(counter->data);
1394         if (!data)
1395                 goto unlock;
1396
1397         userpg = data->user_page;
1398
1399         /*
1400          * Disable preemption so as to not let the corresponding user-space
1401          * spin too long if we get preempted.
1402          */
1403         preempt_disable();
1404         ++userpg->lock;
1405         barrier();
1406         userpg->index = counter->hw.idx;
1407         userpg->offset = atomic64_read(&counter->count);
1408         if (counter->state == PERF_COUNTER_STATE_ACTIVE)
1409                 userpg->offset -= atomic64_read(&counter->hw.prev_count);
1410
1411         barrier();
1412         ++userpg->lock;
1413         preempt_enable();
1414 unlock:
1415         rcu_read_unlock();
1416 }
1417
1418 static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1419 {
1420         struct perf_counter *counter = vma->vm_file->private_data;
1421         struct perf_mmap_data *data;
1422         int ret = VM_FAULT_SIGBUS;
1423
1424         rcu_read_lock();
1425         data = rcu_dereference(counter->data);
1426         if (!data)
1427                 goto unlock;
1428
1429         if (vmf->pgoff == 0) {
1430                 vmf->page = virt_to_page(data->user_page);
1431         } else {
1432                 int nr = vmf->pgoff - 1;
1433
1434                 if ((unsigned)nr > data->nr_pages)
1435                         goto unlock;
1436
1437                 vmf->page = virt_to_page(data->data_pages[nr]);
1438         }
1439         get_page(vmf->page);
1440         ret = 0;
1441 unlock:
1442         rcu_read_unlock();
1443
1444         return ret;
1445 }
1446
1447 static int perf_mmap_data_alloc(struct perf_counter *counter, int nr_pages)
1448 {
1449         struct perf_mmap_data *data;
1450         unsigned long size;
1451         int i;
1452
1453         WARN_ON(atomic_read(&counter->mmap_count));
1454
1455         size = sizeof(struct perf_mmap_data);
1456         size += nr_pages * sizeof(void *);
1457
1458         data = kzalloc(size, GFP_KERNEL);
1459         if (!data)
1460                 goto fail;
1461
1462         data->user_page = (void *)get_zeroed_page(GFP_KERNEL);
1463         if (!data->user_page)
1464                 goto fail_user_page;
1465
1466         for (i = 0; i < nr_pages; i++) {
1467                 data->data_pages[i] = (void *)get_zeroed_page(GFP_KERNEL);
1468                 if (!data->data_pages[i])
1469                         goto fail_data_pages;
1470         }
1471
1472         data->nr_pages = nr_pages;
1473         atomic_set(&data->lock, -1);
1474
1475         rcu_assign_pointer(counter->data, data);
1476
1477         return 0;
1478
1479 fail_data_pages:
1480         for (i--; i >= 0; i--)
1481                 free_page((unsigned long)data->data_pages[i]);
1482
1483         free_page((unsigned long)data->user_page);
1484
1485 fail_user_page:
1486         kfree(data);
1487
1488 fail:
1489         return -ENOMEM;
1490 }
1491
1492 static void __perf_mmap_data_free(struct rcu_head *rcu_head)
1493 {
1494         struct perf_mmap_data *data = container_of(rcu_head,
1495                         struct perf_mmap_data, rcu_head);
1496         int i;
1497
1498         free_page((unsigned long)data->user_page);
1499         for (i = 0; i < data->nr_pages; i++)
1500                 free_page((unsigned long)data->data_pages[i]);
1501         kfree(data);
1502 }
1503
1504 static void perf_mmap_data_free(struct perf_counter *counter)
1505 {
1506         struct perf_mmap_data *data = counter->data;
1507
1508         WARN_ON(atomic_read(&counter->mmap_count));
1509
1510         rcu_assign_pointer(counter->data, NULL);
1511         call_rcu(&data->rcu_head, __perf_mmap_data_free);
1512 }
1513
1514 static void perf_mmap_open(struct vm_area_struct *vma)
1515 {
1516         struct perf_counter *counter = vma->vm_file->private_data;
1517
1518         atomic_inc(&counter->mmap_count);
1519 }
1520
1521 static void perf_mmap_close(struct vm_area_struct *vma)
1522 {
1523         struct perf_counter *counter = vma->vm_file->private_data;
1524
1525         if (atomic_dec_and_mutex_lock(&counter->mmap_count,
1526                                       &counter->mmap_mutex)) {
1527                 vma->vm_mm->locked_vm -= counter->data->nr_locked;
1528                 perf_mmap_data_free(counter);
1529                 mutex_unlock(&counter->mmap_mutex);
1530         }
1531 }
1532
1533 static struct vm_operations_struct perf_mmap_vmops = {
1534         .open  = perf_mmap_open,
1535         .close = perf_mmap_close,
1536         .fault = perf_mmap_fault,
1537 };
1538
1539 static int perf_mmap(struct file *file, struct vm_area_struct *vma)
1540 {
1541         struct perf_counter *counter = file->private_data;
1542         unsigned long vma_size;
1543         unsigned long nr_pages;
1544         unsigned long locked, lock_limit;
1545         int ret = 0;
1546         long extra;
1547
1548         if (!(vma->vm_flags & VM_SHARED) || (vma->vm_flags & VM_WRITE))
1549                 return -EINVAL;
1550
1551         vma_size = vma->vm_end - vma->vm_start;
1552         nr_pages = (vma_size / PAGE_SIZE) - 1;
1553
1554         /*
1555          * If we have data pages ensure they're a power-of-two number, so we
1556          * can do bitmasks instead of modulo.
1557          */
1558         if (nr_pages != 0 && !is_power_of_2(nr_pages))
1559                 return -EINVAL;
1560
1561         if (vma_size != PAGE_SIZE * (1 + nr_pages))
1562                 return -EINVAL;
1563
1564         if (vma->vm_pgoff != 0)
1565                 return -EINVAL;
1566
1567         mutex_lock(&counter->mmap_mutex);
1568         if (atomic_inc_not_zero(&counter->mmap_count)) {
1569                 if (nr_pages != counter->data->nr_pages)
1570                         ret = -EINVAL;
1571                 goto unlock;
1572         }
1573
1574         extra = nr_pages /* + 1 only account the data pages */;
1575         extra -= sysctl_perf_counter_mlock >> (PAGE_SHIFT - 10);
1576         if (extra < 0)
1577                 extra = 0;
1578
1579         locked = vma->vm_mm->locked_vm + extra;
1580
1581         lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur;
1582         lock_limit >>= PAGE_SHIFT;
1583
1584         if ((locked > lock_limit) && !capable(CAP_IPC_LOCK)) {
1585                 ret = -EPERM;
1586                 goto unlock;
1587         }
1588
1589         WARN_ON(counter->data);
1590         ret = perf_mmap_data_alloc(counter, nr_pages);
1591         if (ret)
1592                 goto unlock;
1593
1594         atomic_set(&counter->mmap_count, 1);
1595         vma->vm_mm->locked_vm += extra;
1596         counter->data->nr_locked = extra;
1597 unlock:
1598         mutex_unlock(&counter->mmap_mutex);
1599
1600         vma->vm_flags &= ~VM_MAYWRITE;
1601         vma->vm_flags |= VM_RESERVED;
1602         vma->vm_ops = &perf_mmap_vmops;
1603
1604         return ret;
1605 }
1606
1607 static int perf_fasync(int fd, struct file *filp, int on)
1608 {
1609         struct perf_counter *counter = filp->private_data;
1610         struct inode *inode = filp->f_path.dentry->d_inode;
1611         int retval;
1612
1613         mutex_lock(&inode->i_mutex);
1614         retval = fasync_helper(fd, filp, on, &counter->fasync);
1615         mutex_unlock(&inode->i_mutex);
1616
1617         if (retval < 0)
1618                 return retval;
1619
1620         return 0;
1621 }
1622
1623 static const struct file_operations perf_fops = {
1624         .release                = perf_release,
1625         .read                   = perf_read,
1626         .poll                   = perf_poll,
1627         .unlocked_ioctl         = perf_ioctl,
1628         .compat_ioctl           = perf_ioctl,
1629         .mmap                   = perf_mmap,
1630         .fasync                 = perf_fasync,
1631 };
1632
1633 /*
1634  * Perf counter wakeup
1635  *
1636  * If there's data, ensure we set the poll() state and publish everything
1637  * to user-space before waking everybody up.
1638  */
1639
1640 void perf_counter_wakeup(struct perf_counter *counter)
1641 {
1642         wake_up_all(&counter->waitq);
1643
1644         if (counter->pending_kill) {
1645                 kill_fasync(&counter->fasync, SIGIO, counter->pending_kill);
1646                 counter->pending_kill = 0;
1647         }
1648 }
1649
1650 /*
1651  * Pending wakeups
1652  *
1653  * Handle the case where we need to wakeup up from NMI (or rq->lock) context.
1654  *
1655  * The NMI bit means we cannot possibly take locks. Therefore, maintain a
1656  * single linked list and use cmpxchg() to add entries lockless.
1657  */
1658
1659 static void perf_pending_counter(struct perf_pending_entry *entry)
1660 {
1661         struct perf_counter *counter = container_of(entry,
1662                         struct perf_counter, pending);
1663
1664         if (counter->pending_disable) {
1665                 counter->pending_disable = 0;
1666                 perf_counter_disable(counter);
1667         }
1668
1669         if (counter->pending_wakeup) {
1670                 counter->pending_wakeup = 0;
1671                 perf_counter_wakeup(counter);
1672         }
1673 }
1674
1675 #define PENDING_TAIL ((struct perf_pending_entry *)-1UL)
1676
1677 static DEFINE_PER_CPU(struct perf_pending_entry *, perf_pending_head) = {
1678         PENDING_TAIL,
1679 };
1680
1681 static void perf_pending_queue(struct perf_pending_entry *entry,
1682                                void (*func)(struct perf_pending_entry *))
1683 {
1684         struct perf_pending_entry **head;
1685
1686         if (cmpxchg(&entry->next, NULL, PENDING_TAIL) != NULL)
1687                 return;
1688
1689         entry->func = func;
1690
1691         head = &get_cpu_var(perf_pending_head);
1692
1693         do {
1694                 entry->next = *head;
1695         } while (cmpxchg(head, entry->next, entry) != entry->next);
1696
1697         set_perf_counter_pending();
1698
1699         put_cpu_var(perf_pending_head);
1700 }
1701
1702 static int __perf_pending_run(void)
1703 {
1704         struct perf_pending_entry *list;
1705         int nr = 0;
1706
1707         list = xchg(&__get_cpu_var(perf_pending_head), PENDING_TAIL);
1708         while (list != PENDING_TAIL) {
1709                 void (*func)(struct perf_pending_entry *);
1710                 struct perf_pending_entry *entry = list;
1711
1712                 list = list->next;
1713
1714                 func = entry->func;
1715                 entry->next = NULL;
1716                 /*
1717                  * Ensure we observe the unqueue before we issue the wakeup,
1718                  * so that we won't be waiting forever.
1719                  * -- see perf_not_pending().
1720                  */
1721                 smp_wmb();
1722
1723                 func(entry);
1724                 nr++;
1725         }
1726
1727         return nr;
1728 }
1729
1730 static inline int perf_not_pending(struct perf_counter *counter)
1731 {
1732         /*
1733          * If we flush on whatever cpu we run, there is a chance we don't
1734          * need to wait.
1735          */
1736         get_cpu();
1737         __perf_pending_run();
1738         put_cpu();
1739
1740         /*
1741          * Ensure we see the proper queue state before going to sleep
1742          * so that we do not miss the wakeup. -- see perf_pending_handle()
1743          */
1744         smp_rmb();
1745         return counter->pending.next == NULL;
1746 }
1747
1748 static void perf_pending_sync(struct perf_counter *counter)
1749 {
1750         wait_event(counter->waitq, perf_not_pending(counter));
1751 }
1752
1753 void perf_counter_do_pending(void)
1754 {
1755         __perf_pending_run();
1756 }
1757
1758 /*
1759  * Callchain support -- arch specific
1760  */
1761
1762 __weak struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
1763 {
1764         return NULL;
1765 }
1766
1767 /*
1768  * Output
1769  */
1770
1771 struct perf_output_handle {
1772         struct perf_counter     *counter;
1773         struct perf_mmap_data   *data;
1774         unsigned int            offset;
1775         unsigned int            head;
1776         int                     nmi;
1777         int                     overflow;
1778         int                     locked;
1779         unsigned long           flags;
1780 };
1781
1782 static void perf_output_wakeup(struct perf_output_handle *handle)
1783 {
1784         atomic_set(&handle->data->poll, POLL_IN);
1785
1786         if (handle->nmi) {
1787                 handle->counter->pending_wakeup = 1;
1788                 perf_pending_queue(&handle->counter->pending,
1789                                    perf_pending_counter);
1790         } else
1791                 perf_counter_wakeup(handle->counter);
1792 }
1793
1794 /*
1795  * Curious locking construct.
1796  *
1797  * We need to ensure a later event doesn't publish a head when a former
1798  * event isn't done writing. However since we need to deal with NMIs we
1799  * cannot fully serialize things.
1800  *
1801  * What we do is serialize between CPUs so we only have to deal with NMI
1802  * nesting on a single CPU.
1803  *
1804  * We only publish the head (and generate a wakeup) when the outer-most
1805  * event completes.
1806  */
1807 static void perf_output_lock(struct perf_output_handle *handle)
1808 {
1809         struct perf_mmap_data *data = handle->data;
1810         int cpu;
1811
1812         handle->locked = 0;
1813
1814         local_irq_save(handle->flags);
1815         cpu = smp_processor_id();
1816
1817         if (in_nmi() && atomic_read(&data->lock) == cpu)
1818                 return;
1819
1820         while (atomic_cmpxchg(&data->lock, -1, cpu) != -1)
1821                 cpu_relax();
1822
1823         handle->locked = 1;
1824 }
1825
1826 static void perf_output_unlock(struct perf_output_handle *handle)
1827 {
1828         struct perf_mmap_data *data = handle->data;
1829         int head, cpu;
1830
1831         data->done_head = data->head;
1832
1833         if (!handle->locked)
1834                 goto out;
1835
1836 again:
1837         /*
1838          * The xchg implies a full barrier that ensures all writes are done
1839          * before we publish the new head, matched by a rmb() in userspace when
1840          * reading this position.
1841          */
1842         while ((head = atomic_xchg(&data->done_head, 0)))
1843                 data->user_page->data_head = head;
1844
1845         /*
1846          * NMI can happen here, which means we can miss a done_head update.
1847          */
1848
1849         cpu = atomic_xchg(&data->lock, -1);
1850         WARN_ON_ONCE(cpu != smp_processor_id());
1851
1852         /*
1853          * Therefore we have to validate we did not indeed do so.
1854          */
1855         if (unlikely(atomic_read(&data->done_head))) {
1856                 /*
1857                  * Since we had it locked, we can lock it again.
1858                  */
1859                 while (atomic_cmpxchg(&data->lock, -1, cpu) != -1)
1860                         cpu_relax();
1861
1862                 goto again;
1863         }
1864
1865         if (atomic_xchg(&data->wakeup, 0))
1866                 perf_output_wakeup(handle);
1867 out:
1868         local_irq_restore(handle->flags);
1869 }
1870
1871 static int perf_output_begin(struct perf_output_handle *handle,
1872                              struct perf_counter *counter, unsigned int size,
1873                              int nmi, int overflow)
1874 {
1875         struct perf_mmap_data *data;
1876         unsigned int offset, head;
1877
1878         /*
1879          * For inherited counters we send all the output towards the parent.
1880          */
1881         if (counter->parent)
1882                 counter = counter->parent;
1883
1884         rcu_read_lock();
1885         data = rcu_dereference(counter->data);
1886         if (!data)
1887                 goto out;
1888
1889         handle->data     = data;
1890         handle->counter  = counter;
1891         handle->nmi      = nmi;
1892         handle->overflow = overflow;
1893
1894         if (!data->nr_pages)
1895                 goto fail;
1896
1897         perf_output_lock(handle);
1898
1899         do {
1900                 offset = head = atomic_read(&data->head);
1901                 head += size;
1902         } while (atomic_cmpxchg(&data->head, offset, head) != offset);
1903
1904         handle->offset  = offset;
1905         handle->head    = head;
1906
1907         if ((offset >> PAGE_SHIFT) != (head >> PAGE_SHIFT))
1908                 atomic_set(&data->wakeup, 1);
1909
1910         return 0;
1911
1912 fail:
1913         perf_output_wakeup(handle);
1914 out:
1915         rcu_read_unlock();
1916
1917         return -ENOSPC;
1918 }
1919
1920 static void perf_output_copy(struct perf_output_handle *handle,
1921                              void *buf, unsigned int len)
1922 {
1923         unsigned int pages_mask;
1924         unsigned int offset;
1925         unsigned int size;
1926         void **pages;
1927
1928         offset          = handle->offset;
1929         pages_mask      = handle->data->nr_pages - 1;
1930         pages           = handle->data->data_pages;
1931
1932         do {
1933                 unsigned int page_offset;
1934                 int nr;
1935
1936                 nr          = (offset >> PAGE_SHIFT) & pages_mask;
1937                 page_offset = offset & (PAGE_SIZE - 1);
1938                 size        = min_t(unsigned int, PAGE_SIZE - page_offset, len);
1939
1940                 memcpy(pages[nr] + page_offset, buf, size);
1941
1942                 len         -= size;
1943                 buf         += size;
1944                 offset      += size;
1945         } while (len);
1946
1947         handle->offset = offset;
1948
1949         /*
1950          * Check we didn't copy past our reservation window, taking the
1951          * possible unsigned int wrap into account.
1952          */
1953         WARN_ON_ONCE(((int)(handle->head - handle->offset)) < 0);
1954 }
1955
1956 #define perf_output_put(handle, x) \
1957         perf_output_copy((handle), &(x), sizeof(x))
1958
1959 static void perf_output_end(struct perf_output_handle *handle)
1960 {
1961         struct perf_counter *counter = handle->counter;
1962         struct perf_mmap_data *data = handle->data;
1963
1964         int wakeup_events = counter->hw_event.wakeup_events;
1965
1966         if (handle->overflow && wakeup_events) {
1967                 int events = atomic_inc_return(&data->events);
1968                 if (events >= wakeup_events) {
1969                         atomic_sub(wakeup_events, &data->events);
1970                         atomic_set(&data->wakeup, 1);
1971                 }
1972         }
1973
1974         perf_output_unlock(handle);
1975         rcu_read_unlock();
1976 }
1977
1978 static void perf_counter_output(struct perf_counter *counter,
1979                                 int nmi, struct pt_regs *regs, u64 addr)
1980 {
1981         int ret;
1982         u64 record_type = counter->hw_event.record_type;
1983         struct perf_output_handle handle;
1984         struct perf_event_header header;
1985         u64 ip;
1986         struct {
1987                 u32 pid, tid;
1988         } tid_entry;
1989         struct {
1990                 u64 event;
1991                 u64 counter;
1992         } group_entry;
1993         struct perf_callchain_entry *callchain = NULL;
1994         int callchain_size = 0;
1995         u64 time;
1996         struct {
1997                 u32 cpu, reserved;
1998         } cpu_entry;
1999
2000         header.type = 0;
2001         header.size = sizeof(header);
2002
2003         header.misc = PERF_EVENT_MISC_OVERFLOW;
2004         header.misc |= user_mode(regs) ?
2005                 PERF_EVENT_MISC_USER : PERF_EVENT_MISC_KERNEL;
2006
2007         if (record_type & PERF_RECORD_IP) {
2008                 ip = instruction_pointer(regs);
2009                 header.type |= PERF_RECORD_IP;
2010                 header.size += sizeof(ip);
2011         }
2012
2013         if (record_type & PERF_RECORD_TID) {
2014                 /* namespace issues */
2015                 tid_entry.pid = current->group_leader->pid;
2016                 tid_entry.tid = current->pid;
2017
2018                 header.type |= PERF_RECORD_TID;
2019                 header.size += sizeof(tid_entry);
2020         }
2021
2022         if (record_type & PERF_RECORD_TIME) {
2023                 /*
2024                  * Maybe do better on x86 and provide cpu_clock_nmi()
2025                  */
2026                 time = sched_clock();
2027
2028                 header.type |= PERF_RECORD_TIME;
2029                 header.size += sizeof(u64);
2030         }
2031
2032         if (record_type & PERF_RECORD_ADDR) {
2033                 header.type |= PERF_RECORD_ADDR;
2034                 header.size += sizeof(u64);
2035         }
2036
2037         if (record_type & PERF_RECORD_CONFIG) {
2038                 header.type |= PERF_RECORD_CONFIG;
2039                 header.size += sizeof(u64);
2040         }
2041
2042         if (record_type & PERF_RECORD_CPU) {
2043                 header.type |= PERF_RECORD_CPU;
2044                 header.size += sizeof(cpu_entry);
2045
2046                 cpu_entry.cpu = raw_smp_processor_id();
2047         }
2048
2049         if (record_type & PERF_RECORD_GROUP) {
2050                 header.type |= PERF_RECORD_GROUP;
2051                 header.size += sizeof(u64) +
2052                         counter->nr_siblings * sizeof(group_entry);
2053         }
2054
2055         if (record_type & PERF_RECORD_CALLCHAIN) {
2056                 callchain = perf_callchain(regs);
2057
2058                 if (callchain) {
2059                         callchain_size = (1 + callchain->nr) * sizeof(u64);
2060
2061                         header.type |= PERF_RECORD_CALLCHAIN;
2062                         header.size += callchain_size;
2063                 }
2064         }
2065
2066         ret = perf_output_begin(&handle, counter, header.size, nmi, 1);
2067         if (ret)
2068                 return;
2069
2070         perf_output_put(&handle, header);
2071
2072         if (record_type & PERF_RECORD_IP)
2073                 perf_output_put(&handle, ip);
2074
2075         if (record_type & PERF_RECORD_TID)
2076                 perf_output_put(&handle, tid_entry);
2077
2078         if (record_type & PERF_RECORD_TIME)
2079                 perf_output_put(&handle, time);
2080
2081         if (record_type & PERF_RECORD_ADDR)
2082                 perf_output_put(&handle, addr);
2083
2084         if (record_type & PERF_RECORD_CONFIG)
2085                 perf_output_put(&handle, counter->hw_event.config);
2086
2087         if (record_type & PERF_RECORD_CPU)
2088                 perf_output_put(&handle, cpu_entry);
2089
2090         /*
2091          * XXX PERF_RECORD_GROUP vs inherited counters seems difficult.
2092          */
2093         if (record_type & PERF_RECORD_GROUP) {
2094                 struct perf_counter *leader, *sub;
2095                 u64 nr = counter->nr_siblings;
2096
2097                 perf_output_put(&handle, nr);
2098
2099                 leader = counter->group_leader;
2100                 list_for_each_entry(sub, &leader->sibling_list, list_entry) {
2101                         if (sub != counter)
2102                                 sub->pmu->read(sub);
2103
2104                         group_entry.event = sub->hw_event.config;
2105                         group_entry.counter = atomic64_read(&sub->count);
2106
2107                         perf_output_put(&handle, group_entry);
2108                 }
2109         }
2110
2111         if (callchain)
2112                 perf_output_copy(&handle, callchain, callchain_size);
2113
2114         perf_output_end(&handle);
2115 }
2116
2117 /*
2118  * comm tracking
2119  */
2120
2121 struct perf_comm_event {
2122         struct task_struct      *task;
2123         char                    *comm;
2124         int                     comm_size;
2125
2126         struct {
2127                 struct perf_event_header        header;
2128
2129                 u32                             pid;
2130                 u32                             tid;
2131         } event;
2132 };
2133
2134 static void perf_counter_comm_output(struct perf_counter *counter,
2135                                      struct perf_comm_event *comm_event)
2136 {
2137         struct perf_output_handle handle;
2138         int size = comm_event->event.header.size;
2139         int ret = perf_output_begin(&handle, counter, size, 0, 0);
2140
2141         if (ret)
2142                 return;
2143
2144         perf_output_put(&handle, comm_event->event);
2145         perf_output_copy(&handle, comm_event->comm,
2146                                    comm_event->comm_size);
2147         perf_output_end(&handle);
2148 }
2149
2150 static int perf_counter_comm_match(struct perf_counter *counter,
2151                                    struct perf_comm_event *comm_event)
2152 {
2153         if (counter->hw_event.comm &&
2154             comm_event->event.header.type == PERF_EVENT_COMM)
2155                 return 1;
2156
2157         return 0;
2158 }
2159
2160 static void perf_counter_comm_ctx(struct perf_counter_context *ctx,
2161                                   struct perf_comm_event *comm_event)
2162 {
2163         struct perf_counter *counter;
2164
2165         if (system_state != SYSTEM_RUNNING || list_empty(&ctx->event_list))
2166                 return;
2167
2168         rcu_read_lock();
2169         list_for_each_entry_rcu(counter, &ctx->event_list, event_entry) {
2170                 if (perf_counter_comm_match(counter, comm_event))
2171                         perf_counter_comm_output(counter, comm_event);
2172         }
2173         rcu_read_unlock();
2174 }
2175
2176 static void perf_counter_comm_event(struct perf_comm_event *comm_event)
2177 {
2178         struct perf_cpu_context *cpuctx;
2179         unsigned int size;
2180         char *comm = comm_event->task->comm;
2181
2182         size = ALIGN(strlen(comm)+1, sizeof(u64));
2183
2184         comm_event->comm = comm;
2185         comm_event->comm_size = size;
2186
2187         comm_event->event.header.size = sizeof(comm_event->event) + size;
2188
2189         cpuctx = &get_cpu_var(perf_cpu_context);
2190         perf_counter_comm_ctx(&cpuctx->ctx, comm_event);
2191         put_cpu_var(perf_cpu_context);
2192
2193         perf_counter_comm_ctx(&current->perf_counter_ctx, comm_event);
2194 }
2195
2196 void perf_counter_comm(struct task_struct *task)
2197 {
2198         struct perf_comm_event comm_event;
2199
2200         if (!atomic_read(&nr_comm_tracking))
2201                 return;
2202
2203         comm_event = (struct perf_comm_event){
2204                 .task   = task,
2205                 .event  = {
2206                         .header = { .type = PERF_EVENT_COMM, },
2207                         .pid    = task->group_leader->pid,
2208                         .tid    = task->pid,
2209                 },
2210         };
2211
2212         perf_counter_comm_event(&comm_event);
2213 }
2214
2215 /*
2216  * mmap tracking
2217  */
2218
2219 struct perf_mmap_event {
2220         struct file     *file;
2221         char            *file_name;
2222         int             file_size;
2223
2224         struct {
2225                 struct perf_event_header        header;
2226
2227                 u32                             pid;
2228                 u32                             tid;
2229                 u64                             start;
2230                 u64                             len;
2231                 u64                             pgoff;
2232         } event;
2233 };
2234
2235 static void perf_counter_mmap_output(struct perf_counter *counter,
2236                                      struct perf_mmap_event *mmap_event)
2237 {
2238         struct perf_output_handle handle;
2239         int size = mmap_event->event.header.size;
2240         int ret = perf_output_begin(&handle, counter, size, 0, 0);
2241
2242         if (ret)
2243                 return;
2244
2245         perf_output_put(&handle, mmap_event->event);
2246         perf_output_copy(&handle, mmap_event->file_name,
2247                                    mmap_event->file_size);
2248         perf_output_end(&handle);
2249 }
2250
2251 static int perf_counter_mmap_match(struct perf_counter *counter,
2252                                    struct perf_mmap_event *mmap_event)
2253 {
2254         if (counter->hw_event.mmap &&
2255             mmap_event->event.header.type == PERF_EVENT_MMAP)
2256                 return 1;
2257
2258         if (counter->hw_event.munmap &&
2259             mmap_event->event.header.type == PERF_EVENT_MUNMAP)
2260                 return 1;
2261
2262         return 0;
2263 }
2264
2265 static void perf_counter_mmap_ctx(struct perf_counter_context *ctx,
2266                                   struct perf_mmap_event *mmap_event)
2267 {
2268         struct perf_counter *counter;
2269
2270         if (system_state != SYSTEM_RUNNING || list_empty(&ctx->event_list))
2271                 return;
2272
2273         rcu_read_lock();
2274         list_for_each_entry_rcu(counter, &ctx->event_list, event_entry) {
2275                 if (perf_counter_mmap_match(counter, mmap_event))
2276                         perf_counter_mmap_output(counter, mmap_event);
2277         }
2278         rcu_read_unlock();
2279 }
2280
2281 static void perf_counter_mmap_event(struct perf_mmap_event *mmap_event)
2282 {
2283         struct perf_cpu_context *cpuctx;
2284         struct file *file = mmap_event->file;
2285         unsigned int size;
2286         char tmp[16];
2287         char *buf = NULL;
2288         char *name;
2289
2290         if (file) {
2291                 buf = kzalloc(PATH_MAX, GFP_KERNEL);
2292                 if (!buf) {
2293                         name = strncpy(tmp, "//enomem", sizeof(tmp));
2294                         goto got_name;
2295                 }
2296                 name = d_path(&file->f_path, buf, PATH_MAX);
2297                 if (IS_ERR(name)) {
2298                         name = strncpy(tmp, "//toolong", sizeof(tmp));
2299                         goto got_name;
2300                 }
2301         } else {
2302                 name = strncpy(tmp, "//anon", sizeof(tmp));
2303                 goto got_name;
2304         }
2305
2306 got_name:
2307         size = ALIGN(strlen(name)+1, sizeof(u64));
2308
2309         mmap_event->file_name = name;
2310         mmap_event->file_size = size;
2311
2312         mmap_event->event.header.size = sizeof(mmap_event->event) + size;
2313
2314         cpuctx = &get_cpu_var(perf_cpu_context);
2315         perf_counter_mmap_ctx(&cpuctx->ctx, mmap_event);
2316         put_cpu_var(perf_cpu_context);
2317
2318         perf_counter_mmap_ctx(&current->perf_counter_ctx, mmap_event);
2319
2320         kfree(buf);
2321 }
2322
2323 void perf_counter_mmap(unsigned long addr, unsigned long len,
2324                        unsigned long pgoff, struct file *file)
2325 {
2326         struct perf_mmap_event mmap_event;
2327
2328         if (!atomic_read(&nr_mmap_tracking))
2329                 return;
2330
2331         mmap_event = (struct perf_mmap_event){
2332                 .file   = file,
2333                 .event  = {
2334                         .header = { .type = PERF_EVENT_MMAP, },
2335                         .pid    = current->group_leader->pid,
2336                         .tid    = current->pid,
2337                         .start  = addr,
2338                         .len    = len,
2339                         .pgoff  = pgoff,
2340                 },
2341         };
2342
2343         perf_counter_mmap_event(&mmap_event);
2344 }
2345
2346 void perf_counter_munmap(unsigned long addr, unsigned long len,
2347                          unsigned long pgoff, struct file *file)
2348 {
2349         struct perf_mmap_event mmap_event;
2350
2351         if (!atomic_read(&nr_munmap_tracking))
2352                 return;
2353
2354         mmap_event = (struct perf_mmap_event){
2355                 .file   = file,
2356                 .event  = {
2357                         .header = { .type = PERF_EVENT_MUNMAP, },
2358                         .pid    = current->group_leader->pid,
2359                         .tid    = current->pid,
2360                         .start  = addr,
2361                         .len    = len,
2362                         .pgoff  = pgoff,
2363                 },
2364         };
2365
2366         perf_counter_mmap_event(&mmap_event);
2367 }
2368
2369 /*
2370  * Generic counter overflow handling.
2371  */
2372
2373 int perf_counter_overflow(struct perf_counter *counter,
2374                           int nmi, struct pt_regs *regs, u64 addr)
2375 {
2376         int events = atomic_read(&counter->event_limit);
2377         int ret = 0;
2378
2379         /*
2380          * XXX event_limit might not quite work as expected on inherited
2381          * counters
2382          */
2383
2384         counter->pending_kill = POLL_IN;
2385         if (events && atomic_dec_and_test(&counter->event_limit)) {
2386                 ret = 1;
2387                 counter->pending_kill = POLL_HUP;
2388                 if (nmi) {
2389                         counter->pending_disable = 1;
2390                         perf_pending_queue(&counter->pending,
2391                                            perf_pending_counter);
2392                 } else
2393                         perf_counter_disable(counter);
2394         }
2395
2396         perf_counter_output(counter, nmi, regs, addr);
2397         return ret;
2398 }
2399
2400 /*
2401  * Generic software counter infrastructure
2402  */
2403
2404 static void perf_swcounter_update(struct perf_counter *counter)
2405 {
2406         struct hw_perf_counter *hwc = &counter->hw;
2407         u64 prev, now;
2408         s64 delta;
2409
2410 again:
2411         prev = atomic64_read(&hwc->prev_count);
2412         now = atomic64_read(&hwc->count);
2413         if (atomic64_cmpxchg(&hwc->prev_count, prev, now) != prev)
2414                 goto again;
2415
2416         delta = now - prev;
2417
2418         atomic64_add(delta, &counter->count);
2419         atomic64_sub(delta, &hwc->period_left);
2420 }
2421
2422 static void perf_swcounter_set_period(struct perf_counter *counter)
2423 {
2424         struct hw_perf_counter *hwc = &counter->hw;
2425         s64 left = atomic64_read(&hwc->period_left);
2426         s64 period = hwc->irq_period;
2427
2428         if (unlikely(left <= -period)) {
2429                 left = period;
2430                 atomic64_set(&hwc->period_left, left);
2431         }
2432
2433         if (unlikely(left <= 0)) {
2434                 left += period;
2435                 atomic64_add(period, &hwc->period_left);
2436         }
2437
2438         atomic64_set(&hwc->prev_count, -left);
2439         atomic64_set(&hwc->count, -left);
2440 }
2441
2442 static enum hrtimer_restart perf_swcounter_hrtimer(struct hrtimer *hrtimer)
2443 {
2444         enum hrtimer_restart ret = HRTIMER_RESTART;
2445         struct perf_counter *counter;
2446         struct pt_regs *regs;
2447
2448         counter = container_of(hrtimer, struct perf_counter, hw.hrtimer);
2449         counter->pmu->read(counter);
2450
2451         regs = get_irq_regs();
2452         /*
2453          * In case we exclude kernel IPs or are somehow not in interrupt
2454          * context, provide the next best thing, the user IP.
2455          */
2456         if ((counter->hw_event.exclude_kernel || !regs) &&
2457                         !counter->hw_event.exclude_user)
2458                 regs = task_pt_regs(current);
2459
2460         if (regs) {
2461                 if (perf_counter_overflow(counter, 0, regs, 0))
2462                         ret = HRTIMER_NORESTART;
2463         }
2464
2465         hrtimer_forward_now(hrtimer, ns_to_ktime(counter->hw.irq_period));
2466
2467         return ret;
2468 }
2469
2470 static void perf_swcounter_overflow(struct perf_counter *counter,
2471                                     int nmi, struct pt_regs *regs, u64 addr)
2472 {
2473         perf_swcounter_update(counter);
2474         perf_swcounter_set_period(counter);
2475         if (perf_counter_overflow(counter, nmi, regs, addr))
2476                 /* soft-disable the counter */
2477                 ;
2478
2479 }
2480
2481 static int perf_swcounter_match(struct perf_counter *counter,
2482                                 enum perf_event_types type,
2483                                 u32 event, struct pt_regs *regs)
2484 {
2485         if (counter->state != PERF_COUNTER_STATE_ACTIVE)
2486                 return 0;
2487
2488         if (perf_event_raw(&counter->hw_event))
2489                 return 0;
2490
2491         if (perf_event_type(&counter->hw_event) != type)
2492                 return 0;
2493
2494         if (perf_event_id(&counter->hw_event) != event)
2495                 return 0;
2496
2497         if (counter->hw_event.exclude_user && user_mode(regs))
2498                 return 0;
2499
2500         if (counter->hw_event.exclude_kernel && !user_mode(regs))
2501                 return 0;
2502
2503         return 1;
2504 }
2505
2506 static void perf_swcounter_add(struct perf_counter *counter, u64 nr,
2507                                int nmi, struct pt_regs *regs, u64 addr)
2508 {
2509         int neg = atomic64_add_negative(nr, &counter->hw.count);
2510         if (counter->hw.irq_period && !neg)
2511                 perf_swcounter_overflow(counter, nmi, regs, addr);
2512 }
2513
2514 static void perf_swcounter_ctx_event(struct perf_counter_context *ctx,
2515                                      enum perf_event_types type, u32 event,
2516                                      u64 nr, int nmi, struct pt_regs *regs,
2517                                      u64 addr)
2518 {
2519         struct perf_counter *counter;
2520
2521         if (system_state != SYSTEM_RUNNING || list_empty(&ctx->event_list))
2522                 return;
2523
2524         rcu_read_lock();
2525         list_for_each_entry_rcu(counter, &ctx->event_list, event_entry) {
2526                 if (perf_swcounter_match(counter, type, event, regs))
2527                         perf_swcounter_add(counter, nr, nmi, regs, addr);
2528         }
2529         rcu_read_unlock();
2530 }
2531
2532 static int *perf_swcounter_recursion_context(struct perf_cpu_context *cpuctx)
2533 {
2534         if (in_nmi())
2535                 return &cpuctx->recursion[3];
2536
2537         if (in_irq())
2538                 return &cpuctx->recursion[2];
2539
2540         if (in_softirq())
2541                 return &cpuctx->recursion[1];
2542
2543         return &cpuctx->recursion[0];
2544 }
2545
2546 static void __perf_swcounter_event(enum perf_event_types type, u32 event,
2547                                    u64 nr, int nmi, struct pt_regs *regs,
2548                                    u64 addr)
2549 {
2550         struct perf_cpu_context *cpuctx = &get_cpu_var(perf_cpu_context);
2551         int *recursion = perf_swcounter_recursion_context(cpuctx);
2552
2553         if (*recursion)
2554                 goto out;
2555
2556         (*recursion)++;
2557         barrier();
2558
2559         perf_swcounter_ctx_event(&cpuctx->ctx, type, event,
2560                                  nr, nmi, regs, addr);
2561         if (cpuctx->task_ctx) {
2562                 perf_swcounter_ctx_event(cpuctx->task_ctx, type, event,
2563                                          nr, nmi, regs, addr);
2564         }
2565
2566         barrier();
2567         (*recursion)--;
2568
2569 out:
2570         put_cpu_var(perf_cpu_context);
2571 }
2572
2573 void
2574 perf_swcounter_event(u32 event, u64 nr, int nmi, struct pt_regs *regs, u64 addr)
2575 {
2576         __perf_swcounter_event(PERF_TYPE_SOFTWARE, event, nr, nmi, regs, addr);
2577 }
2578
2579 static void perf_swcounter_read(struct perf_counter *counter)
2580 {
2581         perf_swcounter_update(counter);
2582 }
2583
2584 static int perf_swcounter_enable(struct perf_counter *counter)
2585 {
2586         perf_swcounter_set_period(counter);
2587         return 0;
2588 }
2589
2590 static void perf_swcounter_disable(struct perf_counter *counter)
2591 {
2592         perf_swcounter_update(counter);
2593 }
2594
2595 static const struct pmu perf_ops_generic = {
2596         .enable         = perf_swcounter_enable,
2597         .disable        = perf_swcounter_disable,
2598         .read           = perf_swcounter_read,
2599 };
2600
2601 /*
2602  * Software counter: cpu wall time clock
2603  */
2604
2605 static void cpu_clock_perf_counter_update(struct perf_counter *counter)
2606 {
2607         int cpu = raw_smp_processor_id();
2608         s64 prev;
2609         u64 now;
2610
2611         now = cpu_clock(cpu);
2612         prev = atomic64_read(&counter->hw.prev_count);
2613         atomic64_set(&counter->hw.prev_count, now);
2614         atomic64_add(now - prev, &counter->count);
2615 }
2616
2617 static int cpu_clock_perf_counter_enable(struct perf_counter *counter)
2618 {
2619         struct hw_perf_counter *hwc = &counter->hw;
2620         int cpu = raw_smp_processor_id();
2621
2622         atomic64_set(&hwc->prev_count, cpu_clock(cpu));
2623         hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
2624         hwc->hrtimer.function = perf_swcounter_hrtimer;
2625         if (hwc->irq_period) {
2626                 __hrtimer_start_range_ns(&hwc->hrtimer,
2627                                 ns_to_ktime(hwc->irq_period), 0,
2628                                 HRTIMER_MODE_REL, 0);
2629         }
2630
2631         return 0;
2632 }
2633
2634 static void cpu_clock_perf_counter_disable(struct perf_counter *counter)
2635 {
2636         hrtimer_cancel(&counter->hw.hrtimer);
2637         cpu_clock_perf_counter_update(counter);
2638 }
2639
2640 static void cpu_clock_perf_counter_read(struct perf_counter *counter)
2641 {
2642         cpu_clock_perf_counter_update(counter);
2643 }
2644
2645 static const struct pmu perf_ops_cpu_clock = {
2646         .enable         = cpu_clock_perf_counter_enable,
2647         .disable        = cpu_clock_perf_counter_disable,
2648         .read           = cpu_clock_perf_counter_read,
2649 };
2650
2651 /*
2652  * Software counter: task time clock
2653  */
2654
2655 static void task_clock_perf_counter_update(struct perf_counter *counter, u64 now)
2656 {
2657         u64 prev;
2658         s64 delta;
2659
2660         prev = atomic64_xchg(&counter->hw.prev_count, now);
2661         delta = now - prev;
2662         atomic64_add(delta, &counter->count);
2663 }
2664
2665 static int task_clock_perf_counter_enable(struct perf_counter *counter)
2666 {
2667         struct hw_perf_counter *hwc = &counter->hw;
2668         u64 now;
2669
2670         now = counter->ctx->time;
2671
2672         atomic64_set(&hwc->prev_count, now);
2673         hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
2674         hwc->hrtimer.function = perf_swcounter_hrtimer;
2675         if (hwc->irq_period) {
2676                 __hrtimer_start_range_ns(&hwc->hrtimer,
2677                                 ns_to_ktime(hwc->irq_period), 0,
2678                                 HRTIMER_MODE_REL, 0);
2679         }
2680
2681         return 0;
2682 }
2683
2684 static void task_clock_perf_counter_disable(struct perf_counter *counter)
2685 {
2686         hrtimer_cancel(&counter->hw.hrtimer);
2687         task_clock_perf_counter_update(counter, counter->ctx->time);
2688
2689 }
2690
2691 static void task_clock_perf_counter_read(struct perf_counter *counter)
2692 {
2693         u64 time;
2694
2695         if (!in_nmi()) {
2696                 update_context_time(counter->ctx);
2697                 time = counter->ctx->time;
2698         } else {
2699                 u64 now = perf_clock();
2700                 u64 delta = now - counter->ctx->timestamp;
2701                 time = counter->ctx->time + delta;
2702         }
2703
2704         task_clock_perf_counter_update(counter, time);
2705 }
2706
2707 static const struct pmu perf_ops_task_clock = {
2708         .enable         = task_clock_perf_counter_enable,
2709         .disable        = task_clock_perf_counter_disable,
2710         .read           = task_clock_perf_counter_read,
2711 };
2712
2713 /*
2714  * Software counter: cpu migrations
2715  */
2716
2717 static inline u64 get_cpu_migrations(struct perf_counter *counter)
2718 {
2719         struct task_struct *curr = counter->ctx->task;
2720
2721         if (curr)
2722                 return curr->se.nr_migrations;
2723         return cpu_nr_migrations(smp_processor_id());
2724 }
2725
2726 static void cpu_migrations_perf_counter_update(struct perf_counter *counter)
2727 {
2728         u64 prev, now;
2729         s64 delta;
2730
2731         prev = atomic64_read(&counter->hw.prev_count);
2732         now = get_cpu_migrations(counter);
2733
2734         atomic64_set(&counter->hw.prev_count, now);
2735
2736         delta = now - prev;
2737
2738         atomic64_add(delta, &counter->count);
2739 }
2740
2741 static void cpu_migrations_perf_counter_read(struct perf_counter *counter)
2742 {
2743         cpu_migrations_perf_counter_update(counter);
2744 }
2745
2746 static int cpu_migrations_perf_counter_enable(struct perf_counter *counter)
2747 {
2748         if (counter->prev_state <= PERF_COUNTER_STATE_OFF)
2749                 atomic64_set(&counter->hw.prev_count,
2750                              get_cpu_migrations(counter));
2751         return 0;
2752 }
2753
2754 static void cpu_migrations_perf_counter_disable(struct perf_counter *counter)
2755 {
2756         cpu_migrations_perf_counter_update(counter);
2757 }
2758
2759 static const struct pmu perf_ops_cpu_migrations = {
2760         .enable         = cpu_migrations_perf_counter_enable,
2761         .disable        = cpu_migrations_perf_counter_disable,
2762         .read           = cpu_migrations_perf_counter_read,
2763 };
2764
2765 #ifdef CONFIG_EVENT_PROFILE
2766 void perf_tpcounter_event(int event_id)
2767 {
2768         struct pt_regs *regs = get_irq_regs();
2769
2770         if (!regs)
2771                 regs = task_pt_regs(current);
2772
2773         __perf_swcounter_event(PERF_TYPE_TRACEPOINT, event_id, 1, 1, regs, 0);
2774 }
2775 EXPORT_SYMBOL_GPL(perf_tpcounter_event);
2776
2777 extern int ftrace_profile_enable(int);
2778 extern void ftrace_profile_disable(int);
2779
2780 static void tp_perf_counter_destroy(struct perf_counter *counter)
2781 {
2782         ftrace_profile_disable(perf_event_id(&counter->hw_event));
2783 }
2784
2785 static const struct pmu *tp_perf_counter_init(struct perf_counter *counter)
2786 {
2787         int event_id = perf_event_id(&counter->hw_event);
2788         int ret;
2789
2790         ret = ftrace_profile_enable(event_id);
2791         if (ret)
2792                 return NULL;
2793
2794         counter->destroy = tp_perf_counter_destroy;
2795         counter->hw.irq_period = counter->hw_event.irq_period;
2796
2797         return &perf_ops_generic;
2798 }
2799 #else
2800 static const struct pmu *tp_perf_counter_init(struct perf_counter *counter)
2801 {
2802         return NULL;
2803 }
2804 #endif
2805
2806 static const struct pmu *sw_perf_counter_init(struct perf_counter *counter)
2807 {
2808         struct perf_counter_hw_event *hw_event = &counter->hw_event;
2809         const struct pmu *pmu = NULL;
2810         struct hw_perf_counter *hwc = &counter->hw;
2811
2812         /*
2813          * Software counters (currently) can't in general distinguish
2814          * between user, kernel and hypervisor events.
2815          * However, context switches and cpu migrations are considered
2816          * to be kernel events, and page faults are never hypervisor
2817          * events.
2818          */
2819         switch (perf_event_id(&counter->hw_event)) {
2820         case PERF_COUNT_CPU_CLOCK:
2821                 pmu = &perf_ops_cpu_clock;
2822
2823                 if (hw_event->irq_period && hw_event->irq_period < 10000)
2824                         hw_event->irq_period = 10000;
2825                 break;
2826         case PERF_COUNT_TASK_CLOCK:
2827                 /*
2828                  * If the user instantiates this as a per-cpu counter,
2829                  * use the cpu_clock counter instead.
2830                  */
2831                 if (counter->ctx->task)
2832                         pmu = &perf_ops_task_clock;
2833                 else
2834                         pmu = &perf_ops_cpu_clock;
2835
2836                 if (hw_event->irq_period && hw_event->irq_period < 10000)
2837                         hw_event->irq_period = 10000;
2838                 break;
2839         case PERF_COUNT_PAGE_FAULTS:
2840         case PERF_COUNT_PAGE_FAULTS_MIN:
2841         case PERF_COUNT_PAGE_FAULTS_MAJ:
2842         case PERF_COUNT_CONTEXT_SWITCHES:
2843                 pmu = &perf_ops_generic;
2844                 break;
2845         case PERF_COUNT_CPU_MIGRATIONS:
2846                 if (!counter->hw_event.exclude_kernel)
2847                         pmu = &perf_ops_cpu_migrations;
2848                 break;
2849         }
2850
2851         if (pmu)
2852                 hwc->irq_period = hw_event->irq_period;
2853
2854         return pmu;
2855 }
2856
2857 /*
2858  * Allocate and initialize a counter structure
2859  */
2860 static struct perf_counter *
2861 perf_counter_alloc(struct perf_counter_hw_event *hw_event,
2862                    int cpu,
2863                    struct perf_counter_context *ctx,
2864                    struct perf_counter *group_leader,
2865                    gfp_t gfpflags)
2866 {
2867         const struct pmu *pmu;
2868         struct perf_counter *counter;
2869         long err;
2870
2871         counter = kzalloc(sizeof(*counter), gfpflags);
2872         if (!counter)
2873                 return ERR_PTR(-ENOMEM);
2874
2875         /*
2876          * Single counters are their own group leaders, with an
2877          * empty sibling list:
2878          */
2879         if (!group_leader)
2880                 group_leader = counter;
2881
2882         mutex_init(&counter->mutex);
2883         INIT_LIST_HEAD(&counter->list_entry);
2884         INIT_LIST_HEAD(&counter->event_entry);
2885         INIT_LIST_HEAD(&counter->sibling_list);
2886         init_waitqueue_head(&counter->waitq);
2887
2888         mutex_init(&counter->mmap_mutex);
2889
2890         INIT_LIST_HEAD(&counter->child_list);
2891
2892         counter->cpu                    = cpu;
2893         counter->hw_event               = *hw_event;
2894         counter->group_leader           = group_leader;
2895         counter->pmu                    = NULL;
2896         counter->ctx                    = ctx;
2897
2898         counter->state = PERF_COUNTER_STATE_INACTIVE;
2899         if (hw_event->disabled)
2900                 counter->state = PERF_COUNTER_STATE_OFF;
2901
2902         pmu = NULL;
2903
2904         /*
2905          * we currently do not support PERF_RECORD_GROUP on inherited counters
2906          */
2907         if (hw_event->inherit && (hw_event->record_type & PERF_RECORD_GROUP))
2908                 goto done;
2909
2910         if (perf_event_raw(hw_event)) {
2911                 pmu = hw_perf_counter_init(counter);
2912                 goto done;
2913         }
2914
2915         switch (perf_event_type(hw_event)) {
2916         case PERF_TYPE_HARDWARE:
2917                 pmu = hw_perf_counter_init(counter);
2918                 break;
2919
2920         case PERF_TYPE_SOFTWARE:
2921                 pmu = sw_perf_counter_init(counter);
2922                 break;
2923
2924         case PERF_TYPE_TRACEPOINT:
2925                 pmu = tp_perf_counter_init(counter);
2926                 break;
2927         }
2928 done:
2929         err = 0;
2930         if (!pmu)
2931                 err = -EINVAL;
2932         else if (IS_ERR(pmu))
2933                 err = PTR_ERR(pmu);
2934
2935         if (err) {
2936                 kfree(counter);
2937                 return ERR_PTR(err);
2938         }
2939
2940         counter->pmu = pmu;
2941
2942         atomic_inc(&nr_counters);
2943         if (counter->hw_event.mmap)
2944                 atomic_inc(&nr_mmap_tracking);
2945         if (counter->hw_event.munmap)
2946                 atomic_inc(&nr_munmap_tracking);
2947         if (counter->hw_event.comm)
2948                 atomic_inc(&nr_comm_tracking);
2949
2950         return counter;
2951 }
2952
2953 /**
2954  * sys_perf_counter_open - open a performance counter, associate it to a task/cpu
2955  *
2956  * @hw_event_uptr:      event type attributes for monitoring/sampling
2957  * @pid:                target pid
2958  * @cpu:                target cpu
2959  * @group_fd:           group leader counter fd
2960  */
2961 SYSCALL_DEFINE5(perf_counter_open,
2962                 const struct perf_counter_hw_event __user *, hw_event_uptr,
2963                 pid_t, pid, int, cpu, int, group_fd, unsigned long, flags)
2964 {
2965         struct perf_counter *counter, *group_leader;
2966         struct perf_counter_hw_event hw_event;
2967         struct perf_counter_context *ctx;
2968         struct file *counter_file = NULL;
2969         struct file *group_file = NULL;
2970         int fput_needed = 0;
2971         int fput_needed2 = 0;
2972         int ret;
2973
2974         /* for future expandability... */
2975         if (flags)
2976                 return -EINVAL;
2977
2978         if (copy_from_user(&hw_event, hw_event_uptr, sizeof(hw_event)) != 0)
2979                 return -EFAULT;
2980
2981         /*
2982          * Get the target context (task or percpu):
2983          */
2984         ctx = find_get_context(pid, cpu);
2985         if (IS_ERR(ctx))
2986                 return PTR_ERR(ctx);
2987
2988         /*
2989          * Look up the group leader (we will attach this counter to it):
2990          */
2991         group_leader = NULL;
2992         if (group_fd != -1) {
2993                 ret = -EINVAL;
2994                 group_file = fget_light(group_fd, &fput_needed);
2995                 if (!group_file)
2996                         goto err_put_context;
2997                 if (group_file->f_op != &perf_fops)
2998                         goto err_put_context;
2999
3000                 group_leader = group_file->private_data;
3001                 /*
3002                  * Do not allow a recursive hierarchy (this new sibling
3003                  * becoming part of another group-sibling):
3004                  */
3005                 if (group_leader->group_leader != group_leader)
3006                         goto err_put_context;
3007                 /*
3008                  * Do not allow to attach to a group in a different
3009                  * task or CPU context:
3010                  */
3011                 if (group_leader->ctx != ctx)
3012                         goto err_put_context;
3013                 /*
3014                  * Only a group leader can be exclusive or pinned
3015                  */
3016                 if (hw_event.exclusive || hw_event.pinned)
3017                         goto err_put_context;
3018         }
3019
3020         counter = perf_counter_alloc(&hw_event, cpu, ctx, group_leader,
3021                                      GFP_KERNEL);
3022         ret = PTR_ERR(counter);
3023         if (IS_ERR(counter))
3024                 goto err_put_context;
3025
3026         ret = anon_inode_getfd("[perf_counter]", &perf_fops, counter, 0);
3027         if (ret < 0)
3028                 goto err_free_put_context;
3029
3030         counter_file = fget_light(ret, &fput_needed2);
3031         if (!counter_file)
3032                 goto err_free_put_context;
3033
3034         counter->filp = counter_file;
3035         mutex_lock(&ctx->mutex);
3036         perf_install_in_context(ctx, counter, cpu);
3037         mutex_unlock(&ctx->mutex);
3038
3039         fput_light(counter_file, fput_needed2);
3040
3041 out_fput:
3042         fput_light(group_file, fput_needed);
3043
3044         return ret;
3045
3046 err_free_put_context:
3047         kfree(counter);
3048
3049 err_put_context:
3050         put_context(ctx);
3051
3052         goto out_fput;
3053 }
3054
3055 /*
3056  * Initialize the perf_counter context in a task_struct:
3057  */
3058 static void
3059 __perf_counter_init_context(struct perf_counter_context *ctx,
3060                             struct task_struct *task)
3061 {
3062         memset(ctx, 0, sizeof(*ctx));
3063         spin_lock_init(&ctx->lock);
3064         mutex_init(&ctx->mutex);
3065         INIT_LIST_HEAD(&ctx->counter_list);
3066         INIT_LIST_HEAD(&ctx->event_list);
3067         ctx->task = task;
3068 }
3069
3070 /*
3071  * inherit a counter from parent task to child task:
3072  */
3073 static struct perf_counter *
3074 inherit_counter(struct perf_counter *parent_counter,
3075               struct task_struct *parent,
3076               struct perf_counter_context *parent_ctx,
3077               struct task_struct *child,
3078               struct perf_counter *group_leader,
3079               struct perf_counter_context *child_ctx)
3080 {
3081         struct perf_counter *child_counter;
3082
3083         /*
3084          * Instead of creating recursive hierarchies of counters,
3085          * we link inherited counters back to the original parent,
3086          * which has a filp for sure, which we use as the reference
3087          * count:
3088          */
3089         if (parent_counter->parent)
3090                 parent_counter = parent_counter->parent;
3091
3092         child_counter = perf_counter_alloc(&parent_counter->hw_event,
3093                                            parent_counter->cpu, child_ctx,
3094                                            group_leader, GFP_KERNEL);
3095         if (IS_ERR(child_counter))
3096                 return child_counter;
3097
3098         /*
3099          * Link it up in the child's context:
3100          */
3101         child_counter->task = child;
3102         add_counter_to_ctx(child_counter, child_ctx);
3103
3104         child_counter->parent = parent_counter;
3105         /*
3106          * inherit into child's child as well:
3107          */
3108         child_counter->hw_event.inherit = 1;
3109
3110         /*
3111          * Get a reference to the parent filp - we will fput it
3112          * when the child counter exits. This is safe to do because
3113          * we are in the parent and we know that the filp still
3114          * exists and has a nonzero count:
3115          */
3116         atomic_long_inc(&parent_counter->filp->f_count);
3117
3118         /*
3119          * Link this into the parent counter's child list
3120          */
3121         mutex_lock(&parent_counter->mutex);
3122         list_add_tail(&child_counter->child_list, &parent_counter->child_list);
3123
3124         /*
3125          * Make the child state follow the state of the parent counter,
3126          * not its hw_event.disabled bit.  We hold the parent's mutex,
3127          * so we won't race with perf_counter_{en,dis}able_family.
3128          */
3129         if (parent_counter->state >= PERF_COUNTER_STATE_INACTIVE)
3130                 child_counter->state = PERF_COUNTER_STATE_INACTIVE;
3131         else
3132                 child_counter->state = PERF_COUNTER_STATE_OFF;
3133
3134         mutex_unlock(&parent_counter->mutex);
3135
3136         return child_counter;
3137 }
3138
3139 static int inherit_group(struct perf_counter *parent_counter,
3140               struct task_struct *parent,
3141               struct perf_counter_context *parent_ctx,
3142               struct task_struct *child,
3143               struct perf_counter_context *child_ctx)
3144 {
3145         struct perf_counter *leader;
3146         struct perf_counter *sub;
3147         struct perf_counter *child_ctr;
3148
3149         leader = inherit_counter(parent_counter, parent, parent_ctx,
3150                                  child, NULL, child_ctx);
3151         if (IS_ERR(leader))
3152                 return PTR_ERR(leader);
3153         list_for_each_entry(sub, &parent_counter->sibling_list, list_entry) {
3154                 child_ctr = inherit_counter(sub, parent, parent_ctx,
3155                                             child, leader, child_ctx);
3156                 if (IS_ERR(child_ctr))
3157                         return PTR_ERR(child_ctr);
3158         }
3159         return 0;
3160 }
3161
3162 static void sync_child_counter(struct perf_counter *child_counter,
3163                                struct perf_counter *parent_counter)
3164 {
3165         u64 parent_val, child_val;
3166
3167         parent_val = atomic64_read(&parent_counter->count);
3168         child_val = atomic64_read(&child_counter->count);
3169
3170         /*
3171          * Add back the child's count to the parent's count:
3172          */
3173         atomic64_add(child_val, &parent_counter->count);
3174         atomic64_add(child_counter->total_time_enabled,
3175                      &parent_counter->child_total_time_enabled);
3176         atomic64_add(child_counter->total_time_running,
3177                      &parent_counter->child_total_time_running);
3178
3179         /*
3180          * Remove this counter from the parent's list
3181          */
3182         mutex_lock(&parent_counter->mutex);
3183         list_del_init(&child_counter->child_list);
3184         mutex_unlock(&parent_counter->mutex);
3185
3186         /*
3187          * Release the parent counter, if this was the last
3188          * reference to it.
3189          */
3190         fput(parent_counter->filp);
3191 }
3192
3193 static void
3194 __perf_counter_exit_task(struct task_struct *child,
3195                          struct perf_counter *child_counter,
3196                          struct perf_counter_context *child_ctx)
3197 {
3198         struct perf_counter *parent_counter;
3199         struct perf_counter *sub, *tmp;
3200
3201         /*
3202          * If we do not self-reap then we have to wait for the
3203          * child task to unschedule (it will happen for sure),
3204          * so that its counter is at its final count. (This
3205          * condition triggers rarely - child tasks usually get
3206          * off their CPU before the parent has a chance to
3207          * get this far into the reaping action)
3208          */
3209         if (child != current) {
3210                 wait_task_inactive(child, 0);
3211                 list_del_init(&child_counter->list_entry);
3212                 update_counter_times(child_counter);
3213         } else {
3214                 struct perf_cpu_context *cpuctx;
3215                 unsigned long flags;
3216
3217                 /*
3218                  * Disable and unlink this counter.
3219                  *
3220                  * Be careful about zapping the list - IRQ/NMI context
3221                  * could still be processing it:
3222                  */
3223                 local_irq_save(flags);
3224                 perf_disable();
3225
3226                 cpuctx = &__get_cpu_var(perf_cpu_context);
3227
3228                 group_sched_out(child_counter, cpuctx, child_ctx);
3229                 update_counter_times(child_counter);
3230
3231                 list_del_init(&child_counter->list_entry);
3232
3233                 child_ctx->nr_counters--;
3234
3235                 perf_enable();
3236                 local_irq_restore(flags);
3237         }
3238
3239         parent_counter = child_counter->parent;
3240         /*
3241          * It can happen that parent exits first, and has counters
3242          * that are still around due to the child reference. These
3243          * counters need to be zapped - but otherwise linger.
3244          */
3245         if (parent_counter) {
3246                 sync_child_counter(child_counter, parent_counter);
3247                 list_for_each_entry_safe(sub, tmp, &child_counter->sibling_list,
3248                                          list_entry) {
3249                         if (sub->parent) {
3250                                 sync_child_counter(sub, sub->parent);
3251                                 free_counter(sub);
3252                         }
3253                 }
3254                 free_counter(child_counter);
3255         }
3256 }
3257
3258 /*
3259  * When a child task exits, feed back counter values to parent counters.
3260  *
3261  * Note: we may be running in child context, but the PID is not hashed
3262  * anymore so new counters will not be added.
3263  */
3264 void perf_counter_exit_task(struct task_struct *child)
3265 {
3266         struct perf_counter *child_counter, *tmp;
3267         struct perf_counter_context *child_ctx;
3268
3269         child_ctx = &child->perf_counter_ctx;
3270
3271         if (likely(!child_ctx->nr_counters))
3272                 return;
3273
3274         list_for_each_entry_safe(child_counter, tmp, &child_ctx->counter_list,
3275                                  list_entry)
3276                 __perf_counter_exit_task(child, child_counter, child_ctx);
3277 }
3278
3279 /*
3280  * Initialize the perf_counter context in task_struct
3281  */
3282 void perf_counter_init_task(struct task_struct *child)
3283 {
3284         struct perf_counter_context *child_ctx, *parent_ctx;
3285         struct perf_counter *counter;
3286         struct task_struct *parent = current;
3287
3288         child_ctx  =  &child->perf_counter_ctx;
3289         parent_ctx = &parent->perf_counter_ctx;
3290
3291         __perf_counter_init_context(child_ctx, child);
3292
3293         /*
3294          * This is executed from the parent task context, so inherit
3295          * counters that have been marked for cloning:
3296          */
3297
3298         if (likely(!parent_ctx->nr_counters))
3299                 return;
3300
3301         /*
3302          * Lock the parent list. No need to lock the child - not PID
3303          * hashed yet and not running, so nobody can access it.
3304          */
3305         mutex_lock(&parent_ctx->mutex);
3306
3307         /*
3308          * We dont have to disable NMIs - we are only looking at
3309          * the list, not manipulating it:
3310          */
3311         list_for_each_entry(counter, &parent_ctx->counter_list, list_entry) {
3312                 if (!counter->hw_event.inherit)
3313                         continue;
3314
3315                 if (inherit_group(counter, parent,
3316                                   parent_ctx, child, child_ctx))
3317                         break;
3318         }
3319
3320         mutex_unlock(&parent_ctx->mutex);
3321 }
3322
3323 static void __cpuinit perf_counter_init_cpu(int cpu)
3324 {
3325         struct perf_cpu_context *cpuctx;
3326
3327         cpuctx = &per_cpu(perf_cpu_context, cpu);
3328         __perf_counter_init_context(&cpuctx->ctx, NULL);
3329
3330         spin_lock(&perf_resource_lock);
3331         cpuctx->max_pertask = perf_max_counters - perf_reserved_percpu;
3332         spin_unlock(&perf_resource_lock);
3333
3334         hw_perf_counter_setup(cpu);
3335 }
3336
3337 #ifdef CONFIG_HOTPLUG_CPU
3338 static void __perf_counter_exit_cpu(void *info)
3339 {
3340         struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
3341         struct perf_counter_context *ctx = &cpuctx->ctx;
3342         struct perf_counter *counter, *tmp;
3343
3344         list_for_each_entry_safe(counter, tmp, &ctx->counter_list, list_entry)
3345                 __perf_counter_remove_from_context(counter);
3346 }
3347 static void perf_counter_exit_cpu(int cpu)
3348 {
3349         struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
3350         struct perf_counter_context *ctx = &cpuctx->ctx;
3351
3352         mutex_lock(&ctx->mutex);
3353         smp_call_function_single(cpu, __perf_counter_exit_cpu, NULL, 1);
3354         mutex_unlock(&ctx->mutex);
3355 }
3356 #else
3357 static inline void perf_counter_exit_cpu(int cpu) { }
3358 #endif
3359
3360 static int __cpuinit
3361 perf_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
3362 {
3363         unsigned int cpu = (long)hcpu;
3364
3365         switch (action) {
3366
3367         case CPU_UP_PREPARE:
3368         case CPU_UP_PREPARE_FROZEN:
3369                 perf_counter_init_cpu(cpu);
3370                 break;
3371
3372         case CPU_DOWN_PREPARE:
3373         case CPU_DOWN_PREPARE_FROZEN:
3374                 perf_counter_exit_cpu(cpu);
3375                 break;
3376
3377         default:
3378                 break;
3379         }
3380
3381         return NOTIFY_OK;
3382 }
3383
3384 static struct notifier_block __cpuinitdata perf_cpu_nb = {
3385         .notifier_call          = perf_cpu_notify,
3386 };
3387
3388 void __init perf_counter_init(void)
3389 {
3390         perf_cpu_notify(&perf_cpu_nb, (unsigned long)CPU_UP_PREPARE,
3391                         (void *)(long)smp_processor_id());
3392         register_cpu_notifier(&perf_cpu_nb);
3393 }
3394
3395 static ssize_t perf_show_reserve_percpu(struct sysdev_class *class, char *buf)
3396 {
3397         return sprintf(buf, "%d\n", perf_reserved_percpu);
3398 }
3399
3400 static ssize_t
3401 perf_set_reserve_percpu(struct sysdev_class *class,
3402                         const char *buf,
3403                         size_t count)
3404 {
3405         struct perf_cpu_context *cpuctx;
3406         unsigned long val;
3407         int err, cpu, mpt;
3408
3409         err = strict_strtoul(buf, 10, &val);
3410         if (err)
3411                 return err;
3412         if (val > perf_max_counters)
3413                 return -EINVAL;
3414
3415         spin_lock(&perf_resource_lock);
3416         perf_reserved_percpu = val;
3417         for_each_online_cpu(cpu) {
3418                 cpuctx = &per_cpu(perf_cpu_context, cpu);
3419                 spin_lock_irq(&cpuctx->ctx.lock);
3420                 mpt = min(perf_max_counters - cpuctx->ctx.nr_counters,
3421                           perf_max_counters - perf_reserved_percpu);
3422                 cpuctx->max_pertask = mpt;
3423                 spin_unlock_irq(&cpuctx->ctx.lock);
3424         }
3425         spin_unlock(&perf_resource_lock);
3426
3427         return count;
3428 }
3429
3430 static ssize_t perf_show_overcommit(struct sysdev_class *class, char *buf)
3431 {
3432         return sprintf(buf, "%d\n", perf_overcommit);
3433 }
3434
3435 static ssize_t
3436 perf_set_overcommit(struct sysdev_class *class, const char *buf, size_t count)
3437 {
3438         unsigned long val;
3439         int err;
3440
3441         err = strict_strtoul(buf, 10, &val);
3442         if (err)
3443                 return err;
3444         if (val > 1)
3445                 return -EINVAL;
3446
3447         spin_lock(&perf_resource_lock);
3448         perf_overcommit = val;
3449         spin_unlock(&perf_resource_lock);
3450
3451         return count;
3452 }
3453
3454 static SYSDEV_CLASS_ATTR(
3455                                 reserve_percpu,
3456                                 0644,
3457                                 perf_show_reserve_percpu,
3458                                 perf_set_reserve_percpu
3459                         );
3460
3461 static SYSDEV_CLASS_ATTR(
3462                                 overcommit,
3463                                 0644,
3464                                 perf_show_overcommit,
3465                                 perf_set_overcommit
3466                         );
3467
3468 static struct attribute *perfclass_attrs[] = {
3469         &attr_reserve_percpu.attr,
3470         &attr_overcommit.attr,
3471         NULL
3472 };
3473
3474 static struct attribute_group perfclass_attr_group = {
3475         .attrs                  = perfclass_attrs,
3476         .name                   = "perf_counters",
3477 };
3478
3479 static int __init perf_counter_sysfs_init(void)
3480 {
3481         return sysfs_create_group(&cpu_sysdev_class.kset.kobj,
3482                                   &perfclass_attr_group);
3483 }
3484 device_initcall(perf_counter_sysfs_init);