4 * @remark Copyright 2002 OProfile authors
5 * @remark Read the file COPYING
7 * @author John Levon <levon@movementarian.org>
8 * @author Barry Kasindorf
10 * This is the core of the buffer management. Each
11 * CPU buffer is processed and entered into the
12 * global event buffer. Such processing is necessary
13 * in several circumstances, mentioned below.
15 * The processing does the job of converting the
16 * transitory EIP value into a persistent dentry/offset
17 * value that the profiler can record at its leisure.
19 * See fs/dcookies.c for a description of the dentry/offset
24 #include <linux/workqueue.h>
25 #include <linux/notifier.h>
26 #include <linux/dcookies.h>
27 #include <linux/profile.h>
28 #include <linux/module.h>
30 #include <linux/oprofile.h>
31 #include <linux/sched.h>
33 #include "oprofile_stats.h"
34 #include "event_buffer.h"
35 #include "cpu_buffer.h"
36 #include "buffer_sync.h"
38 static LIST_HEAD(dying_tasks);
39 static LIST_HEAD(dead_tasks);
40 static cpumask_t marked_cpus = CPU_MASK_NONE;
41 static DEFINE_SPINLOCK(task_mortuary);
42 static void process_task_mortuary(void);
44 /* Take ownership of the task struct and place it on the
45 * list for processing. Only after two full buffer syncs
46 * does the task eventually get freed, because by then
47 * we are sure we will not reference it again.
48 * Can be invoked from softirq via RCU callback due to
49 * call_rcu() of the task struct, hence the _irqsave.
52 task_free_notify(struct notifier_block *self, unsigned long val, void *data)
55 struct task_struct *task = data;
56 spin_lock_irqsave(&task_mortuary, flags);
57 list_add(&task->tasks, &dying_tasks);
58 spin_unlock_irqrestore(&task_mortuary, flags);
63 /* The task is on its way out. A sync of the buffer means we can catch
64 * any remaining samples for this task.
67 task_exit_notify(struct notifier_block *self, unsigned long val, void *data)
69 /* To avoid latency problems, we only process the current CPU,
70 * hoping that most samples for the task are on this CPU
72 sync_buffer(raw_smp_processor_id());
77 /* The task is about to try a do_munmap(). We peek at what it's going to
78 * do, and if it's an executable region, process the samples first, so
79 * we don't lose any. This does not have to be exact, it's a QoI issue
83 munmap_notify(struct notifier_block *self, unsigned long val, void *data)
85 unsigned long addr = (unsigned long)data;
86 struct mm_struct *mm = current->mm;
87 struct vm_area_struct *mpnt;
89 down_read(&mm->mmap_sem);
91 mpnt = find_vma(mm, addr);
92 if (mpnt && mpnt->vm_file && (mpnt->vm_flags & VM_EXEC)) {
93 up_read(&mm->mmap_sem);
94 /* To avoid latency problems, we only process the current CPU,
95 * hoping that most samples for the task are on this CPU
97 sync_buffer(raw_smp_processor_id());
101 up_read(&mm->mmap_sem);
106 /* We need to be told about new modules so we don't attribute to a previously
107 * loaded module, or drop the samples on the floor.
110 module_load_notify(struct notifier_block *self, unsigned long val, void *data)
112 #ifdef CONFIG_MODULES
113 if (val != MODULE_STATE_COMING)
116 /* FIXME: should we process all CPU buffers ? */
117 mutex_lock(&buffer_mutex);
118 add_event_entry(ESCAPE_CODE);
119 add_event_entry(MODULE_LOADED_CODE);
120 mutex_unlock(&buffer_mutex);
126 static struct notifier_block task_free_nb = {
127 .notifier_call = task_free_notify,
130 static struct notifier_block task_exit_nb = {
131 .notifier_call = task_exit_notify,
134 static struct notifier_block munmap_nb = {
135 .notifier_call = munmap_notify,
138 static struct notifier_block module_load_nb = {
139 .notifier_call = module_load_notify,
143 static void end_sync(void)
146 /* make sure we don't leak task structs */
147 process_task_mortuary();
148 process_task_mortuary();
158 err = task_handoff_register(&task_free_nb);
161 err = profile_event_register(PROFILE_TASK_EXIT, &task_exit_nb);
164 err = profile_event_register(PROFILE_MUNMAP, &munmap_nb);
167 err = register_module_notifier(&module_load_nb);
174 profile_event_unregister(PROFILE_MUNMAP, &munmap_nb);
176 profile_event_unregister(PROFILE_TASK_EXIT, &task_exit_nb);
178 task_handoff_unregister(&task_free_nb);
187 unregister_module_notifier(&module_load_nb);
188 profile_event_unregister(PROFILE_MUNMAP, &munmap_nb);
189 profile_event_unregister(PROFILE_TASK_EXIT, &task_exit_nb);
190 task_handoff_unregister(&task_free_nb);
195 /* Optimisation. We can manage without taking the dcookie sem
196 * because we cannot reach this code without at least one
197 * dcookie user still being registered (namely, the reader
198 * of the event buffer). */
199 static inline unsigned long fast_get_dcookie(struct path *path)
201 unsigned long cookie;
203 if (path->dentry->d_cookie)
204 return (unsigned long)path->dentry;
205 get_dcookie(path, &cookie);
210 /* Look up the dcookie for the task's first VM_EXECUTABLE mapping,
211 * which corresponds loosely to "application name". This is
212 * not strictly necessary but allows oprofile to associate
213 * shared-library samples with particular applications
215 static unsigned long get_exec_dcookie(struct mm_struct *mm)
217 unsigned long cookie = NO_COOKIE;
218 struct vm_area_struct *vma;
223 for (vma = mm->mmap; vma; vma = vma->vm_next) {
226 if (!(vma->vm_flags & VM_EXECUTABLE))
228 cookie = fast_get_dcookie(&vma->vm_file->f_path);
237 /* Convert the EIP value of a sample into a persistent dentry/offset
238 * pair that can then be added to the global event buffer. We make
239 * sure to do this lookup before a mm->mmap modification happens so
240 * we don't lose track.
243 lookup_dcookie(struct mm_struct *mm, unsigned long addr, off_t *offset)
245 unsigned long cookie = NO_COOKIE;
246 struct vm_area_struct *vma;
248 for (vma = find_vma(mm, addr); vma; vma = vma->vm_next) {
250 if (addr < vma->vm_start || addr >= vma->vm_end)
254 cookie = fast_get_dcookie(&vma->vm_file->f_path);
255 *offset = (vma->vm_pgoff << PAGE_SHIFT) + addr -
258 /* must be an anonymous map */
266 cookie = INVALID_COOKIE;
271 static unsigned long last_cookie = INVALID_COOKIE;
273 static void add_cpu_switch(int i)
275 add_event_entry(ESCAPE_CODE);
276 add_event_entry(CPU_SWITCH_CODE);
278 last_cookie = INVALID_COOKIE;
281 static void add_kernel_ctx_switch(unsigned int in_kernel)
283 add_event_entry(ESCAPE_CODE);
285 add_event_entry(KERNEL_ENTER_SWITCH_CODE);
287 add_event_entry(KERNEL_EXIT_SWITCH_CODE);
291 add_user_ctx_switch(struct task_struct const *task, unsigned long cookie)
293 add_event_entry(ESCAPE_CODE);
294 add_event_entry(CTX_SWITCH_CODE);
295 add_event_entry(task->pid);
296 add_event_entry(cookie);
297 /* Another code for daemon back-compat */
298 add_event_entry(ESCAPE_CODE);
299 add_event_entry(CTX_TGID_CODE);
300 add_event_entry(task->tgid);
304 static void add_cookie_switch(unsigned long cookie)
306 add_event_entry(ESCAPE_CODE);
307 add_event_entry(COOKIE_SWITCH_CODE);
308 add_event_entry(cookie);
312 static void add_trace_begin(void)
314 add_event_entry(ESCAPE_CODE);
315 add_event_entry(TRACE_BEGIN_CODE);
318 #ifdef CONFIG_OPROFILE_IBS
320 #define IBS_FETCH_CODE_SIZE 2
321 #define IBS_OP_CODE_SIZE 5
324 * Add IBS fetch and op entries to event buffer
326 static void add_ibs_begin(int cpu, int code, struct mm_struct *mm)
330 unsigned long ibs_cookie = 0;
332 struct op_sample *sample;
334 sample = op_cpu_buffer_read_entry(cpu);
340 rip += sample->event << 32;
344 ibs_cookie = lookup_dcookie(mm, rip, &offset);
346 if (ibs_cookie == NO_COOKIE)
348 if (ibs_cookie == INVALID_COOKIE) {
349 atomic_inc(&oprofile_stats.sample_lost_no_mapping);
352 if (ibs_cookie != last_cookie) {
353 add_cookie_switch(ibs_cookie);
354 last_cookie = ibs_cookie;
359 add_event_entry(ESCAPE_CODE);
360 add_event_entry(code);
361 add_event_entry(offset); /* Offset from Dcookie */
363 /* we send the Dcookie offset, but send the raw Linear Add also*/
364 add_event_entry(sample->eip);
365 add_event_entry(sample->event);
367 if (code == IBS_FETCH_CODE)
368 count = IBS_FETCH_CODE_SIZE; /*IBS FETCH is 2 int64s*/
370 count = IBS_OP_CODE_SIZE; /*IBS OP is 5 int64s*/
372 for (i = 0; i < count; i++) {
373 sample = op_cpu_buffer_read_entry(cpu);
376 add_event_entry(sample->eip);
377 add_event_entry(sample->event);
385 static void add_sample_entry(unsigned long offset, unsigned long event)
387 add_event_entry(offset);
388 add_event_entry(event);
393 * Add a sample to the global event buffer. If possible the
394 * sample is converted into a persistent dentry/offset pair
395 * for later lookup from userspace. Return 0 on failure.
398 add_sample(struct mm_struct *mm, struct op_sample *s, int in_kernel)
400 unsigned long cookie;
404 add_sample_entry(s->eip, s->event);
408 /* add userspace sample */
411 atomic_inc(&oprofile_stats.sample_lost_no_mm);
415 cookie = lookup_dcookie(mm, s->eip, &offset);
417 if (cookie == INVALID_COOKIE) {
418 atomic_inc(&oprofile_stats.sample_lost_no_mapping);
422 if (cookie != last_cookie) {
423 add_cookie_switch(cookie);
424 last_cookie = cookie;
427 add_sample_entry(offset, s->event);
433 static void release_mm(struct mm_struct *mm)
437 up_read(&mm->mmap_sem);
442 static struct mm_struct *take_tasks_mm(struct task_struct *task)
444 struct mm_struct *mm = get_task_mm(task);
446 down_read(&mm->mmap_sem);
451 static inline int is_code(unsigned long val)
453 return val == ESCAPE_CODE;
457 /* Move tasks along towards death. Any tasks on dead_tasks
458 * will definitely have no remaining references in any
459 * CPU buffers at this point, because we use two lists,
460 * and to have reached the list, it must have gone through
461 * one full sync already.
463 static void process_task_mortuary(void)
466 LIST_HEAD(local_dead_tasks);
467 struct task_struct *task;
468 struct task_struct *ttask;
470 spin_lock_irqsave(&task_mortuary, flags);
472 list_splice_init(&dead_tasks, &local_dead_tasks);
473 list_splice_init(&dying_tasks, &dead_tasks);
475 spin_unlock_irqrestore(&task_mortuary, flags);
477 list_for_each_entry_safe(task, ttask, &local_dead_tasks, tasks) {
478 list_del(&task->tasks);
484 static void mark_done(int cpu)
488 cpu_set(cpu, marked_cpus);
490 for_each_online_cpu(i) {
491 if (!cpu_isset(i, marked_cpus))
495 /* All CPUs have been processed at least once,
496 * we can process the mortuary once
498 process_task_mortuary();
500 cpus_clear(marked_cpus);
504 /* FIXME: this is not sufficient if we implement syscall barrier backtrace
505 * traversal, the code switch to sb_sample_start at first kernel enter/exit
506 * switch so we need a fifth state and some special handling in sync_buffer()
515 /* Sync one of the CPU's buffers into the global event buffer.
516 * Here we need to go through each batch of samples punctuated
517 * by context switch notes, taking the task's mmap_sem and doing
518 * lookup in task->mm->mmap to convert EIP into dcookie/offset
521 void sync_buffer(int cpu)
523 struct mm_struct *mm = NULL;
524 struct mm_struct *oldmm;
525 struct task_struct *new;
526 unsigned long cookie = 0;
528 sync_buffer_state state = sb_buffer_start;
530 unsigned long available;
532 mutex_lock(&buffer_mutex);
536 op_cpu_buffer_reset(cpu);
537 available = op_cpu_buffer_entries(cpu);
539 for (i = 0; i < available; ++i) {
540 struct op_sample *s = op_cpu_buffer_read_entry(cpu);
544 if (is_code(s->eip)) {
548 /* kernel/userspace switch */
549 in_kernel = s->event;
550 if (state == sb_buffer_start)
551 state = sb_sample_start;
552 add_kernel_ctx_switch(s->event);
554 case CPU_TRACE_BEGIN:
558 #ifdef CONFIG_OPROFILE_IBS
559 case IBS_FETCH_BEGIN:
560 add_ibs_begin(cpu, IBS_FETCH_CODE, mm);
563 add_ibs_begin(cpu, IBS_OP_CODE, mm);
567 /* userspace context switch */
569 new = (struct task_struct *)s->event;
571 mm = take_tasks_mm(new);
573 cookie = get_exec_dcookie(mm);
574 add_user_ctx_switch(new, cookie);
580 if (state < sb_bt_start)
584 if (add_sample(mm, s, in_kernel))
587 /* ignore backtraces if failed to add a sample */
588 if (state == sb_bt_start) {
589 state = sb_bt_ignore;
590 atomic_inc(&oprofile_stats.bt_lost_no_mapping);
597 mutex_unlock(&buffer_mutex);
600 /* The function can be used to add a buffer worth of data directly to
601 * the kernel buffer. The buffer is assumed to be a circular buffer.
602 * Take the entries from index start and end at index end, wrapping
605 void oprofile_put_buff(unsigned long *buf, unsigned int start,
606 unsigned int stop, unsigned int max)
612 mutex_lock(&buffer_mutex);
614 add_event_entry(buf[i++]);
620 mutex_unlock(&buffer_mutex);