#define TIF_SECCOMP 8 /* secure computing */
#define TIF_MCE_NOTIFY 10 /* notify userspace of an MCE */
#define TIF_USER_RETURN_NOTIFY 11 /* notify kernel of userspace return */
+#define TIF_UPROBE 12 /* breakpointed or singlestepping */
#define TIF_NOTSC 16 /* TSC is not accessible in userland */
#define TIF_IA32 17 /* 32bit process */
#define TIF_FORK 18 /* ret_from_fork */
#define _TIF_SECCOMP (1 << TIF_SECCOMP)
#define _TIF_MCE_NOTIFY (1 << TIF_MCE_NOTIFY)
#define _TIF_USER_RETURN_NOTIFY (1 << TIF_USER_RETURN_NOTIFY)
+#define _TIF_UPROBE (1 << TIF_UPROBE)
#define _TIF_NOTSC (1 << TIF_NOTSC)
#define _TIF_IA32 (1 << TIF_IA32)
#define _TIF_FORK (1 << TIF_FORK)
* Jim Keniston
*/
+#include <linux/notifier.h>
+
typedef u8 uprobe_opcode_t;
#define MAX_UINSN_BYTES 16
#define UPROBES_XOL_SLOT_BYTES 128 /* to keep it cache aligned */
#endif
};
+struct uprobe_task_arch_info {
+ unsigned long saved_trap_no;
+#ifdef CONFIG_X86_64
+ unsigned long saved_scratch_register;
+#endif
+};
+
struct uprobe;
+
extern int analyze_insn(struct mm_struct *mm, struct uprobe *uprobe);
+extern void set_instruction_pointer(struct pt_regs *regs, unsigned long vaddr);
+extern int pre_xol(struct uprobe *uprobe, struct pt_regs *regs);
+extern int post_xol(struct uprobe *uprobe, struct pt_regs *regs);
+extern bool xol_was_trapped(struct task_struct *tsk);
+extern int uprobe_exception_notify(struct notifier_block *self,
+ unsigned long val, void *data);
+extern void abort_xol(struct pt_regs *regs, struct uprobe *uprobe);
#endif /* _ASM_UPROBES_H */
#include <linux/personality.h>
#include <linux/uaccess.h>
#include <linux/user-return-notifier.h>
+#include <linux/uprobes.h>
#include <asm/processor.h>
#include <asm/ucontext.h>
mce_notify_process();
#endif /* CONFIG_X86_64 && CONFIG_X86_MCE */
+ if (thread_info_flags & _TIF_UPROBE) {
+ clear_thread_flag(TIF_UPROBE);
+ uprobe_notify_resume(regs);
+ }
+
/* deal with pending signal delivery */
if (thread_info_flags & _TIF_SIGPENDING)
do_signal(regs);
#include <linux/sched.h>
#include <linux/ptrace.h>
#include <linux/uprobes.h>
+#include <linux/uaccess.h>
#include <linux/kdebug.h>
#include <asm/insn.h>
+#ifdef CONFIG_X86_32
+#define is_32bit_app(tsk) 1
+#else
+#define is_32bit_app(tsk) (test_tsk_thread_flag(tsk, TIF_IA32))
+#endif
+
/* Post-execution fixups. */
/* No fixup needed */
prepare_fixups(uprobe, &insn);
return 0;
}
+
+/*
+ * @reg: reflects the saved state of the task
+ * @vaddr: the virtual address to jump to.
+ * Return 0 on success or a -ve number on error.
+ */
+void set_instruction_pointer(struct pt_regs *regs, unsigned long vaddr)
+{
+ regs->ip = vaddr;
+}
+
+#define UPROBE_TRAP_NO UINT_MAX
+
+/*
+ * pre_xol - prepare to execute out of line.
+ * @uprobe: the probepoint information.
+ * @regs: reflects the saved user state of @tsk.
+ *
+ * If we're emulating a rip-relative instruction, save the contents
+ * of the scratch register and store the target address in that register.
+ *
+ * Returns true if @uprobe->opcode is @bkpt_insn.
+ */
+#ifdef CONFIG_X86_64
+int pre_xol(struct uprobe *uprobe, struct pt_regs *regs)
+{
+ struct uprobe_task_arch_info *tskinfo = ¤t->utask->tskinfo;
+
+ tskinfo->saved_trap_no = current->thread.trap_no;
+ current->thread.trap_no = UPROBE_TRAP_NO;
+
+ regs->ip = current->utask->xol_vaddr;
+ if (uprobe->arch_info.fixups & UPROBES_FIX_RIP_AX) {
+ tskinfo->saved_scratch_register = regs->ax;
+ regs->ax = current->utask->vaddr;
+ regs->ax += uprobe->arch_info.rip_rela_target_address;
+ } else if (uprobe->arch_info.fixups & UPROBES_FIX_RIP_CX) {
+ tskinfo->saved_scratch_register = regs->cx;
+ regs->cx = current->utask->vaddr;
+ regs->cx += uprobe->arch_info.rip_rela_target_address;
+ }
+ return 0;
+}
+#else
+int pre_xol(struct uprobe *uprobe, struct pt_regs *regs)
+{
+ struct uprobe_task_arch_info *tskinfo = ¤t->utask->tskinfo;
+
+ tskinfo->saved_trap_no = current->thread.trap_no;
+ current->thread.trap_no = UPROBE_TRAP_NO;
+
+ regs->ip = current->utask->xol_vaddr;
+ return 0;
+}
+#endif
+
+/*
+ * Called by post_xol() to adjust the return address pushed by a call
+ * instruction executed out of line.
+ */
+static int adjust_ret_addr(unsigned long sp, long correction)
+{
+ int rasize, ncopied;
+ long ra = 0;
+
+ if (is_32bit_app(current))
+ rasize = 4;
+ else
+ rasize = 8;
+
+ ncopied = copy_from_user(&ra, (void __user *)sp, rasize);
+ if (unlikely(ncopied))
+ return -EFAULT;
+
+ ra += correction;
+ ncopied = copy_to_user((void __user *)sp, &ra, rasize);
+ if (unlikely(ncopied))
+ return -EFAULT;
+
+ return 0;
+}
+
+#ifdef CONFIG_X86_64
+static bool is_riprel_insn(struct uprobe *uprobe)
+{
+ return ((uprobe->arch_info.fixups &
+ (UPROBES_FIX_RIP_AX | UPROBES_FIX_RIP_CX)) != 0);
+}
+
+static void handle_riprel_post_xol(struct uprobe *uprobe,
+ struct pt_regs *regs, long *correction)
+{
+ if (is_riprel_insn(uprobe)) {
+ struct uprobe_task_arch_info *tskinfo;
+ tskinfo = ¤t->utask->tskinfo;
+
+ if (uprobe->arch_info.fixups & UPROBES_FIX_RIP_AX)
+ regs->ax = tskinfo->saved_scratch_register;
+ else
+ regs->cx = tskinfo->saved_scratch_register;
+ /*
+ * The original instruction includes a displacement, and so
+ * is 4 bytes longer than what we've just single-stepped.
+ * Fall through to handle stuff like "jmpq *...(%rip)" and
+ * "callq *...(%rip)".
+ */
+ if (correction)
+ *correction += 4;
+ }
+}
+#else
+static void handle_riprel_post_xol(struct uprobe *uprobe,
+ struct pt_regs *regs, long *correction)
+{
+}
+#endif
+
+/*
+ * If xol insn itself traps and generates a signal(Say,
+ * SIGILL/SIGSEGV/etc), then detect the case where a singlestepped
+ * instruction jumps back to its own address. It is assumed that anything
+ * like do_page_fault/do_trap/etc sets thread.trap_no != -1.
+ *
+ * pre_xol/post_xol save/restore thread.trap_no, xol_was_trapped() simply
+ * checks that ->trap_no is not equal to UPROBE_TRAP_NO == -1 set by
+ * pre_xol().
+ */
+bool xol_was_trapped(struct task_struct *tsk)
+{
+ if (tsk->thread.trap_no != UPROBE_TRAP_NO)
+ return true;
+
+ return false;
+}
+
+/*
+ * Called after single-stepping. To avoid the SMP problems that can
+ * occur when we temporarily put back the original opcode to
+ * single-step, we single-stepped a copy of the instruction.
+ *
+ * This function prepares to resume execution after the single-step.
+ * We have to fix things up as follows:
+ *
+ * Typically, the new ip is relative to the copied instruction. We need
+ * to make it relative to the original instruction (FIX_IP). Exceptions
+ * are return instructions and absolute or indirect jump or call instructions.
+ *
+ * If the single-stepped instruction was a call, the return address that
+ * is atop the stack is the address following the copied instruction. We
+ * need to make it the address following the original instruction (FIX_CALL).
+ *
+ * If the original instruction was a rip-relative instruction such as
+ * "movl %edx,0xnnnn(%rip)", we have instead executed an equivalent
+ * instruction using a scratch register -- e.g., "movl %edx,(%rax)".
+ * We need to restore the contents of the scratch register and adjust
+ * the ip, keeping in mind that the instruction we executed is 4 bytes
+ * shorter than the original instruction (since we squeezed out the offset
+ * field). (FIX_RIP_AX or FIX_RIP_CX)
+ */
+int post_xol(struct uprobe *uprobe, struct pt_regs *regs)
+{
+ struct uprobe_task *utask = current->utask;
+ int result = 0;
+ long correction;
+
+ WARN_ON_ONCE(current->thread.trap_no != UPROBE_TRAP_NO);
+
+ current->thread.trap_no = utask->tskinfo.saved_trap_no;
+ correction = (long)(utask->vaddr - utask->xol_vaddr);
+ handle_riprel_post_xol(uprobe, regs, &correction);
+ if (uprobe->arch_info.fixups & UPROBES_FIX_IP)
+ regs->ip += correction;
+ if (uprobe->arch_info.fixups & UPROBES_FIX_CALL)
+ result = adjust_ret_addr(regs->sp, correction);
+ return result;
+}
+
+/*
+ * Wrapper routine for handling exceptions.
+ */
+int uprobe_exception_notify(struct notifier_block *self,
+ unsigned long val, void *data)
+{
+ struct die_args *args = data;
+ struct pt_regs *regs = args->regs;
+ int ret = NOTIFY_DONE;
+
+ /* We are only interested in userspace traps */
+ if (regs && !user_mode_vm(regs))
+ return NOTIFY_DONE;
+
+ switch (val) {
+ case DIE_INT3:
+ /* Run your handler here */
+ if (uprobe_bkpt_notifier(regs))
+ ret = NOTIFY_STOP;
+ break;
+ case DIE_DEBUG:
+ if (uprobe_post_notifier(regs))
+ ret = NOTIFY_STOP;
+ default:
+ break;
+ }
+ return ret;
+}
+
+/*
+ * xol insn either trapped or thread has a fatal signal, so reset the
+ * instruction pointer to its probed address.
+ */
+void abort_xol(struct pt_regs *regs, struct uprobe *uprobe)
+{
+ struct uprobe_task *utask = current->utask;
+
+ current->thread.trap_no = utask->tskinfo.saved_trap_no;
+ handle_riprel_post_xol(uprobe, regs, NULL);
+ set_instruction_pointer(regs, utask->vaddr);
+}
+
+/*
+ * Skip these instructions:
+ *
+ * 0f 19 90 90 90 90 90 nopl -0x6f6f6f70(%rax)
+ * 0f 1f 00 nopl (%rax)
+ * 0f 1f 40 00 nopl 0x0(%rax)
+ * 0f 1f 44 00 00 nopl 0x0(%rax,%rax,1)
+ * 0f 1f 80 00 00 00 00 nopl 0x0(%rax)
+ * 0f 1f 84 00 00 00 00 nopl 0x0(%rax,%rax,1)
+ * 66 0f 1f 44 00 00 00 nopw 0x0(%rax,%rax,1)
+ * 66 0f 1f 84 00 00 00 nopw 0x0(%rax,%rax,1)
+ * 66 87 c0 xchg %eax,%eax
+ * 66 90 nop
+ * 87 c0 xchg %eax,%eax
+ * 90 nop
+ */
+
+bool can_skip_xol(struct pt_regs *regs, struct uprobe *u)
+{
+ int i;
+
+ for (i = 0; i < MAX_UINSN_BYTES; i++) {
+ if ((u->insn[i] == 0x66))
+ continue;
+
+ if (u->insn[i] == 0x90)
+ return true;
+
+ if (i == (MAX_UINSN_BYTES - 1))
+ break;
+
+ if ((u->insn[i] == 0x0f) && (u->insn[i+1] == 0x1f))
+ return true;
+
+ if ((u->insn[i] == 0x0f) && (u->insn[i+1] == 0x19))
+ return true;
+
+ if ((u->insn[i] == 0x87) && (u->insn[i+1] == 0xc0))
+ return true;
+
+ break;
+ }
+
+ u->flags &= ~UPROBES_SKIP_SSTEP;
+ return false;
+}
#ifdef CONFIG_HAVE_HW_BREAKPOINT
atomic_t ptrace_bp_refcnt;
#endif
+#ifdef CONFIG_UPROBES
+ struct uprobe_task *utask;
+ int uprobes_srcu_id;
+#endif
};
/* Future-safe accessor for struct task_struct's cpus_allowed. */
typedef u8 uprobe_opcode_t;
struct uprobe_arch_info {};
-
+struct uprobe_task_arch_info {}; /* arch specific task info */
#define MAX_UINSN_BYTES 4
#endif
#define UPROBES_COPY_INSN 0x1
/* Dont run handlers when first register/ last unregister in progress*/
#define UPROBES_RUN_HANDLER 0x2
+/* Can skip singlestep */
+#define UPROBES_SKIP_SSTEP 0x4
struct uprobe_consumer {
int (*handler)(struct uprobe_consumer *self, struct pt_regs *regs);
u8 insn[MAX_UINSN_BYTES];
};
+enum uprobe_task_state {
+ UTASK_RUNNING,
+ UTASK_BP_HIT,
+ UTASK_SSTEP,
+ UTASK_SSTEP_ACK,
+ UTASK_SSTEP_TRAPPED,
+};
+
+/*
+ * uprobe_task: Metadata of a task while it singlesteps.
+ */
+struct uprobe_task {
+ unsigned long xol_vaddr;
+ unsigned long vaddr;
+
+ enum uprobe_task_state state;
+ struct uprobe_task_arch_info tskinfo;
+
+ struct uprobe *active_uprobe;
+};
+
#ifdef CONFIG_UPROBES
extern int __weak set_bkpt(struct mm_struct *mm, struct uprobe *uprobe,
unsigned long vaddr);
struct uprobe_consumer *consumer);
extern void unregister_uprobe(struct inode *inode, loff_t offset,
struct uprobe_consumer *consumer);
+extern void free_uprobe_utask(struct task_struct *tsk);
extern int mmap_uprobe(struct vm_area_struct *vma);
+extern unsigned long __weak get_uprobe_bkpt_addr(struct pt_regs *regs);
+extern int uprobe_post_notifier(struct pt_regs *regs);
+extern int uprobe_bkpt_notifier(struct pt_regs *regs);
+extern void uprobe_notify_resume(struct pt_regs *regs);
+extern bool uprobe_deny_signal(void);
+extern bool __weak can_skip_xol(struct pt_regs *regs, struct uprobe *u);
#else /* CONFIG_UPROBES is not defined */
static inline int register_uprobe(struct inode *inode, loff_t offset,
struct uprobe_consumer *consumer)
{
return 0;
}
+static inline void uprobe_notify_resume(struct pt_regs *regs)
+{
+}
+static inline bool uprobe_deny_signal(void)
+{
+ return false;
+}
+static inline unsigned long get_uprobe_bkpt_addr(struct pt_regs *regs)
+{
+ return 0;
+}
+static inline void free_uprobe_utask(struct task_struct *tsk)
+{
+}
#endif /* CONFIG_UPROBES */
#endif /* _LINUX_UPROBES_H */
#include <linux/user-return-notifier.h>
#include <linux/oom.h>
#include <linux/khugepaged.h>
+#include <linux/uprobes.h>
#include <asm/pgtable.h>
#include <asm/pgalloc.h>
exit_pi_state_list(tsk);
#endif
+ free_uprobe_utask(tsk);
+
/* Get rid of any cached register state */
deactivate_mm(tsk, mm);
#endif
INIT_LIST_HEAD(&p->pi_state_list);
p->pi_state_cache = NULL;
+#endif
+#ifdef CONFIG_UPROBES
+ p->utask = NULL;
+ p->uprobes_srcu_id = -1;
#endif
/*
* sigaltstack should be cleared when sharing the same VM
struct signal_struct *signal = current->signal;
int signr;
+ if (unlikely(uprobe_deny_signal()))
+ return 0;
+
relock:
/*
* We'll jump back here after any time we were stopped in TASK_STOPPED.
#include <linux/rmap.h> /* anon_vma_prepare */
#include <linux/mmu_notifier.h> /* set_pte_at_notify */
#include <linux/swap.h> /* try_to_free_swap */
+#include <linux/ptrace.h> /* user_enable_single_step */
+#include <linux/kdebug.h> /* notifier mechanism */
#include <linux/uprobes.h>
+static struct srcu_struct uprobes_srcu;
static struct rb_root uprobes_tree = RB_ROOT;
static DEFINE_SPINLOCK(uprobes_treelock); /* serialize rbtree access */
spin_lock_irqsave(&uprobes_treelock, flags);
u = __insert_uprobe(uprobe);
spin_unlock_irqrestore(&uprobes_treelock, flags);
+
+ /* For now assume that the instruction need not be single-stepped */
+ uprobe->flags |= UPROBES_SKIP_SSTEP;
return u;
}
return uprobe;
}
+static void handler_chain(struct uprobe *uprobe, struct pt_regs *regs)
+{
+ struct uprobe_consumer *consumer;
+
+ if (!(uprobe->flags & UPROBES_RUN_HANDLER))
+ return;
+
+ down_read(&uprobe->consumer_rwsem);
+ consumer = uprobe->consumers;
+ for (consumer = uprobe->consumers; consumer;
+ consumer = consumer->next) {
+ if (!consumer->filter ||
+ consumer->filter(consumer, current))
+ consumer->handler(consumer, regs);
+ }
+ up_read(&uprobe->consumer_rwsem);
+}
+
/* Returns the previous consumer */
static struct uprobe_consumer *add_consumer(struct uprobe *uprobe,
struct uprobe_consumer *consumer)
set_orig_insn(mm, uprobe, (unsigned long)vaddr, true);
}
+/*
+ * There could be threads that have hit the breakpoint and are entering the
+ * notifier code and trying to acquire the uprobes_treelock. The thread
+ * calling delete_uprobe() that is removing the uprobe from the rb_tree can
+ * race with these threads and might acquire the uprobes_treelock compared
+ * to some of the breakpoint hit threads. In such a case, the breakpoint hit
+ * threads will not find the uprobe. Hence wait till the current breakpoint
+ * hit threads acquire the uprobes_treelock before the uprobe is removed
+ * from the rbtree.
+ */
static void delete_uprobe(struct uprobe *uprobe)
{
unsigned long flags;
+ synchronize_srcu(&uprobes_srcu);
spin_lock_irqsave(&uprobes_treelock, flags);
rb_erase(&uprobe->rb_node, &uprobes_tree);
spin_unlock_irqrestore(&uprobes_treelock, flags);
return ret;
}
+/**
+ * get_uprobe_bkpt_addr - compute address of bkpt given post-bkpt regs
+ * @regs: Reflects the saved state of the task after it has hit a breakpoint
+ * instruction.
+ * Return the address of the breakpoint instruction.
+ */
+unsigned long __weak get_uprobe_bkpt_addr(struct pt_regs *regs)
+{
+ return instruction_pointer(regs) - UPROBES_BKPT_INSN_SIZE;
+}
+
+/*
+ * Called with no locks held.
+ * Called in context of a exiting or a exec-ing thread.
+ */
+void free_uprobe_utask(struct task_struct *tsk)
+{
+ struct uprobe_task *utask = tsk->utask;
+
+ if (tsk->uprobes_srcu_id != -1)
+ srcu_read_unlock_raw(&uprobes_srcu, tsk->uprobes_srcu_id);
+
+ if (!utask)
+ return;
+
+ if (utask->active_uprobe)
+ put_uprobe(utask->active_uprobe);
+
+ kfree(utask);
+ tsk->utask = NULL;
+}
+
+/*
+ * Allocate a uprobe_task object for the task.
+ * Called when the thread hits a breakpoint for the first time.
+ *
+ * Returns:
+ * - pointer to new uprobe_task on success
+ * - NULL otherwise
+ */
+static struct uprobe_task *add_utask(void)
+{
+ struct uprobe_task *utask;
+
+ utask = kzalloc(sizeof *utask, GFP_KERNEL);
+ if (unlikely(utask == NULL))
+ return NULL;
+
+ utask->active_uprobe = NULL;
+ current->utask = utask;
+ return utask;
+}
+
+/* Prepare to single-step probed instruction out of line. */
+static int pre_ssout(struct uprobe *uprobe, struct pt_regs *regs,
+ unsigned long vaddr)
+{
+ return -EFAULT;
+}
+
+/*
+ * If we are singlestepping, then ensure this thread is not connected to
+ * non-fatal signals until completion of singlestep. When xol insn itself
+ * triggers the signal, restart the original insn even if the task is
+ * already SIGKILL'ed (since coredump should report the correct ip). This
+ * is even more important if the task has a handler for SIGSEGV/etc, The
+ * _same_ instruction should be repeated again after return from the signal
+ * handler, and SSTEP can never finish in this case.
+ */
+bool uprobe_deny_signal(void)
+{
+ struct task_struct *tsk = current;
+ struct uprobe_task *utask = tsk->utask;
+
+ if (likely(!utask || !utask->active_uprobe))
+ return false;
+
+ WARN_ON_ONCE(utask->state != UTASK_SSTEP);
+
+ if (signal_pending(tsk)) {
+ spin_lock_irq(&tsk->sighand->siglock);
+ clear_tsk_thread_flag(tsk, TIF_SIGPENDING);
+ spin_unlock_irq(&tsk->sighand->siglock);
+
+ if (__fatal_signal_pending(tsk) || xol_was_trapped(tsk)) {
+ utask->state = UTASK_SSTEP_TRAPPED;
+ set_tsk_thread_flag(tsk, TIF_UPROBE);
+ set_tsk_thread_flag(tsk, TIF_NOTIFY_RESUME);
+ }
+ }
+
+ return true;
+}
+
+bool __weak can_skip_xol(struct pt_regs *regs, struct uprobe *u)
+{
+ u->flags &= ~UPROBES_SKIP_SSTEP;
+ return false;
+}
+
+/*
+ * On breakpoint hit, breakpoint notifier sets the TIF_UPROBE flag. (and on
+ * subsequent probe hits on the thread sets the state to UTASK_BP_HIT) and
+ * allows the thread to return from interrupt. While returning to
+ * userspace, thread noticies the TIF_UPROBE flag and calls
+ * uprobe_notify_resume(). uprobe_notify_resume will run the handler and ask
+ * the thread to singlestep.
+ *
+ * On subsequent singlestep exception, singlestep notifier sets the
+ * TIF_UPROBE flag and also sets the state to UTASK_SSTEP_ACK and allows the
+ * thread to return from interrupt. While returning to userspace, thread
+ * notices the TIF_UPROBE and calls uprobe_notify_resume().
+ * uprobe_notify_resume disables singlestep and performs the required
+ * fix-ups.
+ *
+ * All non-fatal signals cannot interrupt thread while the thread singlesteps.
+ */
+void uprobe_notify_resume(struct pt_regs *regs)
+{
+ struct vm_area_struct *vma;
+ struct uprobe_task *utask;
+ struct mm_struct *mm;
+ struct uprobe *u = NULL;
+ unsigned long probept;
+
+ utask = current->utask;
+ mm = current->mm;
+ if (!utask || utask->state == UTASK_BP_HIT) {
+ probept = get_uprobe_bkpt_addr(regs);
+ down_read(&mm->mmap_sem);
+ vma = find_vma(mm, probept);
+ if (vma && valid_vma(vma, false))
+ u = find_uprobe(vma->vm_file->f_mapping->host,
+ probept - vma->vm_start +
+ (vma->vm_pgoff << PAGE_SHIFT));
+
+ srcu_read_unlock_raw(&uprobes_srcu,
+ current->uprobes_srcu_id);
+ current->uprobes_srcu_id = -1;
+ up_read(&mm->mmap_sem);
+ if (!u)
+ /* No matching uprobe; signal SIGTRAP. */
+ goto cleanup_ret;
+ if (!utask) {
+ utask = add_utask();
+ /* Cannot Allocate; re-execute the instruction. */
+ if (!utask)
+ goto cleanup_ret;
+ }
+ utask->active_uprobe = u;
+ handler_chain(u, regs);
+
+ if (u->flags & UPROBES_SKIP_SSTEP && can_skip_xol(regs, u))
+ goto cleanup_ret;
+
+ utask->state = UTASK_SSTEP;
+ if (!pre_ssout(u, regs, probept))
+ user_enable_single_step(current);
+ else
+ /* Cannot Singlestep; re-execute the instruction. */
+ goto cleanup_ret;
+ } else {
+ u = utask->active_uprobe;
+ if (utask->state == UTASK_SSTEP_ACK)
+ post_xol(u, regs);
+ else if (utask->state == UTASK_SSTEP_TRAPPED)
+ abort_xol(regs, u);
+ else
+ WARN_ON_ONCE(1);
+
+ put_uprobe(u);
+ utask->active_uprobe = NULL;
+ utask->state = UTASK_RUNNING;
+ user_disable_single_step(current);
+
+ spin_lock_irq(¤t->sighand->siglock);
+ recalc_sigpending(); /* see uprobe_deny_signal() */
+ spin_unlock_irq(¤t->sighand->siglock);
+ }
+ return;
+
+cleanup_ret:
+ if (utask) {
+ utask->active_uprobe = NULL;
+ utask->state = UTASK_RUNNING;
+ }
+ if (u) {
+ if (!(u->flags & UPROBES_SKIP_SSTEP))
+ set_instruction_pointer(regs, probept);
+
+ put_uprobe(u);
+ } else
+ send_sig(SIGTRAP, current, 0);
+}
+
+/*
+ * uprobe_bkpt_notifier gets called from interrupt context as part of
+ * notifier mechanism. Set TIF_UPROBE flag and indicate breakpoint hit.
+ */
+int uprobe_bkpt_notifier(struct pt_regs *regs)
+{
+ struct uprobe_task *utask;
+
+ if (!current->mm)
+ return 0;
+
+ utask = current->utask;
+ if (utask)
+ utask->state = UTASK_BP_HIT;
+
+ set_thread_flag(TIF_UPROBE);
+ current->uprobes_srcu_id = srcu_read_lock_raw(&uprobes_srcu);
+ return 1;
+}
+
+/*
+ * uprobe_post_notifier gets called in interrupt context as part of notifier
+ * mechanism. Set TIF_UPROBE flag and indicate completion of singlestep.
+ */
+int uprobe_post_notifier(struct pt_regs *regs)
+{
+ struct uprobe_task *utask = current->utask;
+
+ if (!current->mm || !utask || !utask->active_uprobe)
+ /* task is currently not uprobed */
+ return 0;
+
+ utask->state = UTASK_SSTEP_ACK;
+ set_thread_flag(TIF_UPROBE);
+ return 1;
+}
+
+struct notifier_block uprobe_exception_nb = {
+ .notifier_call = uprobe_exception_notify,
+ .priority = INT_MAX - 1, /* notified after kprobes, kgdb */
+};
+
static int __init init_uprobes(void)
{
int i;
mutex_init(&uprobes_mutex[i]);
mutex_init(&uprobes_mmap_mutex[i]);
}
- return 0;
+ init_srcu_struct(&uprobes_srcu);
+ return register_die_notifier(&uprobe_exception_nb);
}
static void __exit exit_uprobes(void)