mm, fork: make dup_mmap wait for mmap_sem for write killable

[karo-tx-linux.git] / kernel / fork.c

diff --git a/kernel/fork.c b/kernel/fork.c
index 3e8451527cbe2ac87eed8bc3c9d64aeade9d10ef..47887bba944fe61f3284057ead7f9f9637ec3d19 100644 (file)
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -340,13 +340,14 @@ void set_task_stack_end_magic(struct task_struct *tsk)
        *stackend = STACK_END_MAGIC;    /* for overflow detection */
 }
 
-static struct task_struct *dup_task_struct(struct task_struct *orig)
+static struct task_struct *dup_task_struct(struct task_struct *orig, int node)
 {
        struct task_struct *tsk;
        struct thread_info *ti;
-       int node = tsk_fork_get_node(orig);
        int err;
 
+       if (node == NUMA_NO_NODE)
+               node = tsk_fork_get_node(orig);
        tsk = alloc_task_struct_node(node);
        if (!tsk)
                return NULL;
@@ -413,7 +414,10 @@ static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm)
        unsigned long charge;
 
        uprobe_start_dup_mmap();
-       down_write(&oldmm->mmap_sem);
+       if (down_write_killable(&oldmm->mmap_sem)) {
+               retval = -EINTR;
+               goto fail_uprobe_end;
+       }
        flush_cache_dup_mm(oldmm);
        uprobe_dup_mmap(oldmm, mm);
        /*
@@ -525,6 +529,7 @@ out:
        up_write(&mm->mmap_sem);
        flush_tlb_mm(oldmm);
        up_write(&oldmm->mmap_sem);
+fail_uprobe_end:
        uprobe_end_dup_mmap();
        return retval;
 fail_nomem_anon_vma_fork:
@@ -699,6 +704,26 @@ void __mmdrop(struct mm_struct *mm)
 }
 EXPORT_SYMBOL_GPL(__mmdrop);
 
+static inline void __mmput(struct mm_struct *mm)
+{
+       VM_BUG_ON(atomic_read(&mm->mm_users));
+
+       uprobe_clear_state(mm);
+       exit_aio(mm);
+       ksm_exit(mm);
+       khugepaged_exit(mm); /* must run before exit_mmap */
+       exit_mmap(mm);
+       set_mm_exe_file(mm, NULL);
+       if (!list_empty(&mm->mmlist)) {
+               spin_lock(&mmlist_lock);
+               list_del(&mm->mmlist);
+               spin_unlock(&mmlist_lock);
+       }
+       if (mm->binfmt)
+               module_put(mm->binfmt->module);
+       mmdrop(mm);
+}
+
 /*
  * Decrement the use count and release all resources for an mm.
  */
@@ -706,24 +731,24 @@ void mmput(struct mm_struct *mm)
 {
        might_sleep();
 
+       if (atomic_dec_and_test(&mm->mm_users))
+               __mmput(mm);
+}
+EXPORT_SYMBOL_GPL(mmput);
+
+static void mmput_async_fn(struct work_struct *work)
+{
+       struct mm_struct *mm = container_of(work, struct mm_struct, async_put_work);
+       __mmput(mm);
+}
+
+void mmput_async(struct mm_struct *mm)
+{
        if (atomic_dec_and_test(&mm->mm_users)) {
-               uprobe_clear_state(mm);
-               exit_aio(mm);
-               ksm_exit(mm);
-               khugepaged_exit(mm); /* must run before exit_mmap */
-               exit_mmap(mm);
-               set_mm_exe_file(mm, NULL);
-               if (!list_empty(&mm->mmlist)) {
-                       spin_lock(&mmlist_lock);
-                       list_del(&mm->mmlist);
-                       spin_unlock(&mmlist_lock);
-               }
-               if (mm->binfmt)
-                       module_put(mm->binfmt->module);
-               mmdrop(mm);
+               INIT_WORK(&mm->async_put_work, mmput_async_fn);
+               schedule_work(&mm->async_put_work);
        }
 }
-EXPORT_SYMBOL_GPL(mmput);
 
 /**
  * set_mm_exe_file - change a reference to the mm's executable file
@@ -1256,7 +1281,8 @@ static struct task_struct *copy_process(unsigned long clone_flags,
                                        int __user *child_tidptr,
                                        struct pid *pid,
                                        int trace,
-                                       unsigned long tls)
+                                       unsigned long tls,
+                                       int node)
 {
        int retval;
        struct task_struct *p;
@@ -1308,7 +1334,7 @@ static struct task_struct *copy_process(unsigned long clone_flags,
                goto fork_out;
 
        retval = -ENOMEM;
-       p = dup_task_struct(current);
+       p = dup_task_struct(current, node);
        if (!p)
                goto fork_out;
 
@@ -1470,7 +1496,7 @@ static struct task_struct *copy_process(unsigned long clone_flags,
                pid = alloc_pid(p->nsproxy->pid_ns_for_children);
                if (IS_ERR(pid)) {
                        retval = PTR_ERR(pid);
-                       goto bad_fork_cleanup_io;
+                       goto bad_fork_cleanup_thread;
                }
        }
 
@@ -1632,6 +1658,8 @@ bad_fork_cancel_cgroup:
 bad_fork_free_pid:
        if (pid != &init_struct_pid)
                free_pid(pid);
+bad_fork_cleanup_thread:
+       exit_thread(p);
 bad_fork_cleanup_io:
        if (p->io_context)
                exit_io_context(p);
@@ -1684,7 +1712,8 @@ static inline void init_idle_pids(struct pid_link *links)
 struct task_struct *fork_idle(int cpu)
 {
        struct task_struct *task;
-       task = copy_process(CLONE_VM, 0, 0, NULL, &init_struct_pid, 0, 0);
+       task = copy_process(CLONE_VM, 0, 0, NULL, &init_struct_pid, 0, 0,
+                           cpu_to_node(cpu));
        if (!IS_ERR(task)) {
                init_idle_pids(task->pids);
                init_idle(task, cpu);
@@ -1729,7 +1758,7 @@ long _do_fork(unsigned long clone_flags,
        }
 
        p = copy_process(clone_flags, stack_start, stack_size,
-                        child_tidptr, NULL, trace, tls);
+                        child_tidptr, NULL, trace, tls, NUMA_NO_NODE);
        /*
         * Do this prior waking up the new thread - the thread pointer
         * might get invalid after that point, if the thread exits quickly.