memcg: skip memcg kmem allocations in specified code regions

Create a mechanism that skip memcg allocations during certain pieces of
our core code.  It basically works in the same way as
preempt_disable()/preempt_enable(): By marking a region under which all
allocations will be accounted to the root memcg.

We need this to prevent races in early cache creation, when we
allocate data using caches that are not necessarily created already.

Signed-off-by: Glauber Costa <glommer@parallels.com>
yCc: Christoph Lameter <cl@linux.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Frederic Weisbecker <fweisbec@redhat.com>
Cc: Greg Thelen <gthelen@google.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: JoonSoo Kim <js1304@gmail.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Michal Hocko <mhocko@suse.cz>
Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: Rik van Riel <riel@redhat.com>
Cc: Suleiman Souhlal <suleiman@google.com>
Cc: Tejun Heo <tj@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/include/linux/sched.h b/include/linux/sched.h
index 68fa2f8f1efe30e56cff34072216b8f7b4d547d7..8f520d4b9d2ee3d64b97a89327bd2678f5b33be9 100644 (file)
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1577,6 +1577,7 @@ struct task_struct {
                unsigned long nr_pages; /* uncharged usage */
                unsigned long memsw_nr_pages; /* uncharged mem+swap usage */
        } memcg_batch;
+       unsigned int memcg_kmem_skip_account;
 #endif
 #ifdef CONFIG_HAVE_HW_BREAKPOINT
        atomic_t ptrace_bp_refcnt;

diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index bf45c67820fd817273167b844b4d3b61f9fc470f..1555edb7b8b6756a4a30846d02f64c3e2d0836c0 100644 (file)
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -2993,6 +2993,41 @@ out:
        kfree(s->memcg_params);
 }
 
+/*
+ * During the creation a new cache, we need to disable our accounting mechanism
+ * altogether. This is true even if we are not creating, but rather just
+ * enqueing new caches to be created.
+ *
+ * This is because that process will trigger allocations; some visible, like
+ * explicit kmallocs to auxiliary data structures, name strings and internal
+ * cache structures; some well concealed, like INIT_WORK() that can allocate
+ * objects during debug.
+ *
+ * If any allocation happens during memcg_kmem_get_cache, we will recurse back
+ * to it. This may not be a bounded recursion: since the first cache creation
+ * failed to complete (waiting on the allocation), we'll just try to create the
+ * cache again, failing at the same point.
+ *
+ * memcg_kmem_get_cache is prepared to abort after seeing a positive count of
+ * memcg_kmem_skip_account. So we enclose anything that might allocate memory
+ * inside the following two functions.
+ */
+static inline void memcg_stop_kmem_account(void)
+{
+       if (!current->mm)
+               return;
+
+       current->memcg_kmem_skip_account++;
+}
+
+static inline void memcg_resume_kmem_account(void)
+{
+       if (!current->mm)
+               return;
+
+       current->memcg_kmem_skip_account--;
+}
+
 static char *memcg_cache_name(struct mem_cgroup *memcg, struct kmem_cache *s)
 {
        char *name;
@@ -3051,7 +3086,10 @@ static struct kmem_cache *memcg_create_kmem_cache(struct mem_cgroup *memcg,
        if (new_cachep)
                goto out;
 
+       /* Don't block progress to enqueue caches for internal infrastructure */
+       memcg_stop_kmem_account();
        new_cachep = kmem_cache_dup(memcg, cachep);
+       memcg_resume_kmem_account();
 
        if (new_cachep == NULL) {
                new_cachep = cachep;
@@ -3093,8 +3131,8 @@ static void memcg_create_cache_work_func(struct work_struct *w)
  * Enqueue the creation of a per-memcg kmem_cache.
  * Called with rcu_read_lock.
  */
-static void memcg_create_cache_enqueue(struct mem_cgroup *memcg,
-                                      struct kmem_cache *cachep)
+static void __memcg_create_cache_enqueue(struct mem_cgroup *memcg,
+                                        struct kmem_cache *cachep)
 {
        struct create_work *cw;
 
@@ -3115,6 +3153,24 @@ static void memcg_create_cache_enqueue(struct mem_cgroup *memcg,
        schedule_work(&cw->work);
 }
 
+static void memcg_create_cache_enqueue(struct mem_cgroup *memcg,
+                                      struct kmem_cache *cachep)
+{
+       /*
+        * We need to stop accounting when we kmalloc, because if the
+        * corresponding kmalloc cache is not yet created, the first allocation
+        * in __memcg_create_cache_enqueue will recurse.
+        *
+        * However, it is better to enclose the whole function. Depending on
+        * the debugging options enabled, INIT_WORK(), for instance, can
+        * trigger an allocation. This too, will make us recurse. Because at
+        * this point we can't allow ourselves back into memcg_kmem_get_cache,
+        * the safest choice is to do it like this, wrapping the whole function.
+        */
+       memcg_stop_kmem_account();
+       __memcg_create_cache_enqueue(memcg, cachep);
+       memcg_resume_kmem_account();
+}
 /*
  * Return the kmem_cache we're supposed to use for a slab allocation.
  * We try to use the current memcg's version of the cache.
@@ -3137,6 +3193,9 @@ struct kmem_cache *__memcg_kmem_get_cache(struct kmem_cache *cachep,
        VM_BUG_ON(!cachep->memcg_params);
        VM_BUG_ON(!cachep->memcg_params->is_root_cache);
 
+       if (!current->mm || current->memcg_kmem_skip_account)
+               return cachep;
+
        rcu_read_lock();
        memcg = mem_cgroup_from_task(rcu_dereference(current->mm->owner));
        rcu_read_unlock();