sl[au]b: allocate objects from memcg cache

We are able to match a cache allocation to a particular memcg.  If the
task doesn't change groups during the allocation itself - a rare event,
this will give us a good picture about who is the first group to touch a
cache page.

This patch uses the now available infrastructure by calling
memcg_kmem_get_cache() before all the cache allocations.

Signed-off-by: Glauber Costa <glommer@parallels.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Frederic Weisbecker <fweisbec@redhat.com>
Cc: Greg Thelen <gthelen@google.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: JoonSoo Kim <js1304@gmail.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Michal Hocko <mhocko@suse.cz>
Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: Rik van Riel <riel@redhat.com>
Cc: Suleiman Souhlal <suleiman@google.com>
Cc: Tejun Heo <tj@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/include/linux/slub_def.h b/include/linux/slub_def.h
index 961e72eab907b38317d31c7e3e00cae18d087b28..364ba6c9fe2173011a1889ff26d745986124bd29 100644 (file)
--- a/include/linux/slub_def.h
+++ b/include/linux/slub_def.h
@@ -225,7 +225,10 @@ void *__kmalloc(size_t size, gfp_t flags);
 static __always_inline void *
 kmalloc_order(size_t size, gfp_t flags, unsigned int order)
 {
-       void *ret = (void *) __get_free_pages(flags | __GFP_COMP, order);
+       void *ret;
+
+       flags |= (__GFP_COMP | __GFP_KMEMCG);
+       ret = (void *) __get_free_pages(flags, order);
        kmemleak_alloc(ret, size, 1, flags);
        return ret;
 }

diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index b095f7fa97c0a44bccf44cf9866e6911435701c7..9ed641ed72bb146d335cc5091c52203f67ed62f2 100644 (file)
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -3087,6 +3087,9 @@ static struct kmem_cache *kmem_cache_dup(struct mem_cgroup *memcg,
        new = kmem_cache_create_memcg(memcg, name, s->object_size, s->align,
                                      (s->flags & ~SLAB_PANIC), s->ctor);
 
+       if (new)
+               new->allocflags |= __GFP_KMEMCG;
+
        kfree(name);
        return new;
 }

diff --git a/mm/slab.c b/mm/slab.c
index 4a586540c0bcda288580b4ab21e0b1548bedeeee..6c9797698753cbf8eb7120baf13181b10a4fa5b8 100644 (file)
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -1974,7 +1974,7 @@ static void kmem_freepages(struct kmem_cache *cachep, void *addr)
        }
        if (current->reclaim_state)
                current->reclaim_state->reclaimed_slab += nr_freed;
-       free_pages((unsigned long)addr, cachep->gfporder);
+       free_memcg_kmem_pages((unsigned long)addr, cachep->gfporder);
 }
 
 static void kmem_rcu_free(struct rcu_head *head)
@@ -3538,6 +3538,8 @@ slab_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid,
        if (slab_should_failslab(cachep, flags))
                return NULL;
 
+       cachep = memcg_kmem_get_cache(cachep, flags);
+
        cache_alloc_debugcheck_before(cachep, flags);
        local_irq_save(save_flags);
 
@@ -3623,6 +3625,8 @@ slab_alloc(struct kmem_cache *cachep, gfp_t flags, unsigned long caller)
        if (slab_should_failslab(cachep, flags))
                return NULL;
 
+       cachep = memcg_kmem_get_cache(cachep, flags);
+
        cache_alloc_debugcheck_before(cachep, flags);
        local_irq_save(save_flags);
        objp = __do_cache_alloc(cachep, flags);

diff --git a/mm/slub.c b/mm/slub.c
index 7edb99e6bb057ba9a2273bd98171783fd2a040e8..3c3a79cd54fe78a87f2876e87dfb86a8af59064c 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -1407,7 +1407,7 @@ static void __free_slab(struct kmem_cache *s, struct page *page)
        reset_page_mapcount(page);
        if (current->reclaim_state)
                current->reclaim_state->reclaimed_slab += pages;
-       __free_pages(page, order);
+       __free_memcg_kmem_pages(page, order);
 }
 
 #define need_reserve_slab_rcu                                          \
@@ -2323,6 +2323,7 @@ static __always_inline void *slab_alloc_node(struct kmem_cache *s,
        if (slab_pre_alloc_hook(s, gfpflags))
                return NULL;
 
+       s = memcg_kmem_get_cache(s, gfpflags);
 redo:
 
        /*
@@ -3346,7 +3347,7 @@ static void *kmalloc_large_node(size_t size, gfp_t flags, int node)
        struct page *page;
        void *ptr = NULL;
 
-       flags |= __GFP_COMP | __GFP_NOTRACK;
+       flags |= __GFP_COMP | __GFP_NOTRACK | __GFP_KMEMCG;
        page = alloc_pages_node(node, flags, get_order(size));
        if (page)
                ptr = page_address(page);
@@ -3452,7 +3453,7 @@ void kfree(const void *x)
        if (unlikely(!PageSlab(page))) {
                BUG_ON(!PageCompound(page));
                kmemleak_free(x);
-               __free_pages(page, compound_order(page));
+               __free_memcg_kmem_pages(page, compound_order(page));
                return;
        }
        slab_free(page->slab_cache, page, object, _RET_IP_);