mm: get rid of __GFP_KMEMCG

Currently to allocate a page that should be charged to kmemcg (e.g.
threadinfo), we pass __GFP_KMEMCG flag to the page allocator.  The page
allocated is then to be freed by free_memcg_kmem_pages.  Apart from
looking asymmetrical, this also requires intrusion to the general
allocation path.  So let's introduce separate functions that will
alloc/free pages charged to kmemcg.

The new functions are called alloc_kmem_pages and free_kmem_pages.  They
should be used when the caller actually would like to use kmalloc, but has
to fall back to the page allocator for the allocation is large.  They only
differ from alloc_pages and free_pages in that besides allocating or
freeing pages they also charge them to the kmem resource counter of the
current memory cgroup.

Signed-off-by: Vladimir Davydov <vdavydov@parallels.com>
Acked-by: Greg Thelen <gthelen@google.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Michal Hocko <mhocko@suse.cz>
Cc: Glauber Costa <glommer@gmail.com>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: Pekka Enberg <penberg@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/include/linux/gfp.h b/include/linux/gfp.h
index 39b81dc7d01acdb28e6e888fea3399f7866057f8..d382db71e300a8a7bba289a8605efcc1b4518d1f 100644 (file)
--- a/include/linux/gfp.h
+++ b/include/linux/gfp.h
@@ -31,7 +31,6 @@ struct vm_area_struct;
 #define ___GFP_HARDWALL                0x20000u
 #define ___GFP_THISNODE                0x40000u
 #define ___GFP_RECLAIMABLE     0x80000u
-#define ___GFP_KMEMCG          0x100000u
 #define ___GFP_NOTRACK         0x200000u
 #define ___GFP_NO_KSWAPD       0x400000u
 #define ___GFP_OTHER_NODE      0x800000u
@@ -91,7 +90,6 @@ struct vm_area_struct;
 
 #define __GFP_NO_KSWAPD        ((__force gfp_t)___GFP_NO_KSWAPD)
 #define __GFP_OTHER_NODE ((__force gfp_t)___GFP_OTHER_NODE) /* On behalf of other node */
-#define __GFP_KMEMCG   ((__force gfp_t)___GFP_KMEMCG) /* Allocation comes from a memcg-accounted resource */
 #define __GFP_WRITE    ((__force gfp_t)___GFP_WRITE)   /* Allocator intends to dirty page */
 
 /*
@@ -353,6 +351,10 @@ extern struct page *alloc_pages_vma(gfp_t gfp_mask, int order,
 #define alloc_page_vma_node(gfp_mask, vma, addr, node)         \
        alloc_pages_vma(gfp_mask, 0, vma, addr, node)
 
+extern struct page *alloc_kmem_pages(gfp_t gfp_mask, unsigned int order);
+extern struct page *alloc_kmem_pages_node(int nid, gfp_t gfp_mask,
+                                         unsigned int order);
+
 extern unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order);
 extern unsigned long get_zeroed_page(gfp_t gfp_mask);
 
@@ -372,8 +374,8 @@ extern void free_pages(unsigned long addr, unsigned int order);
 extern void free_hot_cold_page(struct page *page, int cold);
 extern void free_hot_cold_page_list(struct list_head *list, int cold);
 
-extern void __free_memcg_kmem_pages(struct page *page, unsigned int order);
-extern void free_memcg_kmem_pages(unsigned long addr, unsigned int order);
+extern void __free_kmem_pages(struct page *page, unsigned int order);
+extern void free_kmem_pages(unsigned long addr, unsigned int order);
 
 #define __free_page(page) __free_pages((page), 0)
 #define free_page(addr) free_pages((addr), 0)

diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index 96e5d2573eb00580b7aaaaeba366c67361f6b185..5155d09e749d216e4b1a6571842d006d89c567a9 100644 (file)
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -537,7 +537,7 @@ memcg_kmem_newpage_charge(gfp_t gfp, struct mem_cgroup **memcg, int order)
         * res_counter_charge_nofail, but we hope those allocations are rare,
         * and won't be worth the trouble.
         */
-       if (!(gfp & __GFP_KMEMCG) || (gfp & __GFP_NOFAIL))
+       if (gfp & __GFP_NOFAIL)
                return true;
        if (in_interrupt() || (!current->mm) || (current->flags & PF_KTHREAD))
                return true;

diff --git a/include/linux/slab.h b/include/linux/slab.h
index 307bfbe62387ad5872c16bba320263fe83f1322d..a6aab2c0dfc59cee228f3c6059f7f3eb78d7e1e9 100644 (file)
--- a/include/linux/slab.h
+++ b/include/linux/slab.h
@@ -369,16 +369,7 @@ kmem_cache_alloc_node_trace(struct kmem_cache *s,
 #include <linux/slub_def.h>
 #endif
 
-static __always_inline void *
-kmalloc_order(size_t size, gfp_t flags, unsigned int order)
-{
-       void *ret;
-
-       flags |= (__GFP_COMP | __GFP_KMEMCG);
-       ret = (void *) __get_free_pages(flags, order);
-       kmemleak_alloc(ret, size, 1, flags);
-       return ret;
-}
+extern void *kmalloc_order(size_t size, gfp_t flags, unsigned int order);
 
 #ifdef CONFIG_TRACING
 extern void *kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order);

diff --git a/include/linux/thread_info.h b/include/linux/thread_info.h
index fddbe2023a5d568717b3b90eabc692bb4612f9bd..1807bb1948169735b6b1cbb0a9da051b87a6d507 100644 (file)
--- a/include/linux/thread_info.h
+++ b/include/linux/thread_info.h
@@ -61,8 +61,6 @@ extern long do_no_restart_syscall(struct restart_block *parm);
 # define THREADINFO_GFP                (GFP_KERNEL | __GFP_NOTRACK)
 #endif
 
-#define THREADINFO_GFP_ACCOUNTED (THREADINFO_GFP | __GFP_KMEMCG)
-
 /*
  * flag set/clear/test wrappers
  * - pass TIF_xxxx constants to these functions

diff --git a/include/trace/events/gfpflags.h b/include/trace/events/gfpflags.h
index 1eddbf1557f2bfab809a766fde5cec4f6038634e..d6fd8e5b14b76c41bfd532c3fa86255e4e92b0f3 100644 (file)
--- a/include/trace/events/gfpflags.h
+++ b/include/trace/events/gfpflags.h
@@ -34,7 +34,6 @@
        {(unsigned long)__GFP_HARDWALL,         "GFP_HARDWALL"},        \
        {(unsigned long)__GFP_THISNODE,         "GFP_THISNODE"},        \
        {(unsigned long)__GFP_RECLAIMABLE,      "GFP_RECLAIMABLE"},     \
-       {(unsigned long)__GFP_KMEMCG,           "GFP_KMEMCG"},          \
        {(unsigned long)__GFP_MOVABLE,          "GFP_MOVABLE"},         \
        {(unsigned long)__GFP_NOTRACK,          "GFP_NOTRACK"},         \
        {(unsigned long)__GFP_NO_KSWAPD,        "GFP_NO_KSWAPD"},       \

diff --git a/kernel/fork.c b/kernel/fork.c
index 54a8d26f612f76f17cfb883e29786df98a9de0e0..59e3dcc5b8f2d49dcc6b22d57a31630a050e95a0 100644 (file)
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -150,15 +150,15 @@ void __weak arch_release_thread_info(struct thread_info *ti)
 static struct thread_info *alloc_thread_info_node(struct task_struct *tsk,
                                                  int node)
 {
-       struct page *page = alloc_pages_node(node, THREADINFO_GFP_ACCOUNTED,
-                                            THREAD_SIZE_ORDER);
+       struct page *page = alloc_kmem_pages_node(node, THREADINFO_GFP,
+                                                 THREAD_SIZE_ORDER);
 
        return page ? page_address(page) : NULL;
 }
 
 static inline void free_thread_info(struct thread_info *ti)
 {
-       free_memcg_kmem_pages((unsigned long)ti, THREAD_SIZE_ORDER);
+       free_kmem_pages((unsigned long)ti, THREAD_SIZE_ORDER);
 }
 # else
 static struct kmem_cache *thread_info_cache;

diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 38e405fbe8660847ab1d8b023eccb86686700a64..5cbe429f15a4efb299c2e88edf7f4829aef175e0 100644 (file)
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -3531,11 +3531,12 @@ __memcg_kmem_newpage_charge(gfp_t gfp, struct mem_cgroup **_memcg, int order)
        /*
         * Disabling accounting is only relevant for some specific memcg
         * internal allocations. Therefore we would initially not have such
-        * check here, since direct calls to the page allocator that are marked
-        * with GFP_KMEMCG only happen outside memcg core. We are mostly
-        * concerned with cache allocations, and by having this test at
-        * memcg_kmem_get_cache, we are already able to relay the allocation to
-        * the root cache and bypass the memcg cache altogether.
+        * check here, since direct calls to the page allocator that are
+        * accounted to kmemcg (alloc_kmem_pages and friends) only happen
+        * outside memcg core. We are mostly concerned with cache allocations,
+        * and by having this test at memcg_kmem_get_cache, we are already able
+        * to relay the allocation to the root cache and bypass the memcg cache
+        * altogether.
         *
         * There is one exception, though: the SLUB allocator does not create
         * large order caches, but rather service large kmallocs directly from

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 5dba2933c9c018b59178a5e88df54db01a7fe41a..7cfdcd808f520ab91f527f750ced42ec338d6a9b 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -2697,7 +2697,6 @@ __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order,
        int migratetype = allocflags_to_migratetype(gfp_mask);
        unsigned int cpuset_mems_cookie;
        int alloc_flags = ALLOC_WMARK_LOW|ALLOC_CPUSET|ALLOC_FAIR;
-       struct mem_cgroup *memcg = NULL;
 
        gfp_mask &= gfp_allowed_mask;
 
@@ -2716,13 +2715,6 @@ __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order,
        if (unlikely(!zonelist->_zonerefs->zone))
                return NULL;
 
-       /*
-        * Will only have any effect when __GFP_KMEMCG is set.  This is
-        * verified in the (always inline) callee
-        */
-       if (!memcg_kmem_newpage_charge(gfp_mask, &memcg, order))
-               return NULL;
-
 retry_cpuset:
        cpuset_mems_cookie = read_mems_allowed_begin();
 
@@ -2782,8 +2774,6 @@ out:
        if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie)))
                goto retry_cpuset;
 
-       memcg_kmem_commit_charge(page, memcg, order);
-
        return page;
 }
 EXPORT_SYMBOL(__alloc_pages_nodemask);
@@ -2837,27 +2827,51 @@ void free_pages(unsigned long addr, unsigned int order)
 EXPORT_SYMBOL(free_pages);
 
 /*
- * __free_memcg_kmem_pages and free_memcg_kmem_pages will free
- * pages allocated with __GFP_KMEMCG.
+ * alloc_kmem_pages charges newly allocated pages to the kmem resource counter
+ * of the current memory cgroup.
  *
- * Those pages are accounted to a particular memcg, embedded in the
- * corresponding page_cgroup. To avoid adding a hit in the allocator to search
- * for that information only to find out that it is NULL for users who have no
- * interest in that whatsoever, we provide these functions.
- *
- * The caller knows better which flags it relies on.
+ * It should be used when the caller would like to use kmalloc, but since the
+ * allocation is large, it has to fall back to the page allocator.
+ */
+struct page *alloc_kmem_pages(gfp_t gfp_mask, unsigned int order)
+{
+       struct page *page;
+       struct mem_cgroup *memcg = NULL;
+
+       if (!memcg_kmem_newpage_charge(gfp_mask, &memcg, order))
+               return NULL;
+       page = alloc_pages(gfp_mask, order);
+       memcg_kmem_commit_charge(page, memcg, order);
+       return page;
+}
+
+struct page *alloc_kmem_pages_node(int nid, gfp_t gfp_mask, unsigned int order)
+{
+       struct page *page;
+       struct mem_cgroup *memcg = NULL;
+
+       if (!memcg_kmem_newpage_charge(gfp_mask, &memcg, order))
+               return NULL;
+       page = alloc_pages_node(nid, gfp_mask, order);
+       memcg_kmem_commit_charge(page, memcg, order);
+       return page;
+}
+
+/*
+ * __free_kmem_pages and free_kmem_pages will free pages allocated with
+ * alloc_kmem_pages.
  */
-void __free_memcg_kmem_pages(struct page *page, unsigned int order)
+void __free_kmem_pages(struct page *page, unsigned int order)
 {
        memcg_kmem_uncharge_pages(page, order);
        __free_pages(page, order);
 }
 
-void free_memcg_kmem_pages(unsigned long addr, unsigned int order)
+void free_kmem_pages(unsigned long addr, unsigned int order)
 {
        if (addr != 0) {
                VM_BUG_ON(!virt_addr_valid((void *)addr));
-               __free_memcg_kmem_pages(virt_to_page((void *)addr), order);
+               __free_kmem_pages(virt_to_page((void *)addr), order);
        }
 }
 

diff --git a/mm/slab_common.c b/mm/slab_common.c
index 06f0c6125632883fead074d2b3020f320135fd5d..e8670628f4b6aac7d8b4f9909ee517fa1e5c6c62 100644 (file)
--- a/mm/slab_common.c
+++ b/mm/slab_common.c
@@ -582,6 +582,18 @@ void __init create_kmalloc_caches(unsigned long flags)
 }
 #endif /* !CONFIG_SLOB */
 
+void *kmalloc_order(size_t size, gfp_t flags, unsigned int order)
+{
+       void *ret;
+       struct page *page;
+
+       flags |= __GFP_COMP;
+       page = alloc_kmem_pages(flags, order);
+       ret = page ? page_address(page) : NULL;
+       kmemleak_alloc(ret, size, 1, flags);
+       return ret;
+}
+
 #ifdef CONFIG_TRACING
 void *kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order)
 {

diff --git a/mm/slub.c b/mm/slub.c
index fc9831851be6a5ee03347dc868832894f66a927d..ddb60795f373eabdf06daee87d3c604021cdae09 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -3311,8 +3311,8 @@ static void *kmalloc_large_node(size_t size, gfp_t flags, int node)
        struct page *page;
        void *ptr = NULL;
 
-       flags |= __GFP_COMP | __GFP_NOTRACK | __GFP_KMEMCG;
-       page = alloc_pages_node(node, flags, get_order(size));
+       flags |= __GFP_COMP | __GFP_NOTRACK;
+       page = alloc_kmem_pages_node(node, flags, get_order(size));
        if (page)
                ptr = page_address(page);
 
@@ -3381,7 +3381,7 @@ void kfree(const void *x)
        if (unlikely(!PageSlab(page))) {
                BUG_ON(!PageCompound(page));
                kfree_hook(x);
-               __free_memcg_kmem_pages(page, compound_order(page));
+               __free_kmem_pages(page, compound_order(page));
                return;
        }
        slab_free(page->slab_cache, page, object, _RET_IP_);