From e9f709f4d5eb4078e9d85e1c0f9d874f23f506af Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Thu, 29 Nov 2012 14:16:42 +1100 Subject: [PATCH] sl[au]b: always get the cache from its page in kmem_cache_free() struct page already has this information. If we start chaining caches, this information will always be more trustworthy than whatever is passed into the function. Signed-off-by: Glauber Costa Cc: Christoph Lameter Cc: David Rientjes Cc: Frederic Weisbecker Cc: Greg Thelen Cc: Johannes Weiner Cc: JoonSoo Kim Cc: KAMEZAWA Hiroyuki Cc: Mel Gorman Cc: Michal Hocko Cc: Pekka Enberg Cc: Rik van Riel Cc: Suleiman Souhlal Cc: Tejun Heo Signed-off-by: Andrew Morton --- include/linux/memcontrol.h | 5 +++++ mm/slab.c | 6 +++++- mm/slab.h | 39 ++++++++++++++++++++++++++++++++++++++ mm/slob.c | 2 +- mm/slub.c | 15 +++------------ 5 files changed, 53 insertions(+), 14 deletions(-) diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 146e5d7b1cd1..b8ee2e31c3ba 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -547,6 +547,11 @@ memcg_kmem_get_cache(struct kmem_cache *cachep, gfp_t gfp) return __memcg_kmem_get_cache(cachep, gfp); } #else +static inline bool memcg_kmem_enabled(void) +{ + return false; +} + static inline bool memcg_kmem_newpage_charge(gfp_t gfp, struct mem_cgroup **memcg, int order) { diff --git a/mm/slab.c b/mm/slab.c index dd9516bbe998..4a586540c0bc 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -87,7 +87,6 @@ */ #include -#include "slab.h" #include #include #include @@ -128,6 +127,8 @@ #include "internal.h" +#include "slab.h" + /* * DEBUG - 1 for kmem_cache_create() to honour; SLAB_RED_ZONE & SLAB_POISON. * 0 for faster, smaller code (especially in the critical paths). @@ -3935,6 +3936,9 @@ EXPORT_SYMBOL(__kmalloc); void kmem_cache_free(struct kmem_cache *cachep, void *objp) { unsigned long flags; + cachep = cache_from_obj(cachep, objp); + if (!cachep) + return; local_irq_save(flags); debug_check_no_locks_freed(objp, cachep->object_size); diff --git a/mm/slab.h b/mm/slab.h index 22eb5aa2f810..fb1c4c4d4965 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -108,6 +108,13 @@ static inline bool cache_match_memcg(struct kmem_cache *cachep, return (is_root_cache(cachep) && !memcg) || (cachep->memcg_params->memcg == memcg); } + +static inline bool slab_equal_or_root(struct kmem_cache *s, + struct kmem_cache *p) +{ + return (p == s) || + (s->memcg_params && (p == s->memcg_params->root_cache)); +} #else static inline bool is_root_cache(struct kmem_cache *s) { @@ -119,5 +126,37 @@ static inline bool cache_match_memcg(struct kmem_cache *cachep, { return true; } + +static inline bool slab_equal_or_root(struct kmem_cache *s, + struct kmem_cache *p) +{ + return true; +} #endif + +static inline struct kmem_cache *cache_from_obj(struct kmem_cache *s, void *x) +{ + struct kmem_cache *cachep; + struct page *page; + + /* + * When kmemcg is not being used, both assignments should return the + * same value. but we don't want to pay the assignment price in that + * case. If it is not compiled in, the compiler should be smart enough + * to not do even the assignment. In that case, slab_equal_or_root + * will also be a constant. + */ + if (!memcg_kmem_enabled() && !unlikely(s->flags & SLAB_DEBUG_FREE)) + return s; + + page = virt_to_head_page(x); + cachep = page->slab_cache; + if (slab_equal_or_root(cachep, s)) + return cachep; + + pr_err("%s: Wrong slab cache. %s but object is from %s\n", + __FUNCTION__, cachep->name, s->name); + WARN_ON_ONCE(1); + return s; +} #endif diff --git a/mm/slob.c b/mm/slob.c index 87e16c4d9143..52381c41f2b8 100644 --- a/mm/slob.c +++ b/mm/slob.c @@ -58,7 +58,6 @@ #include #include -#include "slab.h" #include #include /* struct reclaim_state */ @@ -73,6 +72,7 @@ #include +#include "slab.h" /* * slob_block has a field 'units', which indicates size of block if +ve, * or offset of next block if -ve (in SLOB_UNITs). diff --git a/mm/slub.c b/mm/slub.c index 50ee6e334b48..7edb99e6bb05 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -2611,19 +2611,10 @@ redo: void kmem_cache_free(struct kmem_cache *s, void *x) { - struct page *page; - - page = virt_to_head_page(x); - - if (kmem_cache_debug(s) && page->slab_cache != s) { - pr_err("kmem_cache_free: Wrong slab cache. %s but object" - " is from %s\n", page->slab_cache->name, s->name); - WARN_ON_ONCE(1); + s = cache_from_obj(s, x); + if (!s) return; - } - - slab_free(s, page, x, _RET_IP_); - + slab_free(s, virt_to_head_page(x), x, _RET_IP_); trace_kmem_cache_free(_RET_IP_, x); } EXPORT_SYMBOL(kmem_cache_free); -- 2.39.5