From 0b3785f7e8a6a1d2d243738834e1c45c4bb5f261 Mon Sep 17 00:00:00 2001 From: Vladimir Davydov Date: Thu, 26 Jun 2014 10:42:31 +1000 Subject: [PATCH] memcg: wait for kfree's to finish before destroying cache kmem_cache_free doesn't expect that the cache can be destroyed as soon as the object is freed, e.g. SLUB's implementation may want to update cache stats after putting the object to the free list. Therefore we should wait for all kmem_cache_free's to finish before proceeding to cache destruction. Since both SLAB and SLUB versions of kmem_cache_free are non-preemptable, we wait for rcu-sched grace period to elapse. Signed-off-by: Vladimir Davydov Cc: Michal Hocko Cc: Johannes Weiner Cc: Christoph Lameter Cc: Pekka Enberg Cc: David Rientjes Cc: Joonsoo Kim Signed-off-by: Andrew Morton --- include/linux/slab.h | 6 ++---- mm/memcontrol.c | 34 ++++++++++++++++++++++++++++++++-- 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/include/linux/slab.h b/include/linux/slab.h index d99d5212b815..68b1feaba9d6 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -532,11 +532,9 @@ static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node) */ struct memcg_cache_params { bool is_root_cache; + struct rcu_head rcu_head; union { - struct { - struct rcu_head rcu_head; - struct kmem_cache *memcg_caches[0]; - }; + struct kmem_cache *memcg_caches[0]; struct { struct mem_cgroup *memcg; struct list_head list; diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 2a36a3977e58..8f8f01b34256 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -3194,6 +3194,14 @@ static void memcg_unregister_cache_func(struct work_struct *work) mutex_unlock(&memcg_slab_mutex); } +static void memcg_unregister_cache_rcu_func(struct rcu_head *rcu) +{ + struct memcg_cache_params *params = + container_of(rcu, struct memcg_cache_params, rcu_head); + + schedule_work(¶ms->unregister_work); +} + /* * During the creation a new cache, we need to disable our accounting mechanism * altogether. This is true even if we are not creating, but rather just @@ -3249,6 +3257,7 @@ static void memcg_unregister_all_caches(struct mem_cgroup *memcg) { struct kmem_cache *cachep; struct memcg_cache_params *params, *tmp; + LIST_HEAD(empty_caches); if (!memcg_kmem_is_active(memcg)) return; @@ -3261,7 +3270,26 @@ static void memcg_unregister_all_caches(struct mem_cgroup *memcg) kmem_cache_shrink(cachep); if (atomic_long_dec_and_test(&cachep->memcg_params->refcnt)) - memcg_unregister_cache(cachep); + list_move(&cachep->memcg_params->list, &empty_caches); + } + + /* + * kmem_cache_free doesn't expect that the cache can be destroyed as + * soon as the object is freed, e.g. SLUB's implementation may want to + * update cache stats after putting the object to the free list. + * + * Therefore we should wait for all kmem_cache_free's to finish before + * proceeding to cache destruction. Since both SLAB and SLUB versions + * of kmem_cache_free are non-preemptable, we wait for rcu-sched grace + * period to elapse. + */ + synchronize_sched(); + + while (!list_empty(&empty_caches)) { + params = list_first_entry(&empty_caches, + struct memcg_cache_params, list); + cachep = memcg_params_to_cache(params); + memcg_unregister_cache(cachep); } mutex_unlock(&memcg_slab_mutex); } @@ -3343,7 +3371,9 @@ void __memcg_uncharge_slab(struct kmem_cache *cachep, int order) memcg_uncharge_kmem(cachep->memcg_params->memcg, PAGE_SIZE << order); if (unlikely(atomic_long_dec_and_test(&cachep->memcg_params->refcnt))) - schedule_work(&cachep->memcg_params->unregister_work); + /* see memcg_unregister_all_caches */ + call_rcu_sched(&cachep->memcg_params->rcu_head, + memcg_unregister_cache_rcu_func); } /* -- 2.39.5