From f70504241cea375a5e9963a86858e2c41c5317c6 Mon Sep 17 00:00:00 2001 From: Vladimir Davydov Date: Thu, 26 Jun 2014 10:42:32 +1000 Subject: [PATCH] slab: set free_limit for dead caches to 0 We mustn't keep empty slabs on dead memcg caches, because otherwise they will never be destroyed. Currently, we check if the cache is dead in free_block and drop empty slab if so irrespective of the node's free_limit. This can be pretty expensive. So let's avoid this additional check by zeroing nodes' free_limit for dead caches on kmem_cache_shrink. This way no additional overhead is added to free hot path. Note, since ->free_limit can be updated on cpu/memory hotplug, we must handle it properly there. Signed-off-by: Vladimir Davydov Suggested-by: Joonsoo Kim Cc: Christoph Lameter Cc: Pekka Enberg Cc: David Rientjes Cc: Michal Hocko Cc: Johannes Weiner Signed-off-by: Andrew Morton --- mm/slab.c | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/mm/slab.c b/mm/slab.c index f27ad3741de8..179272f262f2 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -1156,11 +1156,13 @@ static int init_cache_node_node(int node) cachep->node[node] = n; } - spin_lock_irq(&n->list_lock); - n->free_limit = - (1 + nr_cpus_node(node)) * - cachep->batchcount + cachep->num; - spin_unlock_irq(&n->list_lock); + if (!memcg_cache_dead(cachep)) { + spin_lock_irq(&n->list_lock); + n->free_limit = + (1 + nr_cpus_node(node)) * + cachep->batchcount + cachep->num; + spin_unlock_irq(&n->list_lock); + } } return 0; } @@ -1194,7 +1196,8 @@ static void cpuup_canceled(long cpu) spin_lock_irq(&n->list_lock); /* Free limit for this kmem_cache_node */ - n->free_limit -= cachep->batchcount; + if (!memcg_cache_dead(cachep)) + n->free_limit -= cachep->batchcount; if (nc) free_block(cachep, nc->entry, nc->avail, node); @@ -2545,6 +2548,12 @@ int __kmem_cache_shrink(struct kmem_cache *cachep) check_irq_on(); for_each_kmem_cache_node(cachep, node, n) { + if (memcg_cache_dead(cachep)) { + spin_lock_irq(&n->list_lock); + n->free_limit = 0; + spin_unlock_irq(&n->list_lock); + } + drain_freelist(cachep, n, slabs_tofree(cachep, n)); ret += !list_empty(&n->slabs_full) || @@ -3427,8 +3436,7 @@ static void free_block(struct kmem_cache *cachep, void **objpp, int nr_objects, /* fixup slab chains */ if (page->active == 0) { - if (n->free_objects > n->free_limit || - memcg_cache_dead(cachep)) { + if (n->free_objects > n->free_limit) { n->free_objects -= cachep->num; /* No need to drop any previously held * lock here, even if we have a off-slab slab -- 2.39.5