Merge branch 'linux-mfg' into tx6-mfg-devel

[karo-tx-linux.git] / mm / slub.c

diff --git a/mm/slub.c b/mm/slub.c
index d02fc13746470de5024cddc98e1a1130a988ac72..438ebf8bbab1ddaba0e25db37a8f2b1913859b96 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -338,11 +338,13 @@ static inline int oo_objects(struct kmem_cache_order_objects x)
  */
 static __always_inline void slab_lock(struct page *page)
 {
+       VM_BUG_ON_PAGE(PageTail(page), page);
        bit_spin_lock(PG_locked, &page->flags);
 }
 
 static __always_inline void slab_unlock(struct page *page)
 {
+       VM_BUG_ON_PAGE(PageTail(page), page);
        __bit_spin_unlock(PG_locked, &page->flags);
 }
 
@@ -459,8 +461,10 @@ static void get_map(struct kmem_cache *s, struct page *page, unsigned long *map)
 /*
  * Debug settings:
  */
-#ifdef CONFIG_SLUB_DEBUG_ON
+#if defined(CONFIG_SLUB_DEBUG_ON)
 static int slub_debug = DEBUG_DEFAULT_FLAGS;
+#elif defined(CONFIG_KASAN)
+static int slub_debug = SLAB_STORE_USER;
 #else
 static int slub_debug;
 #endif
@@ -1282,7 +1286,7 @@ static inline struct kmem_cache *slab_pre_alloc_hook(struct kmem_cache *s,
 {
        flags &= gfp_allowed_mask;
        lockdep_trace_alloc(flags);
-       might_sleep_if(flags & __GFP_WAIT);
+       might_sleep_if(gfpflags_allow_blocking(flags));
 
        if (should_failslab(s->object_size, flags, s->flags))
                return NULL;
@@ -1370,16 +1374,15 @@ static inline struct page *alloc_slab_page(struct kmem_cache *s,
 
        flags |= __GFP_NOTRACK;
 
-       if (memcg_charge_slab(s, flags, order))
-               return NULL;
-
        if (node == NUMA_NO_NODE)
                page = alloc_pages(flags, order);
        else
                page = __alloc_pages_node(node, flags, order);
 
-       if (!page)
-               memcg_uncharge_slab(s, order);
+       if (page && memcg_charge_slab(page, flags, order, s)) {
+               __free_pages(page, order);
+               page = NULL;
+       }
 
        return page;
 }
@@ -1394,7 +1397,7 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
 
        flags &= gfp_allowed_mask;
 
-       if (flags & __GFP_WAIT)
+       if (gfpflags_allow_blocking(flags))
                local_irq_enable();
 
        flags |= s->allocflags;
@@ -1404,8 +1407,8 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
         * so we fall-back to the minimum order allocation.
         */
        alloc_gfp = (flags | __GFP_NOWARN | __GFP_NORETRY) & ~__GFP_NOFAIL;
-       if ((alloc_gfp & __GFP_WAIT) && oo_order(oo) > oo_order(s->min))
-               alloc_gfp = (alloc_gfp | __GFP_NOMEMALLOC) & ~__GFP_WAIT;
+       if ((alloc_gfp & __GFP_DIRECT_RECLAIM) && oo_order(oo) > oo_order(s->min))
+               alloc_gfp = (alloc_gfp | __GFP_NOMEMALLOC) & ~__GFP_DIRECT_RECLAIM;
 
        page = alloc_slab_page(s, alloc_gfp, node, oo);
        if (unlikely(!page)) {
@@ -1465,7 +1468,7 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
        page->frozen = 1;
 
 out:
-       if (flags & __GFP_WAIT)
+       if (gfpflags_allow_blocking(flags))
                local_irq_disable();
        if (!page)
                return NULL;
@@ -1518,8 +1521,7 @@ static void __free_slab(struct kmem_cache *s, struct page *page)
        page_mapcount_reset(page);
        if (current->reclaim_state)
                current->reclaim_state->reclaimed_slab += pages;
-       __free_pages(page, order);
-       memcg_uncharge_slab(s, order);
+       __free_kmem_pages(page, order);
 }
 
 #define need_reserve_slab_rcu                                          \
@@ -1549,10 +1551,7 @@ static void free_slab(struct kmem_cache *s, struct page *page)
                        VM_BUG_ON(s->reserved != sizeof(*head));
                        head = page_address(page) + offset;
                } else {
-                       /*
-                        * RCU free overloads the RCU head over the LRU
-                        */
-                       head = (void *)&page->lru;
+                       head = &page->rcu_head;
                }
 
                call_rcu(head, rcu_free_slab);
@@ -3022,19 +3021,15 @@ static inline int slab_order(int size, int min_objects,
        if (order_objects(min_order, size, reserved) > MAX_OBJS_PER_PAGE)
                return get_order(size * MAX_OBJS_PER_PAGE) - 1;
 
-       for (order = max(min_order, get_order(min_objects * size));
+       for (order = max(min_order, get_order(min_objects * size + reserved));
                        order <= max_order; order++) {
 
                unsigned long slab_size = PAGE_SIZE << order;
 
-               if (slab_size < min_objects * size + reserved)
-                       continue;
-
                rem = (slab_size - reserved) % size;
 
                if (rem <= slab_size / fract_leftover)
                        break;
-
        }
 
        return order;