Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/dtor/input

[linux-beck.git] / mm / slub.c

diff --git a/mm/slub.c b/mm/slub.c
index c17c5202864d9f091b210eab4dd2eba2e426fcd4..46997517406ede0e987388763c9fa1d07875c1db 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -1292,14 +1292,21 @@ static inline struct kmem_cache *slab_pre_alloc_hook(struct kmem_cache *s,
        return memcg_kmem_get_cache(s, flags);
 }
 
-static inline void slab_post_alloc_hook(struct kmem_cache *s,
-                                       gfp_t flags, void *object)
+static inline void slab_post_alloc_hook(struct kmem_cache *s, gfp_t flags,
+                                       size_t size, void **p)
 {
+       size_t i;
+
        flags &= gfp_allowed_mask;
-       kmemcheck_slab_alloc(s, flags, object, slab_ksize(s));
-       kmemleak_alloc_recursive(object, s->object_size, 1, s->flags, flags);
+       for (i = 0; i < size; i++) {
+               void *object = p[i];
+
+               kmemcheck_slab_alloc(s, flags, object, slab_ksize(s));
+               kmemleak_alloc_recursive(object, s->object_size, 1,
+                                        s->flags, flags);
+               kasan_slab_alloc(s, object);
+       }
        memcg_kmem_put_cache(s);
-       kasan_slab_alloc(s, object);
 }
 
 static inline void slab_free_hook(struct kmem_cache *s, void *x)
@@ -2475,7 +2482,7 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
 static __always_inline void *slab_alloc_node(struct kmem_cache *s,
                gfp_t gfpflags, int node, unsigned long addr)
 {
-       void **object;
+       void *object;
        struct kmem_cache_cpu *c;
        struct page *page;
        unsigned long tid;
@@ -2554,7 +2561,7 @@ redo:
        if (unlikely(gfpflags & __GFP_ZERO) && object)
                memset(object, 0, s->object_size);
 
-       slab_post_alloc_hook(s, gfpflags, object);
+       slab_post_alloc_hook(s, gfpflags, 1, &object);
 
        return object;
 }
@@ -2880,13 +2887,17 @@ static int build_detached_freelist(struct kmem_cache *s, size_t size,
 
 
 /* Note that interrupts must be enabled when calling this function. */
-void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p)
+void kmem_cache_free_bulk(struct kmem_cache *orig_s, size_t size, void **p)
 {
        if (WARN_ON(!size))
                return;
 
        do {
                struct detached_freelist df;
+               struct kmem_cache *s;
+
+               /* Support for memcg */
+               s = cache_from_obj(orig_s, p[size - 1]);
 
                size = build_detached_freelist(s, size, p, &df);
                if (unlikely(!df.page))
@@ -2898,12 +2909,16 @@ void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p)
 EXPORT_SYMBOL(kmem_cache_free_bulk);
 
 /* Note that interrupts must be enabled when calling this function. */
-bool kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
-                          void **p)
+int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
+                         void **p)
 {
        struct kmem_cache_cpu *c;
        int i;
 
+       /* memcg and kmem_cache debug support */
+       s = slab_pre_alloc_hook(s, flags);
+       if (unlikely(!s))
+               return false;
        /*
         * Drain objects in the per cpu slab, while disabling local
         * IRQs, which protects against PREEMPT and interrupts
@@ -2928,17 +2943,8 @@ bool kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
                        c = this_cpu_ptr(s->cpu_slab);
                        continue; /* goto for-loop */
                }
-
-               /* kmem_cache debug support */
-               s = slab_pre_alloc_hook(s, flags);
-               if (unlikely(!s))
-                       goto error;
-
                c->freelist = get_freepointer(s, object);
                p[i] = object;
-
-               /* kmem_cache debug support */
-               slab_post_alloc_hook(s, flags, object);
        }
        c->tid = next_tid(c->tid);
        local_irq_enable();
@@ -2951,12 +2957,14 @@ bool kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
                        memset(p[j], 0, s->object_size);
        }
 
-       return true;
-
+       /* memcg and kmem_cache debug support */
+       slab_post_alloc_hook(s, flags, size, p);
+       return i;
 error:
-       __kmem_cache_free_bulk(s, i, p);
        local_irq_enable();
-       return false;
+       slab_post_alloc_hook(s, flags, i, p);
+       __kmem_cache_free_bulk(s, i, p);
+       return 0;
 }
 EXPORT_SYMBOL(kmem_cache_alloc_bulk);