Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/penberg...

[linux-beck.git] / mm / slub.c

diff --git a/mm/slub.c b/mm/slub.c
index de6f38761d1f7a897ced12bdff0730112c8548ae..992ecd4f0d393c748dceec95e3ddd714b16557b4 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -149,25 +149,6 @@ static inline void ClearSlabDebug(struct page *page)
 /* Enable to test recovery from slab corruption on boot */
 #undef SLUB_RESILIENCY_TEST
 
-#if PAGE_SHIFT <= 12
-
-/*
- * Small page size. Make sure that we do not fragment memory
- */
-#define DEFAULT_MAX_ORDER 1
-#define DEFAULT_MIN_OBJECTS 4
-
-#else
-
-/*
- * Large page machines are customarily able to handle larger
- * page orders.
- */
-#define DEFAULT_MAX_ORDER 2
-#define DEFAULT_MIN_OBJECTS 8
-
-#endif
-
 /*
  * Mininum number of partial slabs. These will be left on the partial
  * lists even if they are empty. kmem_cache_shrink may reclaim them.
@@ -205,11 +186,6 @@ static inline void ClearSlabDebug(struct page *page)
 #define __OBJECT_POISON                0x80000000 /* Poison object */
 #define __SYSFS_ADD_DEFERRED   0x40000000 /* Not yet visible via sysfs */
 
-/* Not all arches define cache_line_size */
-#ifndef cache_line_size
-#define cache_line_size()      L1_CACHE_BYTES
-#endif
-
 static int kmem_size = sizeof(struct kmem_cache);
 
 #ifdef CONFIG_SMP
@@ -1349,7 +1325,9 @@ static struct page *get_any_partial(struct kmem_cache *s, gfp_t flags)
 {
 #ifdef CONFIG_NUMA
        struct zonelist *zonelist;
-       struct zone **z;
+       struct zoneref *z;
+       struct zone *zone;
+       enum zone_type high_zoneidx = gfp_zone(flags);
        struct page *page;
 
        /*
@@ -1374,14 +1352,13 @@ static struct page *get_any_partial(struct kmem_cache *s, gfp_t flags)
                        get_cycles() % 1024 > s->remote_node_defrag_ratio)
                return NULL;
 
-       zonelist = &NODE_DATA(
-               slab_node(current->mempolicy))->node_zonelists[gfp_zone(flags)];
-       for (z = zonelist->zones; *z; z++) {
+       zonelist = node_zonelist(slab_node(current->mempolicy), flags);
+       for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) {
                struct kmem_cache_node *n;
 
-               n = get_node(s, zone_to_nid(*z));
+               n = get_node(s, zone_to_nid(zone));
 
-               if (n && cpuset_zone_allowed_hardwall(*z, flags) &&
+               if (n && cpuset_zone_allowed_hardwall(zone, flags) &&
                                n->nr_partial > MIN_PARTIAL) {
                        page = get_partial_node(n);
                        if (page)
@@ -1821,8 +1798,8 @@ static struct page *get_object_page(const void *x)
  * take the list_lock.
  */
 static int slub_min_order;
-static int slub_max_order = DEFAULT_MAX_ORDER;
-static int slub_min_objects = DEFAULT_MIN_OBJECTS;
+static int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;
+static int slub_min_objects;
 
 /*
  * Merge control. If this is set then no merging of slab caches will occur.
@@ -1837,7 +1814,7 @@ static int slub_nomerge;
  * system components. Generally order 0 allocations should be preferred since
  * order 0 does not cause fragmentation in the page allocator. Larger objects
  * be problematic to put into order 0 slabs because there may be too much
- * unused space left. We go to a higher order if more than 1/8th of the slab
+ * unused space left. We go to a higher order if more than 1/16th of the slab
  * would be wasted.
  *
  * In order to reach satisfactory performance we must ensure that a minimum
@@ -1899,8 +1876,10 @@ static inline int calculate_order(int size)
         * we reduce the minimum objects required in a slab.
         */
        min_objects = slub_min_objects;
+       if (!min_objects)
+               min_objects = 4 * (fls(nr_cpu_ids) + 1);
        while (min_objects > 1) {
-               fraction = 8;
+               fraction = 16;
                while (fraction >= 4) {
                        order = slab_order(size, min_objects,
                                                slub_max_order, fraction);
@@ -2218,7 +2197,7 @@ static int init_kmem_cache_nodes(struct kmem_cache *s, gfp_t gfpflags)
  * calculate_sizes() determines the order and the distribution of data within
  * a slab object.
  */
-static int calculate_sizes(struct kmem_cache *s)
+static int calculate_sizes(struct kmem_cache *s, int forced_order)
 {
        unsigned long flags = s->flags;
        unsigned long size = s->objsize;
@@ -2307,7 +2286,10 @@ static int calculate_sizes(struct kmem_cache *s)
         */
        size = ALIGN(size, align);
        s->size = size;
-       order = calculate_order(size);
+       if (forced_order >= 0)
+               order = forced_order;
+       else
+               order = calculate_order(size);
 
        if (order < 0)
                return 0;
@@ -2346,7 +2328,7 @@ static int kmem_cache_open(struct kmem_cache *s, gfp_t gfpflags,
        s->align = align;
        s->flags = kmem_cache_flags(size, flags, name, ctor);
 
-       if (!calculate_sizes(s))
+       if (!calculate_sizes(s, -1))
                goto error;
 
        s->refcount = 1;
@@ -3772,14 +3754,6 @@ static ssize_t show_slab_objects(struct kmem_cache *s,
 static int any_slab_objects(struct kmem_cache *s)
 {
        int node;
-       int cpu;
-
-       for_each_possible_cpu(cpu) {
-               struct kmem_cache_cpu *c = get_cpu_slab(s, cpu);
-
-               if (c && c->page)
-                       return 1;
-       }
 
        for_each_online_node(node) {
                struct kmem_cache_node *n = get_node(s, node);
@@ -3787,7 +3761,7 @@ static int any_slab_objects(struct kmem_cache *s)
                if (!n)
                        continue;
 
-               if (n->nr_partial || atomic_long_read(&n->nr_slabs))
+               if (atomic_read(&n->total_objects))
                        return 1;
        }
        return 0;
@@ -3833,11 +3807,23 @@ static ssize_t objs_per_slab_show(struct kmem_cache *s, char *buf)
 }
 SLAB_ATTR_RO(objs_per_slab);
 
+static ssize_t order_store(struct kmem_cache *s,
+                               const char *buf, size_t length)
+{
+       int order = simple_strtoul(buf, NULL, 10);
+
+       if (order > slub_max_order || order < slub_min_order)
+               return -EINVAL;
+
+       calculate_sizes(s, order);
+       return length;
+}
+
 static ssize_t order_show(struct kmem_cache *s, char *buf)
 {
        return sprintf(buf, "%d\n", oo_order(s->oo));
 }
-SLAB_ATTR_RO(order);
+SLAB_ATTR(order);
 
 static ssize_t ctor_show(struct kmem_cache *s, char *buf)
 {
@@ -3971,7 +3957,7 @@ static ssize_t red_zone_store(struct kmem_cache *s,
        s->flags &= ~SLAB_RED_ZONE;
        if (buf[0] == '1')
                s->flags |= SLAB_RED_ZONE;
-       calculate_sizes(s);
+       calculate_sizes(s, -1);
        return length;
 }
 SLAB_ATTR(red_zone);
@@ -3990,7 +3976,7 @@ static ssize_t poison_store(struct kmem_cache *s,
        s->flags &= ~SLAB_POISON;
        if (buf[0] == '1')
                s->flags |= SLAB_POISON;
-       calculate_sizes(s);
+       calculate_sizes(s, -1);
        return length;
 }
 SLAB_ATTR(poison);
@@ -4009,7 +3995,7 @@ static ssize_t store_user_store(struct kmem_cache *s,
        s->flags &= ~SLAB_STORE_USER;
        if (buf[0] == '1')
                s->flags |= SLAB_STORE_USER;
-       calculate_sizes(s);
+       calculate_sizes(s, -1);
        return length;
 }
 SLAB_ATTR(store_user);