net: use lib/percpu_counter API for fragmentation mem accounting

Replace the per network namespace shared atomic "mem" accounting
variable, in the fragmentation code, with a lib/percpu_counter.

Getting percpu_counter to scale to the fragmentation code usage
requires some tweaks.

At first view, percpu_counter looks superfast, but it does not
scale on multi-CPU/NUMA machines, because the default batch size
is too small, for frag code usage.  Thus, I have adjusted the
batch size by using __percpu_counter_add() directly, instead of
percpu_counter_sub() and percpu_counter_add().

The batch size is increased to 130.000, based on the largest 64K
fragment memory usage.  This does introduce some imprecise
memory accounting, but its does not need to be strict for this
use-case.

It is also essential, that the percpu_counter, does not
share cacheline with other writers, to make this scale.

Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/include/net/inet_frag.h b/include/net/inet_frag.h
index f2fabc2a79dee2619f4c8081ded47f75a7f0a7cf..e0eec7450f1570440f515ec62d9a75a916868687 100644 (file)
--- a/include/net/inet_frag.h
+++ b/include/net/inet_frag.h
@@ -1,14 +1,17 @@
 #ifndef __NET_FRAG_H__
 #define __NET_FRAG_H__
 
+#include <linux/percpu_counter.h>
+
 struct netns_frags {
        int                     nqueues;
        struct list_head        lru_list;
 
-       /* Its important for performance to keep lru_list and mem on
-        * separate cachelines
+       /* The percpu_counter "mem" need to be cacheline aligned.
+        *  mem.count must not share cacheline with other writers
         */
-       atomic_t                mem ____cacheline_aligned_in_smp;
+       struct percpu_counter   mem ____cacheline_aligned_in_smp;
+
        /* sysctls */
        int                     timeout;
        int                     high_thresh;
@@ -81,29 +84,36 @@ static inline void inet_frag_put(struct inet_frag_queue *q, struct inet_frags *f
 
 /* Memory Tracking Functions. */
 
+/* The default percpu_counter batch size is not big enough to scale to
+ * fragmentation mem acct sizes.
+ * The mem size of a 64K fragment is approx:
+ *  (44 fragments * 2944 truesize) + frag_queue struct(200) = 129736 bytes
+ */
+static unsigned int frag_percpu_counter_batch = 130000;
+
 static inline int frag_mem_limit(struct netns_frags *nf)
 {
-       return atomic_read(&nf->mem);
+       return percpu_counter_read(&nf->mem);
 }
 
 static inline void sub_frag_mem_limit(struct inet_frag_queue *q, int i)
 {
-       atomic_sub(i, &q->net->mem);
+       __percpu_counter_add(&q->net->mem, -i, frag_percpu_counter_batch);
 }
 
 static inline void add_frag_mem_limit(struct inet_frag_queue *q, int i)
 {
-       atomic_add(i, &q->net->mem);
+       __percpu_counter_add(&q->net->mem, i, frag_percpu_counter_batch);
 }
 
 static inline void init_frag_mem_limit(struct netns_frags *nf)
 {
-       atomic_set(&nf->mem, 0);
+       percpu_counter_init(&nf->mem, 0);
 }
 
 static inline int sum_frag_mem_limit(struct netns_frags *nf)
 {
-       return atomic_read(&nf->mem);
+       return percpu_counter_sum_positive(&nf->mem);
 }
 
 #endif

diff --git a/net/ipv4/inet_fragment.c b/net/ipv4/inet_fragment.c
index e348c849c5a3b31c368d490763a4417ac40cb30c..b825205650984418dcc56b4a55fdf6e8ac9dccfc 100644 (file)
--- a/net/ipv4/inet_fragment.c
+++ b/net/ipv4/inet_fragment.c
@@ -91,6 +91,8 @@ void inet_frags_exit_net(struct netns_frags *nf, struct inet_frags *f)
        local_bh_disable();
        inet_frag_evictor(nf, f, true);
        local_bh_enable();
+
+       percpu_counter_destroy(&nf->mem);
 }
 EXPORT_SYMBOL(inet_frags_exit_net);