X-Git-Url: https://git.karo-electronics.de/?a=blobdiff_plain;f=net%2Fnetfilter%2Fnf_conntrack_core.c;h=3d1411012a2c496d5bdf98649321e63074e63e60;hb=0806ca2ab3ef7d7a1bd41a980f661a13ba11acb5;hp=483e927a9ca4130182815670c138b87eca3a856b;hpb=04fc5fd39e09c460fd478ea84b34c445dd376759;p=karo-tx-linux.git diff --git a/net/netfilter/nf_conntrack_core.c b/net/netfilter/nf_conntrack_core.c index 483e927a9ca4..3d1411012a2c 100644 --- a/net/netfilter/nf_conntrack_core.c +++ b/net/netfilter/nf_conntrack_core.c @@ -36,15 +36,10 @@ #include #include #include +#include #define NF_CONNTRACK_VERSION "0.5.0" -#if 0 -#define DEBUGP printk -#else -#define DEBUGP(format, args...) -#endif - DEFINE_RWLOCK(nf_conntrack_lock); EXPORT_SYMBOL_GPL(nf_conntrack_lock); @@ -52,57 +47,27 @@ EXPORT_SYMBOL_GPL(nf_conntrack_lock); atomic_t nf_conntrack_count = ATOMIC_INIT(0); EXPORT_SYMBOL_GPL(nf_conntrack_count); -void (*nf_conntrack_destroyed)(struct nf_conn *conntrack); -EXPORT_SYMBOL_GPL(nf_conntrack_destroyed); - unsigned int nf_conntrack_htable_size __read_mostly; EXPORT_SYMBOL_GPL(nf_conntrack_htable_size); int nf_conntrack_max __read_mostly; EXPORT_SYMBOL_GPL(nf_conntrack_max); -struct list_head *nf_conntrack_hash __read_mostly; +struct hlist_head *nf_conntrack_hash __read_mostly; EXPORT_SYMBOL_GPL(nf_conntrack_hash); struct nf_conn nf_conntrack_untracked __read_mostly; EXPORT_SYMBOL_GPL(nf_conntrack_untracked); unsigned int nf_ct_log_invalid __read_mostly; -LIST_HEAD(unconfirmed); +HLIST_HEAD(unconfirmed); static int nf_conntrack_vmalloc __read_mostly; - +static struct kmem_cache *nf_conntrack_cachep __read_mostly; static unsigned int nf_conntrack_next_id; DEFINE_PER_CPU(struct ip_conntrack_stat, nf_conntrack_stat); EXPORT_PER_CPU_SYMBOL(nf_conntrack_stat); -/* - * This scheme offers various size of "struct nf_conn" dependent on - * features(helper, nat, ...) - */ - -#define NF_CT_FEATURES_NAMELEN 256 -static struct { - /* name of slab cache. printed in /proc/slabinfo */ - char *name; - - /* size of slab cache */ - size_t size; - - /* slab cache pointer */ - struct kmem_cache *cachep; - - /* allocated slab cache + modules which uses this slab cache */ - int use; - -} nf_ct_cache[NF_CT_F_NUM]; - -/* protect members of nf_ct_cache except of "use" */ -DEFINE_RWLOCK(nf_ct_cache_lock); - -/* This avoids calling kmem_cache_create() with same name simultaneously */ -static DEFINE_MUTEX(nf_ct_cache_mutex); - static int nf_conntrack_hash_rnd_initted; static unsigned int nf_conntrack_hash_rnd; @@ -125,122 +90,6 @@ static inline u_int32_t hash_conntrack(const struct nf_conntrack_tuple *tuple) nf_conntrack_hash_rnd); } -int nf_conntrack_register_cache(u_int32_t features, const char *name, - size_t size) -{ - int ret = 0; - char *cache_name; - struct kmem_cache *cachep; - - DEBUGP("nf_conntrack_register_cache: features=0x%x, name=%s, size=%d\n", - features, name, size); - - if (features < NF_CT_F_BASIC || features >= NF_CT_F_NUM) { - DEBUGP("nf_conntrack_register_cache: invalid features.: 0x%x\n", - features); - return -EINVAL; - } - - mutex_lock(&nf_ct_cache_mutex); - - write_lock_bh(&nf_ct_cache_lock); - /* e.g: multiple helpers are loaded */ - if (nf_ct_cache[features].use > 0) { - DEBUGP("nf_conntrack_register_cache: already resisterd.\n"); - if ((!strncmp(nf_ct_cache[features].name, name, - NF_CT_FEATURES_NAMELEN)) - && nf_ct_cache[features].size == size) { - DEBUGP("nf_conntrack_register_cache: reusing.\n"); - nf_ct_cache[features].use++; - ret = 0; - } else - ret = -EBUSY; - - write_unlock_bh(&nf_ct_cache_lock); - mutex_unlock(&nf_ct_cache_mutex); - return ret; - } - write_unlock_bh(&nf_ct_cache_lock); - - /* - * The memory space for name of slab cache must be alive until - * cache is destroyed. - */ - cache_name = kmalloc(sizeof(char)*NF_CT_FEATURES_NAMELEN, GFP_ATOMIC); - if (cache_name == NULL) { - DEBUGP("nf_conntrack_register_cache: can't alloc cache_name\n"); - ret = -ENOMEM; - goto out_up_mutex; - } - - if (strlcpy(cache_name, name, NF_CT_FEATURES_NAMELEN) - >= NF_CT_FEATURES_NAMELEN) { - printk("nf_conntrack_register_cache: name too long\n"); - ret = -EINVAL; - goto out_free_name; - } - - cachep = kmem_cache_create(cache_name, size, 0, 0, - NULL, NULL); - if (!cachep) { - printk("nf_conntrack_register_cache: Can't create slab cache " - "for the features = 0x%x\n", features); - ret = -ENOMEM; - goto out_free_name; - } - - write_lock_bh(&nf_ct_cache_lock); - nf_ct_cache[features].use = 1; - nf_ct_cache[features].size = size; - nf_ct_cache[features].cachep = cachep; - nf_ct_cache[features].name = cache_name; - write_unlock_bh(&nf_ct_cache_lock); - - goto out_up_mutex; - -out_free_name: - kfree(cache_name); -out_up_mutex: - mutex_unlock(&nf_ct_cache_mutex); - return ret; -} -EXPORT_SYMBOL_GPL(nf_conntrack_register_cache); - -/* FIXME: In the current, only nf_conntrack_cleanup() can call this function. */ -void nf_conntrack_unregister_cache(u_int32_t features) -{ - struct kmem_cache *cachep; - char *name; - - /* - * This assures that kmem_cache_create() isn't called before destroying - * slab cache. - */ - DEBUGP("nf_conntrack_unregister_cache: 0x%04x\n", features); - mutex_lock(&nf_ct_cache_mutex); - - write_lock_bh(&nf_ct_cache_lock); - if (--nf_ct_cache[features].use > 0) { - write_unlock_bh(&nf_ct_cache_lock); - mutex_unlock(&nf_ct_cache_mutex); - return; - } - cachep = nf_ct_cache[features].cachep; - name = nf_ct_cache[features].name; - nf_ct_cache[features].cachep = NULL; - nf_ct_cache[features].name = NULL; - nf_ct_cache[features].size = 0; - write_unlock_bh(&nf_ct_cache_lock); - - synchronize_net(); - - kmem_cache_destroy(cachep); - kfree(name); - - mutex_unlock(&nf_ct_cache_mutex); -} -EXPORT_SYMBOL_GPL(nf_conntrack_unregister_cache); - int nf_ct_get_tuple(const struct sk_buff *skb, unsigned int nhoff, @@ -286,9 +135,9 @@ EXPORT_SYMBOL_GPL(nf_ct_invert_tuple); static void clean_from_lists(struct nf_conn *ct) { - DEBUGP("clean_from_lists(%p)\n", ct); - list_del(&ct->tuplehash[IP_CT_DIR_ORIGINAL].list); - list_del(&ct->tuplehash[IP_CT_DIR_REPLY].list); + pr_debug("clean_from_lists(%p)\n", ct); + hlist_del(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnode); + hlist_del(&ct->tuplehash[IP_CT_DIR_REPLY].hnode); /* Destroy all pending expectations */ nf_ct_remove_expectations(ct); @@ -299,9 +148,8 @@ destroy_conntrack(struct nf_conntrack *nfct) { struct nf_conn *ct = (struct nf_conn *)nfct; struct nf_conntrack_l4proto *l4proto; - typeof(nf_conntrack_destroyed) destroyed; - DEBUGP("destroy_conntrack(%p)\n", ct); + pr_debug("destroy_conntrack(%p)\n", ct); NF_CT_ASSERT(atomic_read(&nfct->use) == 0); NF_CT_ASSERT(!timer_pending(&ct->timeout)); @@ -317,9 +165,7 @@ destroy_conntrack(struct nf_conntrack *nfct) if (l4proto && l4proto->destroy) l4proto->destroy(ct); - destroyed = rcu_dereference(nf_conntrack_destroyed); - if (destroyed) - destroyed(ct); + nf_ct_ext_destroy(ct); rcu_read_unlock(); @@ -332,8 +178,8 @@ destroy_conntrack(struct nf_conntrack *nfct) /* We overload first tuple to link into unconfirmed list. */ if (!nf_ct_is_confirmed(ct)) { - BUG_ON(list_empty(&ct->tuplehash[IP_CT_DIR_ORIGINAL].list)); - list_del(&ct->tuplehash[IP_CT_DIR_ORIGINAL].list); + BUG_ON(hlist_unhashed(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnode)); + hlist_del(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnode); } NF_CT_STAT_INC(delete); @@ -342,7 +188,7 @@ destroy_conntrack(struct nf_conntrack *nfct) if (ct->master) nf_ct_put(ct->master); - DEBUGP("destroy_conntrack: returning ct=%p to slab\n", ct); + pr_debug("destroy_conntrack: returning ct=%p to slab\n", ct); nf_conntrack_free(ct); } @@ -350,9 +196,15 @@ static void death_by_timeout(unsigned long ul_conntrack) { struct nf_conn *ct = (void *)ul_conntrack; struct nf_conn_help *help = nfct_help(ct); + struct nf_conntrack_helper *helper; - if (help && help->helper && help->helper->destroy) - help->helper->destroy(ct); + if (help) { + rcu_read_lock(); + helper = rcu_dereference(help->helper); + if (helper && helper->destroy) + helper->destroy(ct); + rcu_read_unlock(); + } write_lock_bh(&nf_conntrack_lock); /* Inside lock so preempt is disabled on module removal path. @@ -368,9 +220,10 @@ __nf_conntrack_find(const struct nf_conntrack_tuple *tuple, const struct nf_conn *ignored_conntrack) { struct nf_conntrack_tuple_hash *h; + struct hlist_node *n; unsigned int hash = hash_conntrack(tuple); - list_for_each_entry(h, &nf_conntrack_hash[hash], list) { + hlist_for_each_entry(h, n, &nf_conntrack_hash[hash], hnode) { if (nf_ct_tuplehash_to_ctrack(h) != ignored_conntrack && nf_ct_tuple_equal(tuple, &h->tuple)) { NF_CT_STAT_INC(found); @@ -385,13 +238,12 @@ EXPORT_SYMBOL_GPL(__nf_conntrack_find); /* Find a connection corresponding to a tuple. */ struct nf_conntrack_tuple_hash * -nf_conntrack_find_get(const struct nf_conntrack_tuple *tuple, - const struct nf_conn *ignored_conntrack) +nf_conntrack_find_get(const struct nf_conntrack_tuple *tuple) { struct nf_conntrack_tuple_hash *h; read_lock_bh(&nf_conntrack_lock); - h = __nf_conntrack_find(tuple, ignored_conntrack); + h = __nf_conntrack_find(tuple, NULL); if (h) atomic_inc(&nf_ct_tuplehash_to_ctrack(h)->ct_general.use); read_unlock_bh(&nf_conntrack_lock); @@ -405,10 +257,10 @@ static void __nf_conntrack_hash_insert(struct nf_conn *ct, unsigned int repl_hash) { ct->id = ++nf_conntrack_next_id; - list_add(&ct->tuplehash[IP_CT_DIR_ORIGINAL].list, - &nf_conntrack_hash[hash]); - list_add(&ct->tuplehash[IP_CT_DIR_REPLY].list, - &nf_conntrack_hash[repl_hash]); + hlist_add_head(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnode, + &nf_conntrack_hash[hash]); + hlist_add_head(&ct->tuplehash[IP_CT_DIR_REPLY].hnode, + &nf_conntrack_hash[repl_hash]); } void nf_conntrack_hash_insert(struct nf_conn *ct) @@ -432,6 +284,7 @@ __nf_conntrack_confirm(struct sk_buff **pskb) struct nf_conntrack_tuple_hash *h; struct nf_conn *ct; struct nf_conn_help *help; + struct hlist_node *n; enum ip_conntrack_info ctinfo; ct = nf_ct_get(*pskb, &ctinfo); @@ -454,24 +307,24 @@ __nf_conntrack_confirm(struct sk_buff **pskb) /* No external references means noone else could have confirmed us. */ NF_CT_ASSERT(!nf_ct_is_confirmed(ct)); - DEBUGP("Confirming conntrack %p\n", ct); + pr_debug("Confirming conntrack %p\n", ct); write_lock_bh(&nf_conntrack_lock); /* See if there's one in the list already, including reverse: NAT could have grabbed it without realizing, since we're not in the hash. If there is, we lost race. */ - list_for_each_entry(h, &nf_conntrack_hash[hash], list) + hlist_for_each_entry(h, n, &nf_conntrack_hash[hash], hnode) if (nf_ct_tuple_equal(&ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple, &h->tuple)) goto out; - list_for_each_entry(h, &nf_conntrack_hash[repl_hash], list) + hlist_for_each_entry(h, n, &nf_conntrack_hash[repl_hash], hnode) if (nf_ct_tuple_equal(&ct->tuplehash[IP_CT_DIR_REPLY].tuple, &h->tuple)) goto out; /* Remove from unconfirmed list */ - list_del(&ct->tuplehash[IP_CT_DIR_ORIGINAL].list); + hlist_del(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnode); __nf_conntrack_hash_insert(ct, hash, repl_hash); /* Timer relative to confirmation time, not original @@ -518,24 +371,33 @@ nf_conntrack_tuple_taken(const struct nf_conntrack_tuple *tuple, } EXPORT_SYMBOL_GPL(nf_conntrack_tuple_taken); +#define NF_CT_EVICTION_RANGE 8 + /* There's a small race here where we may free a just-assured connection. Too bad: we're in trouble anyway. */ -static int early_drop(struct list_head *chain) +static int early_drop(unsigned int hash) { - /* Traverse backwards: gives us oldest, which is roughly LRU */ + /* Use oldest entry, which is roughly LRU */ struct nf_conntrack_tuple_hash *h; struct nf_conn *ct = NULL, *tmp; + struct hlist_node *n; + unsigned int i, cnt = 0; int dropped = 0; read_lock_bh(&nf_conntrack_lock); - list_for_each_entry_reverse(h, chain, list) { - tmp = nf_ct_tuplehash_to_ctrack(h); - if (!test_bit(IPS_ASSURED_BIT, &tmp->status)) { - ct = tmp; - atomic_inc(&ct->ct_general.use); - break; + for (i = 0; i < nf_conntrack_htable_size; i++) { + hlist_for_each_entry(h, n, &nf_conntrack_hash[hash], hnode) { + tmp = nf_ct_tuplehash_to_ctrack(h); + if (!test_bit(IPS_ASSURED_BIT, &tmp->status)) + ct = tmp; + cnt++; } + if (ct || cnt >= NF_CT_EVICTION_RANGE) + break; + hash = (hash + 1) % nf_conntrack_htable_size; } + if (ct) + atomic_inc(&ct->ct_general.use); read_unlock_bh(&nf_conntrack_lock); if (!ct) @@ -550,14 +412,10 @@ static int early_drop(struct list_head *chain) return dropped; } -static struct nf_conn * -__nf_conntrack_alloc(const struct nf_conntrack_tuple *orig, - const struct nf_conntrack_tuple *repl, - const struct nf_conntrack_l3proto *l3proto, - u_int32_t features) +struct nf_conn *nf_conntrack_alloc(const struct nf_conntrack_tuple *orig, + const struct nf_conntrack_tuple *repl) { struct nf_conn *conntrack = NULL; - struct nf_conntrack_helper *helper; if (unlikely(!nf_conntrack_hash_rnd_initted)) { get_random_bytes(&nf_conntrack_hash_rnd, 4); @@ -570,8 +428,7 @@ __nf_conntrack_alloc(const struct nf_conntrack_tuple *orig, if (nf_conntrack_max && atomic_read(&nf_conntrack_count) > nf_conntrack_max) { unsigned int hash = hash_conntrack(orig); - /* Try dropping from this hash chain. */ - if (!early_drop(&nf_conntrack_hash[hash])) { + if (!early_drop(hash)) { atomic_dec(&nf_conntrack_count); if (net_ratelimit()) printk(KERN_WARNING @@ -581,72 +438,28 @@ __nf_conntrack_alloc(const struct nf_conntrack_tuple *orig, } } - /* find features needed by this conntrack. */ - features |= l3proto->get_features(orig); - - /* FIXME: protect helper list per RCU */ - read_lock_bh(&nf_conntrack_lock); - helper = __nf_ct_helper_find(repl); - /* NAT might want to assign a helper later */ - if (helper || features & NF_CT_F_NAT) - features |= NF_CT_F_HELP; - read_unlock_bh(&nf_conntrack_lock); - - DEBUGP("nf_conntrack_alloc: features=0x%x\n", features); - - read_lock_bh(&nf_ct_cache_lock); - - if (unlikely(!nf_ct_cache[features].use)) { - DEBUGP("nf_conntrack_alloc: not supported features = 0x%x\n", - features); - goto out; - } - - conntrack = kmem_cache_alloc(nf_ct_cache[features].cachep, GFP_ATOMIC); + conntrack = kmem_cache_zalloc(nf_conntrack_cachep, GFP_ATOMIC); if (conntrack == NULL) { - DEBUGP("nf_conntrack_alloc: Can't alloc conntrack from cache\n"); - goto out; + pr_debug("nf_conntrack_alloc: Can't alloc conntrack.\n"); + atomic_dec(&nf_conntrack_count); + return ERR_PTR(-ENOMEM); } - memset(conntrack, 0, nf_ct_cache[features].size); - conntrack->features = features; atomic_set(&conntrack->ct_general.use, 1); conntrack->tuplehash[IP_CT_DIR_ORIGINAL].tuple = *orig; conntrack->tuplehash[IP_CT_DIR_REPLY].tuple = *repl; /* Don't set timer yet: wait for confirmation */ setup_timer(&conntrack->timeout, death_by_timeout, (unsigned long)conntrack); - read_unlock_bh(&nf_ct_cache_lock); return conntrack; -out: - read_unlock_bh(&nf_ct_cache_lock); - atomic_dec(&nf_conntrack_count); - return conntrack; -} - -struct nf_conn *nf_conntrack_alloc(const struct nf_conntrack_tuple *orig, - const struct nf_conntrack_tuple *repl) -{ - struct nf_conntrack_l3proto *l3proto; - struct nf_conn *ct; - - rcu_read_lock(); - l3proto = __nf_ct_l3proto_find(orig->src.l3num); - ct = __nf_conntrack_alloc(orig, repl, l3proto, 0); - rcu_read_unlock(); - - return ct; } EXPORT_SYMBOL_GPL(nf_conntrack_alloc); void nf_conntrack_free(struct nf_conn *conntrack) { - u_int32_t features = conntrack->features; - NF_CT_ASSERT(features >= NF_CT_F_BASIC && features < NF_CT_F_NUM); - DEBUGP("nf_conntrack_free: features = 0x%x, conntrack=%p\n", features, - conntrack); - kmem_cache_free(nf_ct_cache[features].cachep, conntrack); + nf_ct_ext_free(conntrack); + kmem_cache_free(nf_conntrack_cachep, conntrack); atomic_dec(&nf_conntrack_count); } EXPORT_SYMBOL_GPL(nf_conntrack_free); @@ -661,44 +474,41 @@ init_conntrack(const struct nf_conntrack_tuple *tuple, unsigned int dataoff) { struct nf_conn *conntrack; + struct nf_conn_help *help; struct nf_conntrack_tuple repl_tuple; struct nf_conntrack_expect *exp; - u_int32_t features = 0; if (!nf_ct_invert_tuple(&repl_tuple, tuple, l3proto, l4proto)) { - DEBUGP("Can't invert tuple.\n"); + pr_debug("Can't invert tuple.\n"); return NULL; } - read_lock_bh(&nf_conntrack_lock); - exp = __nf_conntrack_expect_find(tuple); - if (exp && exp->helper) - features = NF_CT_F_HELP; - read_unlock_bh(&nf_conntrack_lock); - - conntrack = __nf_conntrack_alloc(tuple, &repl_tuple, l3proto, features); + conntrack = nf_conntrack_alloc(tuple, &repl_tuple); if (conntrack == NULL || IS_ERR(conntrack)) { - DEBUGP("Can't allocate conntrack.\n"); + pr_debug("Can't allocate conntrack.\n"); return (struct nf_conntrack_tuple_hash *)conntrack; } if (!l4proto->new(conntrack, skb, dataoff)) { nf_conntrack_free(conntrack); - DEBUGP("init conntrack: can't track with proto module\n"); + pr_debug("init conntrack: can't track with proto module\n"); return NULL; } write_lock_bh(&nf_conntrack_lock); - exp = find_expectation(tuple); - + exp = nf_ct_find_expectation(tuple); if (exp) { - DEBUGP("conntrack: expectation arrives ct=%p exp=%p\n", - conntrack, exp); + pr_debug("conntrack: expectation arrives ct=%p exp=%p\n", + conntrack, exp); /* Welcome, Mr. Bond. We've been expecting you... */ __set_bit(IPS_EXPECTED_BIT, &conntrack->status); conntrack->master = exp->master; - if (exp->helper) - nfct_help(conntrack)->helper = exp->helper; + if (exp->helper) { + help = nf_ct_helper_ext_add(conntrack, GFP_ATOMIC); + if (help) + rcu_assign_pointer(help->helper, exp->helper); + } + #ifdef CONFIG_NF_CONNTRACK_MARK conntrack->mark = exp->master->mark; #endif @@ -708,22 +518,27 @@ init_conntrack(const struct nf_conntrack_tuple *tuple, nf_conntrack_get(&conntrack->master->ct_general); NF_CT_STAT_INC(expect_new); } else { - struct nf_conn_help *help = nfct_help(conntrack); + struct nf_conntrack_helper *helper; - if (help) - help->helper = __nf_ct_helper_find(&repl_tuple); + helper = __nf_ct_helper_find(&repl_tuple); + if (helper) { + help = nf_ct_helper_ext_add(conntrack, GFP_ATOMIC); + if (help) + rcu_assign_pointer(help->helper, helper); + } NF_CT_STAT_INC(new); } /* Overload tuple linked list to put us in unconfirmed list. */ - list_add(&conntrack->tuplehash[IP_CT_DIR_ORIGINAL].list, &unconfirmed); + hlist_add_head(&conntrack->tuplehash[IP_CT_DIR_ORIGINAL].hnode, + &unconfirmed); write_unlock_bh(&nf_conntrack_lock); if (exp) { if (exp->expectfn) exp->expectfn(conntrack, exp); - nf_conntrack_expect_put(exp); + nf_ct_expect_put(exp); } return &conntrack->tuplehash[IP_CT_DIR_ORIGINAL]; @@ -747,12 +562,12 @@ resolve_normal_ct(struct sk_buff *skb, if (!nf_ct_get_tuple(skb, skb_network_offset(skb), dataoff, l3num, protonum, &tuple, l3proto, l4proto)) { - DEBUGP("resolve_normal_ct: Can't get tuple\n"); + pr_debug("resolve_normal_ct: Can't get tuple\n"); return NULL; } /* look for tuple match */ - h = nf_conntrack_find_get(&tuple, NULL); + h = nf_conntrack_find_get(&tuple); if (!h) { h = init_conntrack(&tuple, l3proto, l4proto, skb, dataoff); if (!h) @@ -770,13 +585,14 @@ resolve_normal_ct(struct sk_buff *skb, } else { /* Once we've had two way comms, always ESTABLISHED. */ if (test_bit(IPS_SEEN_REPLY_BIT, &ct->status)) { - DEBUGP("nf_conntrack_in: normal packet for %p\n", ct); + pr_debug("nf_conntrack_in: normal packet for %p\n", ct); *ctinfo = IP_CT_ESTABLISHED; } else if (test_bit(IPS_EXPECTED_BIT, &ct->status)) { - DEBUGP("nf_conntrack_in: related packet for %p\n", ct); + pr_debug("nf_conntrack_in: related packet for %p\n", + ct); *ctinfo = IP_CT_RELATED; } else { - DEBUGP("nf_conntrack_in: new packet for %p\n", ct); + pr_debug("nf_conntrack_in: new packet for %p\n", ct); *ctinfo = IP_CT_NEW; } *set_reply = 0; @@ -808,7 +624,7 @@ nf_conntrack_in(int pf, unsigned int hooknum, struct sk_buff **pskb) l3proto = __nf_ct_l3proto_find((u_int16_t)pf); if ((ret = l3proto->prepare(pskb, hooknum, &dataoff, &protonum)) <= 0) { - DEBUGP("not prepared to track yet or error occured\n"); + pr_debug("not prepared to track yet or error occured\n"); return -ret; } @@ -844,7 +660,7 @@ nf_conntrack_in(int pf, unsigned int hooknum, struct sk_buff **pskb) if (ret < 0) { /* Invalid: inverse of the return code tells * the netfilter core what to do */ - DEBUGP("nf_conntrack_in: Can't track with proto module\n"); + pr_debug("nf_conntrack_in: Can't track with proto module\n"); nf_conntrack_put((*pskb)->nfct); (*pskb)->nfct = NULL; NF_CT_STAT_INC_ATOMIC(invalid); @@ -879,22 +695,36 @@ void nf_conntrack_alter_reply(struct nf_conn *ct, const struct nf_conntrack_tuple *newreply) { struct nf_conn_help *help = nfct_help(ct); + struct nf_conntrack_helper *helper; write_lock_bh(&nf_conntrack_lock); /* Should be unconfirmed, so not in hash table yet */ NF_CT_ASSERT(!nf_ct_is_confirmed(ct)); - DEBUGP("Altering reply tuple of %p to ", ct); + pr_debug("Altering reply tuple of %p to ", ct); NF_CT_DUMP_TUPLE(newreply); ct->tuplehash[IP_CT_DIR_REPLY].tuple = *newreply; - if (!ct->master && help && help->expecting == 0) { - struct nf_conntrack_helper *helper; - helper = __nf_ct_helper_find(newreply); - if (helper) - memset(&help->help, 0, sizeof(help->help)); - help->helper = helper; + if (ct->master || (help && help->expecting != 0)) + goto out; + + helper = __nf_ct_helper_find(newreply); + if (helper == NULL) { + if (help) + rcu_assign_pointer(help->helper, NULL); + goto out; + } + + if (help == NULL) { + help = nf_ct_helper_ext_add(ct, GFP_ATOMIC); + if (help == NULL) + goto out; + } else { + memset(&help->help, 0, sizeof(help->help)); } + + rcu_assign_pointer(help->helper, helper); +out: write_unlock_bh(&nf_conntrack_lock); } EXPORT_SYMBOL_GPL(nf_conntrack_alter_reply); @@ -1038,16 +868,17 @@ get_next_corpse(int (*iter)(struct nf_conn *i, void *data), { struct nf_conntrack_tuple_hash *h; struct nf_conn *ct; + struct hlist_node *n; write_lock_bh(&nf_conntrack_lock); for (; *bucket < nf_conntrack_htable_size; (*bucket)++) { - list_for_each_entry(h, &nf_conntrack_hash[*bucket], list) { + hlist_for_each_entry(h, n, &nf_conntrack_hash[*bucket], hnode) { ct = nf_ct_tuplehash_to_ctrack(h); if (iter(ct, data)) goto found; } } - list_for_each_entry(h, &unconfirmed, list) { + hlist_for_each_entry(h, n, &unconfirmed, hnode) { ct = nf_ct_tuplehash_to_ctrack(h); if (iter(ct, data)) set_bit(IPS_DYING_BIT, &ct->status); @@ -1082,14 +913,15 @@ static int kill_all(struct nf_conn *i, void *data) return 1; } -static void free_conntrack_hash(struct list_head *hash, int vmalloced, int size) +void nf_ct_free_hashtable(struct hlist_head *hash, int vmalloced, int size) { if (vmalloced) vfree(hash); else free_pages((unsigned long)hash, - get_order(sizeof(struct list_head) * size)); + get_order(sizeof(struct hlist_head) * size)); } +EXPORT_SYMBOL_GPL(nf_ct_free_hashtable); void nf_conntrack_flush(void) { @@ -1101,8 +933,6 @@ EXPORT_SYMBOL_GPL(nf_conntrack_flush); supposed to kill the mall. */ void nf_conntrack_cleanup(void) { - int i; - rcu_assign_pointer(ip_ct_attach, NULL); /* This makes sure all current packets have passed through @@ -1123,49 +953,46 @@ void nf_conntrack_cleanup(void) rcu_assign_pointer(nf_ct_destroy, NULL); - for (i = 0; i < NF_CT_F_NUM; i++) { - if (nf_ct_cache[i].use == 0) - continue; - - NF_CT_ASSERT(nf_ct_cache[i].use == 1); - nf_ct_cache[i].use = 1; - nf_conntrack_unregister_cache(i); - } - kmem_cache_destroy(nf_conntrack_expect_cachep); - free_conntrack_hash(nf_conntrack_hash, nf_conntrack_vmalloc, - nf_conntrack_htable_size); + kmem_cache_destroy(nf_conntrack_cachep); + nf_ct_free_hashtable(nf_conntrack_hash, nf_conntrack_vmalloc, + nf_conntrack_htable_size); nf_conntrack_proto_fini(); + nf_conntrack_helper_fini(); + nf_conntrack_expect_fini(); } -static struct list_head *alloc_hashtable(int size, int *vmalloced) +struct hlist_head *nf_ct_alloc_hashtable(int *sizep, int *vmalloced) { - struct list_head *hash; - unsigned int i; + struct hlist_head *hash; + unsigned int size, i; *vmalloced = 0; + + size = *sizep = roundup(*sizep, PAGE_SIZE / sizeof(struct hlist_head)); hash = (void*)__get_free_pages(GFP_KERNEL, - get_order(sizeof(struct list_head) + get_order(sizeof(struct hlist_head) * size)); if (!hash) { *vmalloced = 1; printk(KERN_WARNING "nf_conntrack: falling back to vmalloc.\n"); - hash = vmalloc(sizeof(struct list_head) * size); + hash = vmalloc(sizeof(struct hlist_head) * size); } if (hash) for (i = 0; i < size; i++) - INIT_LIST_HEAD(&hash[i]); + INIT_HLIST_HEAD(&hash[i]); return hash; } +EXPORT_SYMBOL_GPL(nf_ct_alloc_hashtable); int set_hashsize(const char *val, struct kernel_param *kp) { int i, bucket, hashsize, vmalloced; int old_vmalloced, old_size; int rnd; - struct list_head *hash, *old_hash; + struct hlist_head *hash, *old_hash; struct nf_conntrack_tuple_hash *h; /* On boot, we can set this without any fancy locking. */ @@ -1176,7 +1003,7 @@ int set_hashsize(const char *val, struct kernel_param *kp) if (!hashsize) return -EINVAL; - hash = alloc_hashtable(hashsize, &vmalloced); + hash = nf_ct_alloc_hashtable(&hashsize, &vmalloced); if (!hash) return -ENOMEM; @@ -1186,12 +1013,12 @@ int set_hashsize(const char *val, struct kernel_param *kp) write_lock_bh(&nf_conntrack_lock); for (i = 0; i < nf_conntrack_htable_size; i++) { - while (!list_empty(&nf_conntrack_hash[i])) { - h = list_entry(nf_conntrack_hash[i].next, - struct nf_conntrack_tuple_hash, list); - list_del(&h->list); + while (!hlist_empty(&nf_conntrack_hash[i])) { + h = hlist_entry(nf_conntrack_hash[i].first, + struct nf_conntrack_tuple_hash, hnode); + hlist_del(&h->hnode); bucket = __hash_conntrack(&h->tuple, hashsize, rnd); - list_add_tail(&h->list, &hash[bucket]); + hlist_add_head(&h->hnode, &hash[bucket]); } } old_size = nf_conntrack_htable_size; @@ -1204,7 +1031,7 @@ int set_hashsize(const char *val, struct kernel_param *kp) nf_conntrack_hash_rnd = rnd; write_unlock_bh(&nf_conntrack_lock); - free_conntrack_hash(old_hash, old_vmalloced, old_size); + nf_ct_free_hashtable(old_hash, old_vmalloced, old_size); return 0; } @@ -1213,50 +1040,58 @@ module_param_call(hashsize, set_hashsize, param_get_uint, int __init nf_conntrack_init(void) { + int max_factor = 8; int ret; /* Idea from tcp.c: use 1/16384 of memory. On i386: 32MB - * machine has 256 buckets. >= 1GB machines have 8192 buckets. */ + * machine has 512 buckets. >= 1GB machines have 16384 buckets. */ if (!nf_conntrack_htable_size) { nf_conntrack_htable_size = (((num_physpages << PAGE_SHIFT) / 16384) - / sizeof(struct list_head)); + / sizeof(struct hlist_head)); if (num_physpages > (1024 * 1024 * 1024 / PAGE_SIZE)) - nf_conntrack_htable_size = 8192; - if (nf_conntrack_htable_size < 16) - nf_conntrack_htable_size = 16; + nf_conntrack_htable_size = 16384; + if (nf_conntrack_htable_size < 32) + nf_conntrack_htable_size = 32; + + /* Use a max. factor of four by default to get the same max as + * with the old struct list_heads. When a table size is given + * we use the old value of 8 to avoid reducing the max. + * entries. */ + max_factor = 4; } - nf_conntrack_max = 8 * nf_conntrack_htable_size; - - printk("nf_conntrack version %s (%u buckets, %d max)\n", - NF_CONNTRACK_VERSION, nf_conntrack_htable_size, - nf_conntrack_max); - - nf_conntrack_hash = alloc_hashtable(nf_conntrack_htable_size, - &nf_conntrack_vmalloc); + nf_conntrack_hash = nf_ct_alloc_hashtable(&nf_conntrack_htable_size, + &nf_conntrack_vmalloc); if (!nf_conntrack_hash) { printk(KERN_ERR "Unable to create nf_conntrack_hash\n"); goto err_out; } - ret = nf_conntrack_register_cache(NF_CT_F_BASIC, "nf_conntrack:basic", - sizeof(struct nf_conn)); - if (ret < 0) { + nf_conntrack_max = max_factor * nf_conntrack_htable_size; + + printk("nf_conntrack version %s (%u buckets, %d max)\n", + NF_CONNTRACK_VERSION, nf_conntrack_htable_size, + nf_conntrack_max); + + nf_conntrack_cachep = kmem_cache_create("nf_conntrack", + sizeof(struct nf_conn), + 0, 0, NULL, NULL); + if (!nf_conntrack_cachep) { printk(KERN_ERR "Unable to create nf_conn slab cache\n"); goto err_free_hash; } - nf_conntrack_expect_cachep = kmem_cache_create("nf_conntrack_expect", - sizeof(struct nf_conntrack_expect), - 0, 0, NULL, NULL); - if (!nf_conntrack_expect_cachep) { - printk(KERN_ERR "Unable to create nf_expect slab cache\n"); + ret = nf_conntrack_proto_init(); + if (ret < 0) goto err_free_conntrack_slab; - } - ret = nf_conntrack_proto_init(); + ret = nf_conntrack_expect_init(); if (ret < 0) - goto out_free_expect_slab; + goto out_fini_proto; + + ret = nf_conntrack_helper_init(); + if (ret < 0) + goto out_fini_expect; /* For use by REJECT target */ rcu_assign_pointer(ip_ct_attach, __nf_conntrack_attach); @@ -1270,13 +1105,15 @@ int __init nf_conntrack_init(void) return ret; -out_free_expect_slab: - kmem_cache_destroy(nf_conntrack_expect_cachep); +out_fini_expect: + nf_conntrack_expect_fini(); +out_fini_proto: + nf_conntrack_proto_fini(); err_free_conntrack_slab: - nf_conntrack_unregister_cache(NF_CT_F_BASIC); + kmem_cache_destroy(nf_conntrack_cachep); err_free_hash: - free_conntrack_hash(nf_conntrack_hash, nf_conntrack_vmalloc, - nf_conntrack_htable_size); + nf_ct_free_hashtable(nf_conntrack_hash, nf_conntrack_vmalloc, + nf_conntrack_htable_size); err_out: return -ENOMEM; }