gro: Optimise TCP packet reception

gro: Optimise TCP packet reception

As this function can be called more than half a million times for
10GbE, it's important to optimise it as much as we can.

This patch uses bit ops to logical ops, as well as open coding
memcmp to exploit alignment properties.

Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c
index 73266b79c19a8d90643b7239c5e6d9fe1f6a73d5..90b2f3c192ff47eb4c24d8e416d7d1e04f577677 100644 (file)
--- a/net/ipv4/tcp.c
+++ b/net/ipv4/tcp.c
@@ -2478,9 +2478,9 @@ struct sk_buff **tcp_gro_receive(struct sk_buff **head, struct sk_buff *skb)
        struct tcphdr *th2;
        unsigned int thlen;
        unsigned int flags;
-       unsigned int total;
        unsigned int mss = 1;
        int flush = 1;
+       int i;
 
        th = skb_gro_header(skb, sizeof(*th));
        if (unlikely(!th))
@@ -2504,7 +2504,7 @@ struct sk_buff **tcp_gro_receive(struct sk_buff **head, struct sk_buff *skb)
 
                th2 = tcp_hdr(p);
 
-               if (th->source != th2->source || th->dest != th2->dest) {
+               if ((th->source ^ th2->source) | (th->dest ^ th2->dest)) {
                        NAPI_GRO_CB(p)->same_flow = 0;
                        continue;
                }
@@ -2519,14 +2519,15 @@ found:
        flush |= flags & TCP_FLAG_CWR;
        flush |= (flags ^ tcp_flag_word(th2)) &
                  ~(TCP_FLAG_CWR | TCP_FLAG_FIN | TCP_FLAG_PSH);
-       flush |= th->ack_seq != th2->ack_seq || th->window != th2->window;
-       flush |= memcmp(th + 1, th2 + 1, thlen - sizeof(*th));
+       flush |= (th->ack_seq ^ th2->ack_seq) | (th->window ^ th2->window);
+       for (i = sizeof(*th); !flush && i < thlen; i += 4)
+               flush |= *(u32 *)((u8 *)th + i) ^
+                        *(u32 *)((u8 *)th2 + i);
 
-       total = skb_gro_len(p);
        mss = skb_shinfo(p)->gso_size;
 
-       flush |= skb_gro_len(skb) > mss || !skb_gro_len(skb);
-       flush |= ntohl(th2->seq) + total != ntohl(th->seq);
+       flush |= (skb_gro_len(skb) > mss) | !skb_gro_len(skb);
+       flush |= (ntohl(th2->seq) + skb_gro_len(p)) ^ ntohl(th->seq);
 
        if (flush || skb_gro_receive(head, skb)) {
                mss = 1;