| 
									
										
										
										
											2005-04-16 15:20:36 -07:00
										 |  |  | #ifndef __LINUX_TC_EM_META_H
 | 
					
						
							|  |  |  | #define __LINUX_TC_EM_META_H
 | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2009-01-30 20:12:59 +05:30
										 |  |  | #include <linux/types.h>
 | 
					
						
							| 
									
										
										
										
											2005-04-16 15:20:36 -07:00
										 |  |  | #include <linux/pkt_cls.h>
 | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2009-11-04 09:50:58 -08:00
										 |  |  | enum { | 
					
						
							| 
									
										
										
										
											2005-04-16 15:20:36 -07:00
										 |  |  | 	TCA_EM_META_UNSPEC, | 
					
						
							|  |  |  | 	TCA_EM_META_HDR, | 
					
						
							|  |  |  | 	TCA_EM_META_LVALUE, | 
					
						
							|  |  |  | 	TCA_EM_META_RVALUE, | 
					
						
							|  |  |  | 	__TCA_EM_META_MAX | 
					
						
							|  |  |  | }; | 
					
						
							|  |  |  | #define TCA_EM_META_MAX (__TCA_EM_META_MAX - 1)
 | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2009-11-04 09:50:58 -08:00
										 |  |  | struct tcf_meta_val { | 
					
						
							| 
									
										
										
										
											2005-04-16 15:20:36 -07:00
										 |  |  | 	__u16			kind; | 
					
						
							|  |  |  | 	__u8			shift; | 
					
						
							|  |  |  | 	__u8			op; | 
					
						
							|  |  |  | }; | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | #define TCF_META_TYPE_MASK	(0xf << 12)
 | 
					
						
							|  |  |  | #define TCF_META_TYPE(kind)	(((kind) & TCF_META_TYPE_MASK) >> 12)
 | 
					
						
							|  |  |  | #define TCF_META_ID_MASK	0x7ff
 | 
					
						
							|  |  |  | #define TCF_META_ID(kind)	((kind) & TCF_META_ID_MASK)
 | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2009-11-04 09:50:58 -08:00
										 |  |  | enum { | 
					
						
							| 
									
										
										
										
											2005-04-16 15:20:36 -07:00
										 |  |  | 	TCF_META_TYPE_VAR, | 
					
						
							|  |  |  | 	TCF_META_TYPE_INT, | 
					
						
							|  |  |  | 	__TCF_META_TYPE_MAX | 
					
						
							|  |  |  | }; | 
					
						
							|  |  |  | #define TCF_META_TYPE_MAX (__TCF_META_TYPE_MAX - 1)
 | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2009-11-04 09:50:58 -08:00
										 |  |  | enum { | 
					
						
							| 
									
										
										
										
											2005-04-16 15:20:36 -07:00
										 |  |  | 	TCF_META_ID_VALUE, | 
					
						
							|  |  |  | 	TCF_META_ID_RANDOM, | 
					
						
							|  |  |  | 	TCF_META_ID_LOADAVG_0, | 
					
						
							|  |  |  | 	TCF_META_ID_LOADAVG_1, | 
					
						
							|  |  |  | 	TCF_META_ID_LOADAVG_2, | 
					
						
							|  |  |  | 	TCF_META_ID_DEV, | 
					
						
							|  |  |  | 	TCF_META_ID_PRIORITY, | 
					
						
							|  |  |  | 	TCF_META_ID_PROTOCOL, | 
					
						
							|  |  |  | 	TCF_META_ID_PKTTYPE, | 
					
						
							|  |  |  | 	TCF_META_ID_PKTLEN, | 
					
						
							|  |  |  | 	TCF_META_ID_DATALEN, | 
					
						
							|  |  |  | 	TCF_META_ID_MACLEN, | 
					
						
							|  |  |  | 	TCF_META_ID_NFMARK, | 
					
						
							|  |  |  | 	TCF_META_ID_TCINDEX, | 
					
						
							|  |  |  | 	TCF_META_ID_RTCLASSID, | 
					
						
							|  |  |  | 	TCF_META_ID_RTIIF, | 
					
						
							| 
									
										
										
										
											2005-06-08 15:10:48 -07:00
										 |  |  | 	TCF_META_ID_SK_FAMILY, | 
					
						
							|  |  |  | 	TCF_META_ID_SK_STATE, | 
					
						
							|  |  |  | 	TCF_META_ID_SK_REUSE, | 
					
						
							|  |  |  | 	TCF_META_ID_SK_BOUND_IF, | 
					
						
							|  |  |  | 	TCF_META_ID_SK_REFCNT, | 
					
						
							|  |  |  | 	TCF_META_ID_SK_SHUTDOWN, | 
					
						
							|  |  |  | 	TCF_META_ID_SK_PROTO, | 
					
						
							|  |  |  | 	TCF_META_ID_SK_TYPE, | 
					
						
							|  |  |  | 	TCF_META_ID_SK_RCVBUF, | 
					
						
							|  |  |  | 	TCF_META_ID_SK_RMEM_ALLOC, | 
					
						
							|  |  |  | 	TCF_META_ID_SK_WMEM_ALLOC, | 
					
						
							|  |  |  | 	TCF_META_ID_SK_OMEM_ALLOC, | 
					
						
							|  |  |  | 	TCF_META_ID_SK_WMEM_QUEUED, | 
					
						
							|  |  |  | 	TCF_META_ID_SK_RCV_QLEN, | 
					
						
							|  |  |  | 	TCF_META_ID_SK_SND_QLEN, | 
					
						
							|  |  |  |  	TCF_META_ID_SK_ERR_QLEN, | 
					
						
							|  |  |  | 	TCF_META_ID_SK_FORWARD_ALLOCS, | 
					
						
							|  |  |  | 	TCF_META_ID_SK_SNDBUF, | 
					
						
							|  |  |  |  	TCF_META_ID_SK_ALLOCS, | 
					
						
							| 
									
										
										
										
											2011-07-14 14:45:59 -07:00
										 |  |  | 	__TCF_META_ID_SK_ROUTE_CAPS,	/* unimplemented but in ABI already */ | 
					
						
							| 
									
										
											  
											
												[INET]: speedup inet (tcp/dccp) lookups
Arnaldo and I agreed it could be applied now, because I have other
pending patches depending on this one (Thank you Arnaldo)
(The other important patch moves skc_refcnt in a separate cache line,
so that the SMP/NUMA performance doesnt suffer from cache line ping pongs)
1) First some performance data :
--------------------------------
tcp_v4_rcv() wastes a *lot* of time in __inet_lookup_established()
The most time critical code is :
sk_for_each(sk, node, &head->chain) {
     if (INET_MATCH(sk, acookie, saddr, daddr, ports, dif))
         goto hit; /* You sunk my battleship! */
}
The sk_for_each() does use prefetch() hints but only the begining of
"struct sock" is prefetched.
As INET_MATCH first comparison uses inet_sk(__sk)->daddr, wich is far
away from the begining of "struct sock", it has to bring into CPU
cache cold cache line. Each iteration has to use at least 2 cache
lines.
This can be problematic if some chains are very long.
2) The goal
-----------
The idea I had is to change things so that INET_MATCH() may return
FALSE in 99% of cases only using the data already in the CPU cache,
using one cache line per iteration.
3) Description of the patch
---------------------------
Adds a new 'unsigned int skc_hash' field in 'struct sock_common',
filling a 32 bits hole on 64 bits platform.
struct sock_common {
	unsigned short		skc_family;
	volatile unsigned char	skc_state;
	unsigned char		skc_reuse;
	int			skc_bound_dev_if;
	struct hlist_node	skc_node;
	struct hlist_node	skc_bind_node;
	atomic_t		skc_refcnt;
+	unsigned int		skc_hash;
	struct proto		*skc_prot;
};
Store in this 32 bits field the full hash, not masked by (ehash_size -
1) Using this full hash as the first comparison done in INET_MATCH
permits us immediatly skip the element without touching a second cache
line in case of a miss.
Suppress the sk_hashent/tw_hashent fields since skc_hash (aliased to
sk_hash and tw_hash) already contains the slot number if we mask with
(ehash_size - 1)
File include/net/inet_hashtables.h
64 bits platforms :
#define INET_MATCH(__sk, __hash, __cookie, __saddr, __daddr, __ports, __dif)\
     (((__sk)->sk_hash == (__hash))
     ((*((__u64 *)&(inet_sk(__sk)->daddr)))== (__cookie))   &&  \
     ((*((__u32 *)&(inet_sk(__sk)->dport))) == (__ports))   &&  \
     (!((__sk)->sk_bound_dev_if) || ((__sk)->sk_bound_dev_if == (__dif))))
32bits platforms:
#define TCP_IPV4_MATCH(__sk, __hash, __cookie, __saddr, __daddr, __ports, __dif)\
     (((__sk)->sk_hash == (__hash))                 &&  \
     (inet_sk(__sk)->daddr          == (__saddr))   &&  \
     (inet_sk(__sk)->rcv_saddr      == (__daddr))   &&  \
     (!((__sk)->sk_bound_dev_if) || ((__sk)->sk_bound_dev_if == (__dif))))
- Adds a prefetch(head->chain.first) in 
__inet_lookup_established()/__tcp_v4_check_established() and 
__inet6_lookup_established()/__tcp_v6_check_established() and 
__dccp_v4_check_established() to bring into cache the first element of the 
list, before the {read|write}_lock(&head->lock);
Signed-off-by: Eric Dumazet <dada1@cosmosbay.com>
Acked-by: Arnaldo Carvalho de Melo <acme@ghostprotocols.net>
Signed-off-by: David S. Miller <davem@davemloft.net>
											
										 
											2005-10-03 14:13:38 -07:00
										 |  |  |  	TCF_META_ID_SK_HASH, | 
					
						
							| 
									
										
										
										
											2005-06-08 15:10:48 -07:00
										 |  |  |  	TCF_META_ID_SK_LINGERTIME, | 
					
						
							|  |  |  |  	TCF_META_ID_SK_ACK_BACKLOG, | 
					
						
							|  |  |  |  	TCF_META_ID_SK_MAX_ACK_BACKLOG, | 
					
						
							|  |  |  |  	TCF_META_ID_SK_PRIO, | 
					
						
							|  |  |  |  	TCF_META_ID_SK_RCVLOWAT, | 
					
						
							|  |  |  |  	TCF_META_ID_SK_RCVTIMEO, | 
					
						
							|  |  |  |  	TCF_META_ID_SK_SNDTIMEO, | 
					
						
							|  |  |  |  	TCF_META_ID_SK_SENDMSG_OFF, | 
					
						
							|  |  |  |  	TCF_META_ID_SK_WRITE_PENDING, | 
					
						
							| 
									
										
										
										
											2008-02-05 03:20:13 -08:00
										 |  |  | 	TCF_META_ID_VLAN_TAG, | 
					
						
							| 
									
										
										
										
											2010-08-24 14:48:10 -07:00
										 |  |  | 	TCF_META_ID_RXHASH, | 
					
						
							| 
									
										
										
										
											2005-04-16 15:20:36 -07:00
										 |  |  | 	__TCF_META_ID_MAX | 
					
						
							|  |  |  | }; | 
					
						
							|  |  |  | #define TCF_META_ID_MAX (__TCF_META_ID_MAX - 1)
 | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2009-11-04 09:50:58 -08:00
										 |  |  | struct tcf_meta_hdr { | 
					
						
							| 
									
										
										
										
											2005-04-16 15:20:36 -07:00
										 |  |  | 	struct tcf_meta_val	left; | 
					
						
							|  |  |  | 	struct tcf_meta_val	right; | 
					
						
							|  |  |  | }; | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | #endif
 |