2005-08-10 03:09:30 +00:00
|
|
|
/*
|
|
|
|
|
* INET An implementation of the TCP/IP protocol suite for the LINUX
|
|
|
|
|
* operating system. INET is implemented using the BSD Socket
|
|
|
|
|
* interface as the means of communication with the user level.
|
|
|
|
|
*
|
|
|
|
|
* Definitions for a generic INET TIMEWAIT sock
|
|
|
|
|
*
|
|
|
|
|
* From code originally in net/tcp.h
|
|
|
|
|
*
|
|
|
|
|
* This program is free software; you can redistribute it and/or
|
|
|
|
|
* modify it under the terms of the GNU General Public License
|
|
|
|
|
* as published by the Free Software Foundation; either version
|
|
|
|
|
* 2 of the License, or (at your option) any later version.
|
|
|
|
|
*/
|
|
|
|
|
#ifndef _INET_TIMEWAIT_SOCK_
|
|
|
|
|
#define _INET_TIMEWAIT_SOCK_
|
|
|
|
|
|
|
|
|
|
|
2008-10-18 15:37:51 +00:00
|
|
|
#include <linux/kmemcheck.h>
|
2005-08-10 03:09:30 +00:00
|
|
|
#include <linux/list.h>
|
2005-08-10 03:44:40 +00:00
|
|
|
#include <linux/timer.h>
|
2005-08-10 03:09:30 +00:00
|
|
|
#include <linux/types.h>
|
2005-08-10 03:44:40 +00:00
|
|
|
#include <linux/workqueue.h>
|
2005-08-10 03:09:30 +00:00
|
|
|
|
2005-12-27 04:43:12 +00:00
|
|
|
#include <net/inet_sock.h>
|
2005-08-10 03:09:30 +00:00
|
|
|
#include <net/sock.h>
|
|
|
|
|
#include <net/tcp_states.h>
|
2005-12-14 07:25:19 +00:00
|
|
|
#include <net/timewait_sock.h>
|
2005-08-10 03:09:30 +00:00
|
|
|
|
2011-07-26 23:09:06 +00:00
|
|
|
#include <linux/atomic.h>
|
2005-08-10 03:09:30 +00:00
|
|
|
|
2005-08-10 03:44:40 +00:00
|
|
|
struct inet_hashinfo;
|
|
|
|
|
|
|
|
|
|
#define INET_TWDR_RECYCLE_SLOTS_LOG 5
|
|
|
|
|
#define INET_TWDR_RECYCLE_SLOTS (1 << INET_TWDR_RECYCLE_SLOTS_LOG)
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* If time > 4sec, it is "slow" path, no recycling is required,
|
|
|
|
|
* so that we select tick to get range about 4 seconds.
|
|
|
|
|
*/
|
|
|
|
|
#if HZ <= 16 || HZ > 4096
|
|
|
|
|
# error Unsupported: HZ <= 16 or HZ > 4096
|
|
|
|
|
#elif HZ <= 32
|
|
|
|
|
# define INET_TWDR_RECYCLE_TICK (5 + 2 - INET_TWDR_RECYCLE_SLOTS_LOG)
|
|
|
|
|
#elif HZ <= 64
|
|
|
|
|
# define INET_TWDR_RECYCLE_TICK (6 + 2 - INET_TWDR_RECYCLE_SLOTS_LOG)
|
|
|
|
|
#elif HZ <= 128
|
|
|
|
|
# define INET_TWDR_RECYCLE_TICK (7 + 2 - INET_TWDR_RECYCLE_SLOTS_LOG)
|
|
|
|
|
#elif HZ <= 256
|
|
|
|
|
# define INET_TWDR_RECYCLE_TICK (8 + 2 - INET_TWDR_RECYCLE_SLOTS_LOG)
|
|
|
|
|
#elif HZ <= 512
|
|
|
|
|
# define INET_TWDR_RECYCLE_TICK (9 + 2 - INET_TWDR_RECYCLE_SLOTS_LOG)
|
|
|
|
|
#elif HZ <= 1024
|
|
|
|
|
# define INET_TWDR_RECYCLE_TICK (10 + 2 - INET_TWDR_RECYCLE_SLOTS_LOG)
|
|
|
|
|
#elif HZ <= 2048
|
|
|
|
|
# define INET_TWDR_RECYCLE_TICK (11 + 2 - INET_TWDR_RECYCLE_SLOTS_LOG)
|
|
|
|
|
#else
|
|
|
|
|
# define INET_TWDR_RECYCLE_TICK (12 + 2 - INET_TWDR_RECYCLE_SLOTS_LOG)
|
|
|
|
|
#endif
|
|
|
|
|
|
2013-10-03 21:27:25 +00:00
|
|
|
static inline u32 inet_tw_time_stamp(void)
|
|
|
|
|
{
|
|
|
|
|
return jiffies;
|
|
|
|
|
}
|
|
|
|
|
|
2005-08-10 03:44:40 +00:00
|
|
|
/* TIME_WAIT reaping mechanism. */
|
|
|
|
|
#define INET_TWDR_TWKILL_SLOTS 8 /* Please keep this a power of 2. */
|
|
|
|
|
|
|
|
|
|
#define INET_TWDR_TWKILL_QUOTA 100
|
|
|
|
|
|
|
|
|
|
struct inet_timewait_death_row {
|
|
|
|
|
/* Short-time timewait calendar */
|
|
|
|
|
int twcal_hand;
|
2007-03-05 21:32:48 +00:00
|
|
|
unsigned long twcal_jiffie;
|
2005-08-10 03:44:40 +00:00
|
|
|
struct timer_list twcal_timer;
|
|
|
|
|
struct hlist_head twcal_row[INET_TWDR_RECYCLE_SLOTS];
|
|
|
|
|
|
|
|
|
|
spinlock_t death_lock;
|
|
|
|
|
int tw_count;
|
|
|
|
|
int period;
|
|
|
|
|
u32 thread_slots;
|
|
|
|
|
struct work_struct twkill_work;
|
|
|
|
|
struct timer_list tw_timer;
|
|
|
|
|
int slot;
|
|
|
|
|
struct hlist_head cells[INET_TWDR_TWKILL_SLOTS];
|
|
|
|
|
struct inet_hashinfo *hashinfo;
|
|
|
|
|
int sysctl_tw_recycle;
|
|
|
|
|
int sysctl_max_tw_buckets;
|
|
|
|
|
};
|
|
|
|
|
|
2013-09-21 17:22:41 +00:00
|
|
|
void inet_twdr_hangman(unsigned long data);
|
|
|
|
|
void inet_twdr_twkill_work(struct work_struct *work);
|
|
|
|
|
void inet_twdr_twcal_tick(unsigned long data);
|
2005-08-10 03:45:03 +00:00
|
|
|
|
2005-08-10 03:09:30 +00:00
|
|
|
struct inet_bind_bucket;
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* This is a TIME_WAIT sock. It works around the memory consumption
|
|
|
|
|
* problems of sockets in such a state on heavily loaded servers, but
|
|
|
|
|
* without violating the protocol specification.
|
|
|
|
|
*/
|
|
|
|
|
struct inet_timewait_sock {
|
|
|
|
|
/*
|
|
|
|
|
* Now struct sock also uses sock_common, so please just
|
|
|
|
|
* don't add nothing before this first member (__tw_common) --acme
|
|
|
|
|
*/
|
|
|
|
|
struct sock_common __tw_common;
|
|
|
|
|
#define tw_family __tw_common.skc_family
|
|
|
|
|
#define tw_state __tw_common.skc_state
|
|
|
|
|
#define tw_reuse __tw_common.skc_reuse
|
|
|
|
|
#define tw_bound_dev_if __tw_common.skc_bound_dev_if
|
2008-11-17 03:40:17 +00:00
|
|
|
#define tw_node __tw_common.skc_nulls_node
|
2005-08-10 03:09:30 +00:00
|
|
|
#define tw_bind_node __tw_common.skc_bind_node
|
|
|
|
|
#define tw_refcnt __tw_common.skc_refcnt
|
[INET]: speedup inet (tcp/dccp) lookups
Arnaldo and I agreed it could be applied now, because I have other
pending patches depending on this one (Thank you Arnaldo)
(The other important patch moves skc_refcnt in a separate cache line,
so that the SMP/NUMA performance doesnt suffer from cache line ping pongs)
1) First some performance data :
--------------------------------
tcp_v4_rcv() wastes a *lot* of time in __inet_lookup_established()
The most time critical code is :
sk_for_each(sk, node, &head->chain) {
if (INET_MATCH(sk, acookie, saddr, daddr, ports, dif))
goto hit; /* You sunk my battleship! */
}
The sk_for_each() does use prefetch() hints but only the begining of
"struct sock" is prefetched.
As INET_MATCH first comparison uses inet_sk(__sk)->daddr, wich is far
away from the begining of "struct sock", it has to bring into CPU
cache cold cache line. Each iteration has to use at least 2 cache
lines.
This can be problematic if some chains are very long.
2) The goal
-----------
The idea I had is to change things so that INET_MATCH() may return
FALSE in 99% of cases only using the data already in the CPU cache,
using one cache line per iteration.
3) Description of the patch
---------------------------
Adds a new 'unsigned int skc_hash' field in 'struct sock_common',
filling a 32 bits hole on 64 bits platform.
struct sock_common {
unsigned short skc_family;
volatile unsigned char skc_state;
unsigned char skc_reuse;
int skc_bound_dev_if;
struct hlist_node skc_node;
struct hlist_node skc_bind_node;
atomic_t skc_refcnt;
+ unsigned int skc_hash;
struct proto *skc_prot;
};
Store in this 32 bits field the full hash, not masked by (ehash_size -
1) Using this full hash as the first comparison done in INET_MATCH
permits us immediatly skip the element without touching a second cache
line in case of a miss.
Suppress the sk_hashent/tw_hashent fields since skc_hash (aliased to
sk_hash and tw_hash) already contains the slot number if we mask with
(ehash_size - 1)
File include/net/inet_hashtables.h
64 bits platforms :
#define INET_MATCH(__sk, __hash, __cookie, __saddr, __daddr, __ports, __dif)\
(((__sk)->sk_hash == (__hash))
((*((__u64 *)&(inet_sk(__sk)->daddr)))== (__cookie)) && \
((*((__u32 *)&(inet_sk(__sk)->dport))) == (__ports)) && \
(!((__sk)->sk_bound_dev_if) || ((__sk)->sk_bound_dev_if == (__dif))))
32bits platforms:
#define TCP_IPV4_MATCH(__sk, __hash, __cookie, __saddr, __daddr, __ports, __dif)\
(((__sk)->sk_hash == (__hash)) && \
(inet_sk(__sk)->daddr == (__saddr)) && \
(inet_sk(__sk)->rcv_saddr == (__daddr)) && \
(!((__sk)->sk_bound_dev_if) || ((__sk)->sk_bound_dev_if == (__dif))))
- Adds a prefetch(head->chain.first) in
__inet_lookup_established()/__tcp_v4_check_established() and
__inet6_lookup_established()/__tcp_v6_check_established() and
__dccp_v4_check_established() to bring into cache the first element of the
list, before the {read|write}_lock(&head->lock);
Signed-off-by: Eric Dumazet <dada1@cosmosbay.com>
Acked-by: Arnaldo Carvalho de Melo <acme@ghostprotocols.net>
Signed-off-by: David S. Miller <davem@davemloft.net>
2005-10-03 21:13:38 +00:00
|
|
|
#define tw_hash __tw_common.skc_hash
|
2005-08-10 03:09:30 +00:00
|
|
|
#define tw_prot __tw_common.skc_prot
|
2007-09-12 09:58:02 +00:00
|
|
|
#define tw_net __tw_common.skc_net
|
net: optimize INET input path further
Followup of commit b178bb3dfc30 (net: reorder struct sock fields)
Optimize INET input path a bit further, by :
1) moving sk_refcnt close to sk_lock.
This reduces number of dirtied cache lines by one on 64bit arches (and
64 bytes cache line size).
2) moving inet_daddr & inet_rcv_saddr at the beginning of sk
(same cache line than hash / family / bound_dev_if / nulls_node)
This reduces number of accessed cache lines in lookups by one, and dont
increase size of inet and timewait socks.
inet and tw sockets now share same place-holder for these fields.
Before patch :
offsetof(struct sock, sk_refcnt) = 0x10
offsetof(struct sock, sk_lock) = 0x40
offsetof(struct sock, sk_receive_queue) = 0x60
offsetof(struct inet_sock, inet_daddr) = 0x270
offsetof(struct inet_sock, inet_rcv_saddr) = 0x274
After patch :
offsetof(struct sock, sk_refcnt) = 0x44
offsetof(struct sock, sk_lock) = 0x48
offsetof(struct sock, sk_receive_queue) = 0x68
offsetof(struct inet_sock, inet_daddr) = 0x0
offsetof(struct inet_sock, inet_rcv_saddr) = 0x4
compute_score() (udp or tcp) now use a single cache line per ignored
item, instead of two.
Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2010-11-30 19:04:07 +00:00
|
|
|
#define tw_daddr __tw_common.skc_daddr
|
ipv6: make lookups simpler and faster
TCP listener refactoring, part 4 :
To speed up inet lookups, we moved IPv4 addresses from inet to struct
sock_common
Now is time to do the same for IPv6, because it permits us to have fast
lookups for all kind of sockets, including upcoming SYN_RECV.
Getting IPv6 addresses in TCP lookups currently requires two extra cache
lines, plus a dereference (and memory stall).
inet6_sk(sk) does the dereference of inet_sk(__sk)->pinet6
This patch is way bigger than its IPv4 counter part, because for IPv4,
we could add aliases (inet_daddr, inet_rcv_saddr), while on IPv6,
it's not doable easily.
inet6_sk(sk)->daddr becomes sk->sk_v6_daddr
inet6_sk(sk)->rcv_saddr becomes sk->sk_v6_rcv_saddr
And timewait socket also have tw->tw_v6_daddr & tw->tw_v6_rcv_saddr
at the same offset.
We get rid of INET6_TW_MATCH() as INET6_MATCH() is now the generic
macro.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-10-03 22:42:29 +00:00
|
|
|
#define tw_v6_daddr __tw_common.skc_v6_daddr
|
net: optimize INET input path further
Followup of commit b178bb3dfc30 (net: reorder struct sock fields)
Optimize INET input path a bit further, by :
1) moving sk_refcnt close to sk_lock.
This reduces number of dirtied cache lines by one on 64bit arches (and
64 bytes cache line size).
2) moving inet_daddr & inet_rcv_saddr at the beginning of sk
(same cache line than hash / family / bound_dev_if / nulls_node)
This reduces number of accessed cache lines in lookups by one, and dont
increase size of inet and timewait socks.
inet and tw sockets now share same place-holder for these fields.
Before patch :
offsetof(struct sock, sk_refcnt) = 0x10
offsetof(struct sock, sk_lock) = 0x40
offsetof(struct sock, sk_receive_queue) = 0x60
offsetof(struct inet_sock, inet_daddr) = 0x270
offsetof(struct inet_sock, inet_rcv_saddr) = 0x274
After patch :
offsetof(struct sock, sk_refcnt) = 0x44
offsetof(struct sock, sk_lock) = 0x48
offsetof(struct sock, sk_receive_queue) = 0x68
offsetof(struct inet_sock, inet_daddr) = 0x0
offsetof(struct inet_sock, inet_rcv_saddr) = 0x4
compute_score() (udp or tcp) now use a single cache line per ignored
item, instead of two.
Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2010-11-30 19:04:07 +00:00
|
|
|
#define tw_rcv_saddr __tw_common.skc_rcv_saddr
|
ipv6: make lookups simpler and faster
TCP listener refactoring, part 4 :
To speed up inet lookups, we moved IPv4 addresses from inet to struct
sock_common
Now is time to do the same for IPv6, because it permits us to have fast
lookups for all kind of sockets, including upcoming SYN_RECV.
Getting IPv6 addresses in TCP lookups currently requires two extra cache
lines, plus a dereference (and memory stall).
inet6_sk(sk) does the dereference of inet_sk(__sk)->pinet6
This patch is way bigger than its IPv4 counter part, because for IPv4,
we could add aliases (inet_daddr, inet_rcv_saddr), while on IPv6,
it's not doable easily.
inet6_sk(sk)->daddr becomes sk->sk_v6_daddr
inet6_sk(sk)->rcv_saddr becomes sk->sk_v6_rcv_saddr
And timewait socket also have tw->tw_v6_daddr & tw->tw_v6_rcv_saddr
at the same offset.
We get rid of INET6_TW_MATCH() as INET6_MATCH() is now the generic
macro.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-10-03 22:42:29 +00:00
|
|
|
#define tw_v6_rcv_saddr __tw_common.skc_v6_rcv_saddr
|
2012-11-30 09:49:27 +00:00
|
|
|
#define tw_dport __tw_common.skc_dport
|
|
|
|
|
#define tw_num __tw_common.skc_num
|
|
|
|
|
|
2008-02-03 12:08:26 +00:00
|
|
|
int tw_timeout;
|
2005-08-10 03:09:30 +00:00
|
|
|
volatile unsigned char tw_substate;
|
|
|
|
|
unsigned char tw_rcv_wscale;
|
net: optimize INET input path further
Followup of commit b178bb3dfc30 (net: reorder struct sock fields)
Optimize INET input path a bit further, by :
1) moving sk_refcnt close to sk_lock.
This reduces number of dirtied cache lines by one on 64bit arches (and
64 bytes cache line size).
2) moving inet_daddr & inet_rcv_saddr at the beginning of sk
(same cache line than hash / family / bound_dev_if / nulls_node)
This reduces number of accessed cache lines in lookups by one, and dont
increase size of inet and timewait socks.
inet and tw sockets now share same place-holder for these fields.
Before patch :
offsetof(struct sock, sk_refcnt) = 0x10
offsetof(struct sock, sk_lock) = 0x40
offsetof(struct sock, sk_receive_queue) = 0x60
offsetof(struct inet_sock, inet_daddr) = 0x270
offsetof(struct inet_sock, inet_rcv_saddr) = 0x274
After patch :
offsetof(struct sock, sk_refcnt) = 0x44
offsetof(struct sock, sk_lock) = 0x48
offsetof(struct sock, sk_receive_queue) = 0x68
offsetof(struct inet_sock, inet_daddr) = 0x0
offsetof(struct inet_sock, inet_rcv_saddr) = 0x4
compute_score() (udp or tcp) now use a single cache line per ignored
item, instead of two.
Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2010-11-30 19:04:07 +00:00
|
|
|
|
2005-08-10 03:09:30 +00:00
|
|
|
/* Socket demultiplex comparisons on incoming packets. */
|
net: optimize INET input path further
Followup of commit b178bb3dfc30 (net: reorder struct sock fields)
Optimize INET input path a bit further, by :
1) moving sk_refcnt close to sk_lock.
This reduces number of dirtied cache lines by one on 64bit arches (and
64 bytes cache line size).
2) moving inet_daddr & inet_rcv_saddr at the beginning of sk
(same cache line than hash / family / bound_dev_if / nulls_node)
This reduces number of accessed cache lines in lookups by one, and dont
increase size of inet and timewait socks.
inet and tw sockets now share same place-holder for these fields.
Before patch :
offsetof(struct sock, sk_refcnt) = 0x10
offsetof(struct sock, sk_lock) = 0x40
offsetof(struct sock, sk_receive_queue) = 0x60
offsetof(struct inet_sock, inet_daddr) = 0x270
offsetof(struct inet_sock, inet_rcv_saddr) = 0x274
After patch :
offsetof(struct sock, sk_refcnt) = 0x44
offsetof(struct sock, sk_lock) = 0x48
offsetof(struct sock, sk_receive_queue) = 0x68
offsetof(struct inet_sock, inet_daddr) = 0x0
offsetof(struct inet_sock, inet_rcv_saddr) = 0x4
compute_score() (udp or tcp) now use a single cache line per ignored
item, instead of two.
Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2010-11-30 19:04:07 +00:00
|
|
|
/* these three are in inet_sock */
|
2006-09-28 01:43:50 +00:00
|
|
|
__be16 tw_sport;
|
2008-10-18 15:37:51 +00:00
|
|
|
kmemcheck_bitfield_begin(flags);
|
2005-08-10 03:09:30 +00:00
|
|
|
/* And these are ours. */
|
2009-10-18 22:48:51 +00:00
|
|
|
unsigned int tw_ipv6only : 1,
|
|
|
|
|
tw_transparent : 1,
|
2011-10-24 07:06:21 +00:00
|
|
|
tw_pad : 6, /* 6 bits hole */
|
|
|
|
|
tw_tos : 8,
|
2013-10-14 19:36:32 +00:00
|
|
|
tw_pad2 : 16; /* 16 bits hole */
|
2008-10-18 15:37:51 +00:00
|
|
|
kmemcheck_bitfield_end(flags);
|
2013-10-03 21:27:25 +00:00
|
|
|
u32 tw_ttd;
|
2005-08-10 03:09:30 +00:00
|
|
|
struct inet_bind_bucket *tw_tb;
|
|
|
|
|
struct hlist_node tw_death_node;
|
|
|
|
|
};
|
2011-10-27 04:44:35 +00:00
|
|
|
#define tw_tclass tw_tos
|
2005-08-10 03:09:30 +00:00
|
|
|
|
|
|
|
|
static inline int inet_twsk_dead_hashed(const struct inet_timewait_sock *tw)
|
|
|
|
|
{
|
2006-04-28 22:21:23 +00:00
|
|
|
return !hlist_unhashed(&tw->tw_death_node);
|
2005-08-10 03:09:30 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void inet_twsk_dead_node_init(struct inet_timewait_sock *tw)
|
|
|
|
|
{
|
|
|
|
|
tw->tw_death_node.pprev = NULL;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void __inet_twsk_del_dead_node(struct inet_timewait_sock *tw)
|
|
|
|
|
{
|
|
|
|
|
__hlist_del(&tw->tw_death_node);
|
|
|
|
|
inet_twsk_dead_node_init(tw);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline int inet_twsk_del_dead_node(struct inet_timewait_sock *tw)
|
|
|
|
|
{
|
|
|
|
|
if (inet_twsk_dead_hashed(tw)) {
|
|
|
|
|
__inet_twsk_del_dead_node(tw);
|
|
|
|
|
return 1;
|
|
|
|
|
}
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#define inet_twsk_for_each(tw, node, head) \
|
2008-11-17 03:40:17 +00:00
|
|
|
hlist_nulls_for_each_entry(tw, node, head, tw_node)
|
2005-08-10 03:09:30 +00:00
|
|
|
|
hlist: drop the node parameter from iterators
I'm not sure why, but the hlist for each entry iterators were conceived
list_for_each_entry(pos, head, member)
The hlist ones were greedy and wanted an extra parameter:
hlist_for_each_entry(tpos, pos, head, member)
Why did they need an extra pos parameter? I'm not quite sure. Not only
they don't really need it, it also prevents the iterator from looking
exactly like the list iterator, which is unfortunate.
Besides the semantic patch, there was some manual work required:
- Fix up the actual hlist iterators in linux/list.h
- Fix up the declaration of other iterators based on the hlist ones.
- A very small amount of places were using the 'node' parameter, this
was modified to use 'obj->member' instead.
- Coccinelle didn't handle the hlist_for_each_entry_safe iterator
properly, so those had to be fixed up manually.
The semantic patch which is mostly the work of Peter Senna Tschudin is here:
@@
iterator name hlist_for_each_entry, hlist_for_each_entry_continue, hlist_for_each_entry_from, hlist_for_each_entry_rcu, hlist_for_each_entry_rcu_bh, hlist_for_each_entry_continue_rcu_bh, for_each_busy_worker, ax25_uid_for_each, ax25_for_each, inet_bind_bucket_for_each, sctp_for_each_hentry, sk_for_each, sk_for_each_rcu, sk_for_each_from, sk_for_each_safe, sk_for_each_bound, hlist_for_each_entry_safe, hlist_for_each_entry_continue_rcu, nr_neigh_for_each, nr_neigh_for_each_safe, nr_node_for_each, nr_node_for_each_safe, for_each_gfn_indirect_valid_sp, for_each_gfn_sp, for_each_host;
type T;
expression a,c,d,e;
identifier b;
statement S;
@@
-T b;
<+... when != b
(
hlist_for_each_entry(a,
- b,
c, d) S
|
hlist_for_each_entry_continue(a,
- b,
c) S
|
hlist_for_each_entry_from(a,
- b,
c) S
|
hlist_for_each_entry_rcu(a,
- b,
c, d) S
|
hlist_for_each_entry_rcu_bh(a,
- b,
c, d) S
|
hlist_for_each_entry_continue_rcu_bh(a,
- b,
c) S
|
for_each_busy_worker(a, c,
- b,
d) S
|
ax25_uid_for_each(a,
- b,
c) S
|
ax25_for_each(a,
- b,
c) S
|
inet_bind_bucket_for_each(a,
- b,
c) S
|
sctp_for_each_hentry(a,
- b,
c) S
|
sk_for_each(a,
- b,
c) S
|
sk_for_each_rcu(a,
- b,
c) S
|
sk_for_each_from
-(a, b)
+(a)
S
+ sk_for_each_from(a) S
|
sk_for_each_safe(a,
- b,
c, d) S
|
sk_for_each_bound(a,
- b,
c) S
|
hlist_for_each_entry_safe(a,
- b,
c, d, e) S
|
hlist_for_each_entry_continue_rcu(a,
- b,
c) S
|
nr_neigh_for_each(a,
- b,
c) S
|
nr_neigh_for_each_safe(a,
- b,
c, d) S
|
nr_node_for_each(a,
- b,
c) S
|
nr_node_for_each_safe(a,
- b,
c, d) S
|
- for_each_gfn_sp(a, c, d, b) S
+ for_each_gfn_sp(a, c, d) S
|
- for_each_gfn_indirect_valid_sp(a, c, d, b) S
+ for_each_gfn_indirect_valid_sp(a, c, d) S
|
for_each_host(a,
- b,
c) S
|
for_each_host_safe(a,
- b,
c, d) S
|
for_each_mesh_entry(a,
- b,
c, d) S
)
...+>
[akpm@linux-foundation.org: drop bogus change from net/ipv4/raw.c]
[akpm@linux-foundation.org: drop bogus hunk from net/ipv6/raw.c]
[akpm@linux-foundation.org: checkpatch fixes]
[akpm@linux-foundation.org: fix warnings]
[akpm@linux-foudnation.org: redo intrusive kvm changes]
Tested-by: Peter Senna Tschudin <peter.senna@gmail.com>
Acked-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Signed-off-by: Sasha Levin <sasha.levin@oracle.com>
Cc: Wu Fengguang <fengguang.wu@intel.com>
Cc: Marcelo Tosatti <mtosatti@redhat.com>
Cc: Gleb Natapov <gleb@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-28 01:06:00 +00:00
|
|
|
#define inet_twsk_for_each_inmate(tw, jail) \
|
|
|
|
|
hlist_for_each_entry(tw, jail, tw_death_node)
|
2005-08-10 03:09:30 +00:00
|
|
|
|
hlist: drop the node parameter from iterators
I'm not sure why, but the hlist for each entry iterators were conceived
list_for_each_entry(pos, head, member)
The hlist ones were greedy and wanted an extra parameter:
hlist_for_each_entry(tpos, pos, head, member)
Why did they need an extra pos parameter? I'm not quite sure. Not only
they don't really need it, it also prevents the iterator from looking
exactly like the list iterator, which is unfortunate.
Besides the semantic patch, there was some manual work required:
- Fix up the actual hlist iterators in linux/list.h
- Fix up the declaration of other iterators based on the hlist ones.
- A very small amount of places were using the 'node' parameter, this
was modified to use 'obj->member' instead.
- Coccinelle didn't handle the hlist_for_each_entry_safe iterator
properly, so those had to be fixed up manually.
The semantic patch which is mostly the work of Peter Senna Tschudin is here:
@@
iterator name hlist_for_each_entry, hlist_for_each_entry_continue, hlist_for_each_entry_from, hlist_for_each_entry_rcu, hlist_for_each_entry_rcu_bh, hlist_for_each_entry_continue_rcu_bh, for_each_busy_worker, ax25_uid_for_each, ax25_for_each, inet_bind_bucket_for_each, sctp_for_each_hentry, sk_for_each, sk_for_each_rcu, sk_for_each_from, sk_for_each_safe, sk_for_each_bound, hlist_for_each_entry_safe, hlist_for_each_entry_continue_rcu, nr_neigh_for_each, nr_neigh_for_each_safe, nr_node_for_each, nr_node_for_each_safe, for_each_gfn_indirect_valid_sp, for_each_gfn_sp, for_each_host;
type T;
expression a,c,d,e;
identifier b;
statement S;
@@
-T b;
<+... when != b
(
hlist_for_each_entry(a,
- b,
c, d) S
|
hlist_for_each_entry_continue(a,
- b,
c) S
|
hlist_for_each_entry_from(a,
- b,
c) S
|
hlist_for_each_entry_rcu(a,
- b,
c, d) S
|
hlist_for_each_entry_rcu_bh(a,
- b,
c, d) S
|
hlist_for_each_entry_continue_rcu_bh(a,
- b,
c) S
|
for_each_busy_worker(a, c,
- b,
d) S
|
ax25_uid_for_each(a,
- b,
c) S
|
ax25_for_each(a,
- b,
c) S
|
inet_bind_bucket_for_each(a,
- b,
c) S
|
sctp_for_each_hentry(a,
- b,
c) S
|
sk_for_each(a,
- b,
c) S
|
sk_for_each_rcu(a,
- b,
c) S
|
sk_for_each_from
-(a, b)
+(a)
S
+ sk_for_each_from(a) S
|
sk_for_each_safe(a,
- b,
c, d) S
|
sk_for_each_bound(a,
- b,
c) S
|
hlist_for_each_entry_safe(a,
- b,
c, d, e) S
|
hlist_for_each_entry_continue_rcu(a,
- b,
c) S
|
nr_neigh_for_each(a,
- b,
c) S
|
nr_neigh_for_each_safe(a,
- b,
c, d) S
|
nr_node_for_each(a,
- b,
c) S
|
nr_node_for_each_safe(a,
- b,
c, d) S
|
- for_each_gfn_sp(a, c, d, b) S
+ for_each_gfn_sp(a, c, d) S
|
- for_each_gfn_indirect_valid_sp(a, c, d, b) S
+ for_each_gfn_indirect_valid_sp(a, c, d) S
|
for_each_host(a,
- b,
c) S
|
for_each_host_safe(a,
- b,
c, d) S
|
for_each_mesh_entry(a,
- b,
c, d) S
)
...+>
[akpm@linux-foundation.org: drop bogus change from net/ipv4/raw.c]
[akpm@linux-foundation.org: drop bogus hunk from net/ipv6/raw.c]
[akpm@linux-foundation.org: checkpatch fixes]
[akpm@linux-foundation.org: fix warnings]
[akpm@linux-foudnation.org: redo intrusive kvm changes]
Tested-by: Peter Senna Tschudin <peter.senna@gmail.com>
Acked-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Signed-off-by: Sasha Levin <sasha.levin@oracle.com>
Cc: Wu Fengguang <fengguang.wu@intel.com>
Cc: Marcelo Tosatti <mtosatti@redhat.com>
Cc: Gleb Natapov <gleb@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-02-28 01:06:00 +00:00
|
|
|
#define inet_twsk_for_each_inmate_safe(tw, safe, jail) \
|
|
|
|
|
hlist_for_each_entry_safe(tw, safe, jail, tw_death_node)
|
2005-08-10 03:09:30 +00:00
|
|
|
|
|
|
|
|
static inline struct inet_timewait_sock *inet_twsk(const struct sock *sk)
|
|
|
|
|
{
|
|
|
|
|
return (struct inet_timewait_sock *)sk;
|
|
|
|
|
}
|
|
|
|
|
|
tcp/dccp: remove twchain
TCP listener refactoring, part 3 :
Our goal is to hash SYN_RECV sockets into main ehash for fast lookup,
and parallel SYN processing.
Current inet_ehash_bucket contains two chains, one for ESTABLISH (and
friend states) sockets, another for TIME_WAIT sockets only.
As the hash table is sized to get at most one socket per bucket, it
makes little sense to have separate twchain, as it makes the lookup
slightly more complicated, and doubles hash table memory usage.
If we make sure all socket types have the lookup keys at the same
offsets, we can use a generic and faster lookup. It turns out TIME_WAIT
and ESTABLISHED sockets already have common lookup fields for IPv4.
[ INET_TW_MATCH() is no longer needed ]
I'll provide a follow-up to factorize IPv6 lookup as well, to remove
INET6_TW_MATCH()
This way, SYN_RECV pseudo sockets will be supported the same.
A new sock_gen_put() helper is added, doing either a sock_put() or
inet_twsk_put() [ and will support SYN_RECV later ].
Note this helper should only be called in real slow path, when rcu
lookup found a socket that was moved to another identity (freed/reused
immediately), but could eventually be used in other contexts, like
sock_edemux()
Before patch :
dmesg | grep "TCP established"
TCP established hash table entries: 524288 (order: 11, 8388608 bytes)
After patch :
TCP established hash table entries: 524288 (order: 10, 4194304 bytes)
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2013-10-03 07:22:02 +00:00
|
|
|
void inet_twsk_free(struct inet_timewait_sock *tw);
|
2013-09-21 17:22:41 +00:00
|
|
|
void inet_twsk_put(struct inet_timewait_sock *tw);
|
2005-08-10 03:09:46 +00:00
|
|
|
|
2013-09-21 17:22:41 +00:00
|
|
|
int inet_twsk_unhash(struct inet_timewait_sock *tw);
|
2009-12-02 22:31:19 +00:00
|
|
|
|
2013-09-21 17:22:41 +00:00
|
|
|
int inet_twsk_bind_unhash(struct inet_timewait_sock *tw,
|
|
|
|
|
struct inet_hashinfo *hashinfo);
|
2009-12-04 03:47:42 +00:00
|
|
|
|
2013-09-21 17:22:41 +00:00
|
|
|
struct inet_timewait_sock *inet_twsk_alloc(const struct sock *sk,
|
|
|
|
|
const int state);
|
2005-08-10 03:09:59 +00:00
|
|
|
|
2013-09-21 17:22:41 +00:00
|
|
|
void __inet_twsk_hashdance(struct inet_timewait_sock *tw, struct sock *sk,
|
|
|
|
|
struct inet_hashinfo *hashinfo);
|
2005-08-10 03:45:03 +00:00
|
|
|
|
2013-09-21 17:22:41 +00:00
|
|
|
void inet_twsk_schedule(struct inet_timewait_sock *tw,
|
|
|
|
|
struct inet_timewait_death_row *twdr,
|
|
|
|
|
const int timeo, const int timewait_len);
|
|
|
|
|
void inet_twsk_deschedule(struct inet_timewait_sock *tw,
|
|
|
|
|
struct inet_timewait_death_row *twdr);
|
2008-03-25 17:26:21 +00:00
|
|
|
|
2013-09-21 17:22:41 +00:00
|
|
|
void inet_twsk_purge(struct inet_hashinfo *hashinfo,
|
|
|
|
|
struct inet_timewait_death_row *twdr, int family);
|
netns : fix kernel panic in timewait socket destruction
How to reproduce ?
- create a network namespace
- use tcp protocol and get timewait socket
- exit the network namespace
- after a moment (when the timewait socket is destroyed), the kernel
panics.
# BUG: unable to handle kernel NULL pointer dereference at
0000000000000007
IP: [<ffffffff821e394d>] inet_twdr_do_twkill_work+0x6e/0xb8
PGD 119985067 PUD 11c5c0067 PMD 0
Oops: 0000 [1] SMP
CPU 1
Modules linked in: ipv6 button battery ac loop dm_mod tg3 libphy ext3 jbd
edd fan thermal processor thermal_sys sg sata_svw libata dock serverworks
sd_mod scsi_mod ide_disk ide_core [last unloaded: freq_table]
Pid: 0, comm: swapper Not tainted 2.6.27-rc2 #3
RIP: 0010:[<ffffffff821e394d>] [<ffffffff821e394d>]
inet_twdr_do_twkill_work+0x6e/0xb8
RSP: 0018:ffff88011ff7fed0 EFLAGS: 00010246
RAX: ffffffffffffffff RBX: ffffffff82339420 RCX: ffff88011ff7ff30
RDX: 0000000000000001 RSI: ffff88011a4d03c0 RDI: ffff88011ac2fc00
RBP: ffffffff823392e0 R08: 0000000000000000 R09: ffff88002802a200
R10: ffff8800a5c4b000 R11: ffffffff823e4080 R12: ffff88011ac2fc00
R13: 0000000000000001 R14: 0000000000000001 R15: 0000000000000000
FS: 0000000041cbd940(0000) GS:ffff8800bff839c0(0000)
knlGS:0000000000000000
CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b
CR2: 0000000000000007 CR3: 00000000bd87c000 CR4: 00000000000006e0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400
Process swapper (pid: 0, threadinfo ffff8800bff9e000, task
ffff88011ff76690)
Stack: ffffffff823392e0 0000000000000100 ffffffff821e3a3a
0000000000000008
0000000000000000 ffffffff821e3a61 ffff8800bff7c000 ffffffff8203c7e7
ffff88011ff7ff10 ffff88011ff7ff10 0000000000000021 ffffffff82351108
Call Trace:
<IRQ> [<ffffffff821e3a3a>] ? inet_twdr_hangman+0x0/0x9e
[<ffffffff821e3a61>] ? inet_twdr_hangman+0x27/0x9e
[<ffffffff8203c7e7>] ? run_timer_softirq+0x12c/0x193
[<ffffffff820390d1>] ? __do_softirq+0x5e/0xcd
[<ffffffff8200d08c>] ? call_softirq+0x1c/0x28
[<ffffffff8200e611>] ? do_softirq+0x2c/0x68
[<ffffffff8201a055>] ? smp_apic_timer_interrupt+0x8e/0xa9
[<ffffffff8200cad6>] ? apic_timer_interrupt+0x66/0x70
<EOI> [<ffffffff82011f4c>] ? default_idle+0x27/0x3b
[<ffffffff8200abbd>] ? cpu_idle+0x5f/0x7d
Code: e8 01 00 00 4c 89 e7 41 ff c5 e8 8d fd ff ff 49 8b 44 24 38 4c 89 e7
65 8b 14 25 24 00 00 00 89 d2 48 8b 80 e8 00 00 00 48 f7 d0 <48> 8b 04 d0
48 ff 40 58 e8 fc fc ff ff 48 89 df e8 c0 5f 04 00
RIP [<ffffffff821e394d>] inet_twdr_do_twkill_work+0x6e/0xb8
RSP <ffff88011ff7fed0>
CR2: 0000000000000007
This patch provides a function to purge all timewait sockets related
to a network namespace. The timewait sockets life cycle is not tied with
the network namespace, that means the timewait sockets stay alive while
the network namespace dies. The timewait sockets are for avoiding to
receive a duplicate packet from the network, if the network namespace is
freed, the network stack is removed, so no chance to receive any packets
from the outside world. Furthermore, having a pending destruction timer
on these sockets with a network namespace freed is not safe and will lead
to an oops if the timer callback which try to access data belonging to
the namespace like for example in:
inet_twdr_do_twkill_work
-> NET_INC_STATS_BH(twsk_net(tw), LINUX_MIB_TIMEWAITED);
Purging the timewait sockets at the network namespace destruction will:
1) speed up memory freeing for the namespace
2) fix kernel panic on asynchronous timewait destruction
Signed-off-by: Daniel Lezcano <dlezcano@fr.ibm.com>
Acked-by: Denis V. Lunev <den@openvz.org>
Acked-by: Eric W. Biederman <ebiederm@xmission.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2008-09-08 20:17:27 +00:00
|
|
|
|
2008-03-25 17:26:21 +00:00
|
|
|
static inline
|
|
|
|
|
struct net *twsk_net(const struct inet_timewait_sock *twsk)
|
|
|
|
|
{
|
2011-12-14 04:58:33 +00:00
|
|
|
return read_pnet(&twsk->tw_net);
|
2008-03-25 17:26:21 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline
|
2008-03-26 07:48:17 +00:00
|
|
|
void twsk_net_set(struct inet_timewait_sock *twsk, struct net *net)
|
2008-03-25 17:26:21 +00:00
|
|
|
{
|
2011-12-14 04:58:33 +00:00
|
|
|
write_pnet(&twsk->tw_net, net);
|
2008-03-25 17:26:21 +00:00
|
|
|
}
|
2005-08-10 03:09:30 +00:00
|
|
|
#endif /* _INET_TIMEWAIT_SOCK_ */
|