mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-09-30 14:19:16 +00:00
65466904b0
Back when tcp_tso_autosize() and TCP pacing were introduced,
our focus was really to reduce burst sizes for long distance
flows.
The simple heuristic of using sk_pacing_rate/1024 has worked
well, but can lead to too small packets for hosts in the same
rack/cluster, when thousands of flows compete for the bottleneck.
Neal Cardwell had the idea of making the TSO burst size
a function of both sk_pacing_rate and tcp_min_rtt()
Indeed, for local flows, sending bigger bursts is better
to reduce cpu costs, as occasional losses can be repaired
quite fast.
This patch is based on Neal Cardwell implementation
done more than two years ago.
bbr is adjusting max_pacing_rate based on measured bandwidth,
while cubic would over estimate max_pacing_rate.
/proc/sys/net/ipv4/tcp_tso_rtt_log can be used to tune or disable
this new feature, in logarithmic steps.
Tested:
100Gbit NIC, two hosts in the same rack, 4K MTU.
600 flows rate-limited to 20000000 bytes per second.
Before patch: (TSO sizes would be limited to 20000000/1024/4096 -> 4 segments per TSO)
~# echo 0 >/proc/sys/net/ipv4/tcp_tso_rtt_log
~# nstat -n;perf stat ./super_netperf 600 -H otrv6 -l 20 -- -K dctcp -q 20000000;nstat|egrep "TcpInSegs|TcpOutSegs|TcpRetransSegs|Delivered"
96005
Performance counter stats for './super_netperf 600 -H otrv6 -l 20 -- -K dctcp -q 20000000':
65,945.29 msec task-clock # 2.845 CPUs utilized
1,314,632 context-switches # 19935.279 M/sec
5,292 cpu-migrations # 80.249 M/sec
940,641 page-faults # 14264.023 M/sec
201,117,030,926 cycles # 3049769.216 GHz (83.45%)
17,699,435,405 stalled-cycles-frontend # 8.80% frontend cycles idle (83.48%)
136,584,015,071 stalled-cycles-backend # 67.91% backend cycles idle (83.44%)
53,809,530,436 instructions # 0.27 insn per cycle
# 2.54 stalled cycles per insn (83.36%)
9,062,315,523 branches # 137422329.563 M/sec (83.22%)
153,008,621 branch-misses # 1.69% of all branches (83.32%)
23.182970846 seconds time elapsed
TcpInSegs 15648792 0.0
TcpOutSegs 58659110 0.0 # Average of 3.7 4K segments per TSO packet
TcpExtTCPDelivered 58654791 0.0
TcpExtTCPDeliveredCE 19 0.0
After patch:
~# echo 9 >/proc/sys/net/ipv4/tcp_tso_rtt_log
~# nstat -n;perf stat ./super_netperf 600 -H otrv6 -l 20 -- -K dctcp -q 20000000;nstat|egrep "TcpInSegs|TcpOutSegs|TcpRetransSegs|Delivered"
96046
Performance counter stats for './super_netperf 600 -H otrv6 -l 20 -- -K dctcp -q 20000000':
48,982.58 msec task-clock # 2.104 CPUs utilized
186,014 context-switches # 3797.599 M/sec
3,109 cpu-migrations # 63.472 M/sec
941,180 page-faults # 19214.814 M/sec
153,459,763,868 cycles # 3132982.807 GHz (83.56%)
12,069,861,356 stalled-cycles-frontend # 7.87% frontend cycles idle (83.32%)
120,485,917,953 stalled-cycles-backend # 78.51% backend cycles idle (83.24%)
36,803,672,106 instructions # 0.24 insn per cycle
# 3.27 stalled cycles per insn (83.18%)
5,947,266,275 branches # 121417383.427 M/sec (83.64%)
87,984,616 branch-misses # 1.48% of all branches (83.43%)
23.281200256 seconds time elapsed
TcpInSegs 1434706 0.0
TcpOutSegs 58883378 0.0 # Average of 41 4K segments per TSO packet
TcpExtTCPDelivered 58878971 0.0
TcpExtTCPDeliveredCE 9664 0.0
Signed-off-by: Eric Dumazet <edumazet@google.com>
Reviewed-by: Neal Cardwell <ncardwell@google.com>
Link: https://lore.kernel.org/r/20220309015757.2532973-1-eric.dumazet@gmail.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
228 lines
5.6 KiB
C
228 lines
5.6 KiB
C
/* SPDX-License-Identifier: GPL-2.0 */
|
|
/*
|
|
* ipv4 in net namespaces
|
|
*/
|
|
|
|
#ifndef __NETNS_IPV4_H__
|
|
#define __NETNS_IPV4_H__
|
|
|
|
#include <linux/uidgid.h>
|
|
#include <net/inet_frag.h>
|
|
#include <linux/rcupdate.h>
|
|
#include <linux/siphash.h>
|
|
|
|
struct ctl_table_header;
|
|
struct ipv4_devconf;
|
|
struct fib_rules_ops;
|
|
struct hlist_head;
|
|
struct fib_table;
|
|
struct sock;
|
|
struct local_ports {
|
|
seqlock_t lock;
|
|
int range[2];
|
|
bool warned;
|
|
};
|
|
|
|
struct ping_group_range {
|
|
seqlock_t lock;
|
|
kgid_t range[2];
|
|
};
|
|
|
|
struct inet_hashinfo;
|
|
|
|
struct inet_timewait_death_row {
|
|
refcount_t tw_refcount;
|
|
|
|
struct inet_hashinfo *hashinfo ____cacheline_aligned_in_smp;
|
|
int sysctl_max_tw_buckets;
|
|
};
|
|
|
|
struct tcp_fastopen_context;
|
|
|
|
struct netns_ipv4 {
|
|
struct inet_timewait_death_row *tcp_death_row;
|
|
|
|
#ifdef CONFIG_SYSCTL
|
|
struct ctl_table_header *forw_hdr;
|
|
struct ctl_table_header *frags_hdr;
|
|
struct ctl_table_header *ipv4_hdr;
|
|
struct ctl_table_header *route_hdr;
|
|
struct ctl_table_header *xfrm4_hdr;
|
|
#endif
|
|
struct ipv4_devconf *devconf_all;
|
|
struct ipv4_devconf *devconf_dflt;
|
|
struct ip_ra_chain __rcu *ra_chain;
|
|
struct mutex ra_mutex;
|
|
#ifdef CONFIG_IP_MULTIPLE_TABLES
|
|
struct fib_rules_ops *rules_ops;
|
|
struct fib_table __rcu *fib_main;
|
|
struct fib_table __rcu *fib_default;
|
|
unsigned int fib_rules_require_fldissect;
|
|
bool fib_has_custom_rules;
|
|
#endif
|
|
bool fib_has_custom_local_routes;
|
|
bool fib_offload_disabled;
|
|
#ifdef CONFIG_IP_ROUTE_CLASSID
|
|
atomic_t fib_num_tclassid_users;
|
|
#endif
|
|
struct hlist_head *fib_table_hash;
|
|
struct sock *fibnl;
|
|
|
|
struct sock *mc_autojoin_sk;
|
|
|
|
struct inet_peer_base *peers;
|
|
struct fqdir *fqdir;
|
|
|
|
u8 sysctl_icmp_echo_ignore_all;
|
|
u8 sysctl_icmp_echo_enable_probe;
|
|
u8 sysctl_icmp_echo_ignore_broadcasts;
|
|
u8 sysctl_icmp_ignore_bogus_error_responses;
|
|
u8 sysctl_icmp_errors_use_inbound_ifaddr;
|
|
int sysctl_icmp_ratelimit;
|
|
int sysctl_icmp_ratemask;
|
|
|
|
u32 ip_rt_min_pmtu;
|
|
int ip_rt_mtu_expires;
|
|
int ip_rt_min_advmss;
|
|
|
|
struct local_ports ip_local_ports;
|
|
|
|
u8 sysctl_tcp_ecn;
|
|
u8 sysctl_tcp_ecn_fallback;
|
|
|
|
u8 sysctl_ip_default_ttl;
|
|
u8 sysctl_ip_no_pmtu_disc;
|
|
u8 sysctl_ip_fwd_use_pmtu;
|
|
u8 sysctl_ip_fwd_update_priority;
|
|
u8 sysctl_ip_nonlocal_bind;
|
|
u8 sysctl_ip_autobind_reuse;
|
|
/* Shall we try to damage output packets if routing dev changes? */
|
|
u8 sysctl_ip_dynaddr;
|
|
u8 sysctl_ip_early_demux;
|
|
#ifdef CONFIG_NET_L3_MASTER_DEV
|
|
u8 sysctl_raw_l3mdev_accept;
|
|
#endif
|
|
u8 sysctl_tcp_early_demux;
|
|
u8 sysctl_udp_early_demux;
|
|
|
|
u8 sysctl_nexthop_compat_mode;
|
|
|
|
u8 sysctl_fwmark_reflect;
|
|
u8 sysctl_tcp_fwmark_accept;
|
|
#ifdef CONFIG_NET_L3_MASTER_DEV
|
|
u8 sysctl_tcp_l3mdev_accept;
|
|
#endif
|
|
u8 sysctl_tcp_mtu_probing;
|
|
int sysctl_tcp_mtu_probe_floor;
|
|
int sysctl_tcp_base_mss;
|
|
int sysctl_tcp_min_snd_mss;
|
|
int sysctl_tcp_probe_threshold;
|
|
u32 sysctl_tcp_probe_interval;
|
|
|
|
int sysctl_tcp_keepalive_time;
|
|
int sysctl_tcp_keepalive_intvl;
|
|
u8 sysctl_tcp_keepalive_probes;
|
|
|
|
u8 sysctl_tcp_syn_retries;
|
|
u8 sysctl_tcp_synack_retries;
|
|
u8 sysctl_tcp_syncookies;
|
|
u8 sysctl_tcp_migrate_req;
|
|
u8 sysctl_tcp_comp_sack_nr;
|
|
int sysctl_tcp_reordering;
|
|
u8 sysctl_tcp_retries1;
|
|
u8 sysctl_tcp_retries2;
|
|
u8 sysctl_tcp_orphan_retries;
|
|
u8 sysctl_tcp_tw_reuse;
|
|
int sysctl_tcp_fin_timeout;
|
|
unsigned int sysctl_tcp_notsent_lowat;
|
|
u8 sysctl_tcp_sack;
|
|
u8 sysctl_tcp_window_scaling;
|
|
u8 sysctl_tcp_timestamps;
|
|
u8 sysctl_tcp_early_retrans;
|
|
u8 sysctl_tcp_recovery;
|
|
u8 sysctl_tcp_thin_linear_timeouts;
|
|
u8 sysctl_tcp_slow_start_after_idle;
|
|
u8 sysctl_tcp_retrans_collapse;
|
|
u8 sysctl_tcp_stdurg;
|
|
u8 sysctl_tcp_rfc1337;
|
|
u8 sysctl_tcp_abort_on_overflow;
|
|
u8 sysctl_tcp_fack; /* obsolete */
|
|
int sysctl_tcp_max_reordering;
|
|
int sysctl_tcp_adv_win_scale;
|
|
u8 sysctl_tcp_dsack;
|
|
u8 sysctl_tcp_app_win;
|
|
u8 sysctl_tcp_frto;
|
|
u8 sysctl_tcp_nometrics_save;
|
|
u8 sysctl_tcp_no_ssthresh_metrics_save;
|
|
u8 sysctl_tcp_moderate_rcvbuf;
|
|
u8 sysctl_tcp_tso_win_divisor;
|
|
u8 sysctl_tcp_workaround_signed_windows;
|
|
int sysctl_tcp_limit_output_bytes;
|
|
int sysctl_tcp_challenge_ack_limit;
|
|
int sysctl_tcp_min_rtt_wlen;
|
|
u8 sysctl_tcp_min_tso_segs;
|
|
u8 sysctl_tcp_tso_rtt_log;
|
|
u8 sysctl_tcp_autocorking;
|
|
u8 sysctl_tcp_reflect_tos;
|
|
int sysctl_tcp_invalid_ratelimit;
|
|
int sysctl_tcp_pacing_ss_ratio;
|
|
int sysctl_tcp_pacing_ca_ratio;
|
|
int sysctl_tcp_wmem[3];
|
|
int sysctl_tcp_rmem[3];
|
|
unsigned long sysctl_tcp_comp_sack_delay_ns;
|
|
unsigned long sysctl_tcp_comp_sack_slack_ns;
|
|
int sysctl_max_syn_backlog;
|
|
int sysctl_tcp_fastopen;
|
|
const struct tcp_congestion_ops __rcu *tcp_congestion_control;
|
|
struct tcp_fastopen_context __rcu *tcp_fastopen_ctx;
|
|
unsigned int sysctl_tcp_fastopen_blackhole_timeout;
|
|
atomic_t tfo_active_disable_times;
|
|
unsigned long tfo_active_disable_stamp;
|
|
|
|
int sysctl_udp_wmem_min;
|
|
int sysctl_udp_rmem_min;
|
|
|
|
u8 sysctl_fib_notify_on_flag_change;
|
|
|
|
#ifdef CONFIG_NET_L3_MASTER_DEV
|
|
u8 sysctl_udp_l3mdev_accept;
|
|
#endif
|
|
|
|
u8 sysctl_igmp_llm_reports;
|
|
int sysctl_igmp_max_memberships;
|
|
int sysctl_igmp_max_msf;
|
|
int sysctl_igmp_qrv;
|
|
|
|
struct ping_group_range ping_group_range;
|
|
|
|
atomic_t dev_addr_genid;
|
|
|
|
#ifdef CONFIG_SYSCTL
|
|
unsigned long *sysctl_local_reserved_ports;
|
|
int sysctl_ip_prot_sock;
|
|
#endif
|
|
|
|
#ifdef CONFIG_IP_MROUTE
|
|
#ifndef CONFIG_IP_MROUTE_MULTIPLE_TABLES
|
|
struct mr_table *mrt;
|
|
#else
|
|
struct list_head mr_tables;
|
|
struct fib_rules_ops *mr_rules_ops;
|
|
#endif
|
|
#endif
|
|
#ifdef CONFIG_IP_ROUTE_MULTIPATH
|
|
u32 sysctl_fib_multipath_hash_fields;
|
|
u8 sysctl_fib_multipath_use_neigh;
|
|
u8 sysctl_fib_multipath_hash_policy;
|
|
#endif
|
|
|
|
struct fib_notifier_ops *notifier_ops;
|
|
unsigned int fib_seq; /* protected by rtnl_mutex */
|
|
|
|
struct fib_notifier_ops *ipmr_notifier_ops;
|
|
unsigned int ipmr_seq; /* protected by rtnl_mutex */
|
|
|
|
atomic_t rt_genid;
|
|
siphash_key_t ip_id_key;
|
|
};
|
|
#endif
|