2019-05-27 06:55:01 +00:00
|
|
|
// SPDX-License-Identifier: GPL-2.0-or-later
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
* INET An implementation of the TCP/IP protocol suite for the LINUX
|
|
|
|
* operating system. INET is implemented using the BSD Socket
|
|
|
|
* interface as the means of communication with the user level.
|
|
|
|
*
|
|
|
|
* The User Datagram Protocol (UDP).
|
|
|
|
*
|
2005-05-05 23:16:16 +00:00
|
|
|
* Authors: Ross Biro
|
2005-04-16 22:20:36 +00:00
|
|
|
* Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
|
|
|
|
* Arnt Gulbrandsen, <agulbra@nvg.unit.no>
|
2008-10-14 02:01:08 +00:00
|
|
|
* Alan Cox, <alan@lxorguk.ukuu.org.uk>
|
2005-04-16 22:20:36 +00:00
|
|
|
* Hirokazu Takahashi, <taka@valinux.co.jp>
|
|
|
|
*
|
|
|
|
* Fixes:
|
|
|
|
* Alan Cox : verify_area() calls
|
|
|
|
* Alan Cox : stopped close while in use off icmp
|
|
|
|
* messages. Not a fix but a botch that
|
|
|
|
* for udp at least is 'valid'.
|
|
|
|
* Alan Cox : Fixed icmp handling properly
|
|
|
|
* Alan Cox : Correct error for oversized datagrams
|
2007-02-09 14:24:47 +00:00
|
|
|
* Alan Cox : Tidied select() semantics.
|
|
|
|
* Alan Cox : udp_err() fixed properly, also now
|
2005-04-16 22:20:36 +00:00
|
|
|
* select and read wake correctly on errors
|
|
|
|
* Alan Cox : udp_send verify_area moved to avoid mem leak
|
|
|
|
* Alan Cox : UDP can count its memory
|
|
|
|
* Alan Cox : send to an unknown connection causes
|
|
|
|
* an ECONNREFUSED off the icmp, but
|
|
|
|
* does NOT close.
|
|
|
|
* Alan Cox : Switched to new sk_buff handlers. No more backlog!
|
|
|
|
* Alan Cox : Using generic datagram code. Even smaller and the PEEK
|
|
|
|
* bug no longer crashes it.
|
|
|
|
* Fred Van Kempen : Net2e support for sk->broadcast.
|
|
|
|
* Alan Cox : Uses skb_free_datagram
|
|
|
|
* Alan Cox : Added get/set sockopt support.
|
|
|
|
* Alan Cox : Broadcasting without option set returns EACCES.
|
|
|
|
* Alan Cox : No wakeup calls. Instead we now use the callbacks.
|
|
|
|
* Alan Cox : Use ip_tos and ip_ttl
|
|
|
|
* Alan Cox : SNMP Mibs
|
|
|
|
* Alan Cox : MSG_DONTROUTE, and 0.0.0.0 support.
|
|
|
|
* Matt Dillon : UDP length checks.
|
|
|
|
* Alan Cox : Smarter af_inet used properly.
|
|
|
|
* Alan Cox : Use new kernel side addressing.
|
|
|
|
* Alan Cox : Incorrect return on truncated datagram receive.
|
|
|
|
* Arnt Gulbrandsen : New udp_send and stuff
|
|
|
|
* Alan Cox : Cache last socket
|
|
|
|
* Alan Cox : Route cache
|
|
|
|
* Jon Peatfield : Minor efficiency fix to sendto().
|
|
|
|
* Mike Shaver : RFC1122 checks.
|
|
|
|
* Alan Cox : Nonblocking error fix.
|
|
|
|
* Willy Konynenberg : Transparent proxying support.
|
|
|
|
* Mike McLagan : Routing by source
|
|
|
|
* David S. Miller : New socket lookup architecture.
|
|
|
|
* Last socket cache retained as it
|
|
|
|
* does have a high hit rate.
|
|
|
|
* Olaf Kirch : Don't linearise iovec on sendmsg.
|
|
|
|
* Andi Kleen : Some cleanups, cache destination entry
|
2007-02-09 14:24:47 +00:00
|
|
|
* for connect.
|
2005-04-16 22:20:36 +00:00
|
|
|
* Vitaly E. Lavrov : Transparent proxy revived after year coma.
|
|
|
|
* Melvin Smith : Check msg_name not msg_namelen in sendto(),
|
|
|
|
* return ENOTCONN for unconnected sockets (POSIX)
|
|
|
|
* Janos Farkas : don't deliver multi/broadcasts to a different
|
|
|
|
* bound-to-device socket
|
|
|
|
* Hirokazu Takahashi : HW checksumming for outgoing UDP
|
|
|
|
* datagrams.
|
|
|
|
* Hirokazu Takahashi : sendfile() on UDP works now.
|
|
|
|
* Arnaldo C. Melo : convert /proc/net/udp to seq_file
|
|
|
|
* YOSHIFUJI Hideaki @USAGI and: Support IPV6_V6ONLY socket option, which
|
|
|
|
* Alexey Kuznetsov: allow both IPv4 and IPv6 sockets to bind
|
|
|
|
* a single port at the same time.
|
|
|
|
* Derek Atkins <derek@ihtfp.com>: Add Encapulation Support
|
2007-06-27 22:37:46 +00:00
|
|
|
* James Chapman : Add L2TP encapsulation type.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
2007-02-09 14:24:47 +00:00
|
|
|
|
2012-03-12 07:03:32 +00:00
|
|
|
#define pr_fmt(fmt) "UDP: " fmt
|
|
|
|
|
2021-12-16 02:55:37 +00:00
|
|
|
#include <linux/bpf-cgroup.h>
|
2016-12-24 19:46:01 +00:00
|
|
|
#include <linux/uaccess.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
#include <asm/ioctls.h>
|
2018-10-30 22:09:49 +00:00
|
|
|
#include <linux/memblock.h>
|
2008-10-29 09:32:32 +00:00
|
|
|
#include <linux/highmem.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
#include <linux/types.h>
|
|
|
|
#include <linux/fcntl.h>
|
|
|
|
#include <linux/module.h>
|
|
|
|
#include <linux/socket.h>
|
|
|
|
#include <linux/sockios.h>
|
2005-12-27 04:43:12 +00:00
|
|
|
#include <linux/igmp.h>
|
2015-06-03 21:27:38 +00:00
|
|
|
#include <linux/inetdevice.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
#include <linux/in.h>
|
|
|
|
#include <linux/errno.h>
|
|
|
|
#include <linux/timer.h>
|
|
|
|
#include <linux/mm.h>
|
|
|
|
#include <linux/inet.h>
|
|
|
|
#include <linux/netdevice.h>
|
include cleanup: Update gfp.h and slab.h includes to prepare for breaking implicit slab.h inclusion from percpu.h
percpu.h is included by sched.h and module.h and thus ends up being
included when building most .c files. percpu.h includes slab.h which
in turn includes gfp.h making everything defined by the two files
universally available and complicating inclusion dependencies.
percpu.h -> slab.h dependency is about to be removed. Prepare for
this change by updating users of gfp and slab facilities include those
headers directly instead of assuming availability. As this conversion
needs to touch large number of source files, the following script is
used as the basis of conversion.
http://userweb.kernel.org/~tj/misc/slabh-sweep.py
The script does the followings.
* Scan files for gfp and slab usages and update includes such that
only the necessary includes are there. ie. if only gfp is used,
gfp.h, if slab is used, slab.h.
* When the script inserts a new include, it looks at the include
blocks and try to put the new include such that its order conforms
to its surrounding. It's put in the include block which contains
core kernel includes, in the same order that the rest are ordered -
alphabetical, Christmas tree, rev-Xmas-tree or at the end if there
doesn't seem to be any matching order.
* If the script can't find a place to put a new include (mostly
because the file doesn't have fitting include block), it prints out
an error message indicating which .h file needs to be added to the
file.
The conversion was done in the following steps.
1. The initial automatic conversion of all .c files updated slightly
over 4000 files, deleting around 700 includes and adding ~480 gfp.h
and ~3000 slab.h inclusions. The script emitted errors for ~400
files.
2. Each error was manually checked. Some didn't need the inclusion,
some needed manual addition while adding it to implementation .h or
embedding .c file was more appropriate for others. This step added
inclusions to around 150 files.
3. The script was run again and the output was compared to the edits
from #2 to make sure no file was left behind.
4. Several build tests were done and a couple of problems were fixed.
e.g. lib/decompress_*.c used malloc/free() wrappers around slab
APIs requiring slab.h to be added manually.
5. The script was run on all .h files but without automatically
editing them as sprinkling gfp.h and slab.h inclusions around .h
files could easily lead to inclusion dependency hell. Most gfp.h
inclusion directives were ignored as stuff from gfp.h was usually
wildly available and often used in preprocessor macros. Each
slab.h inclusion directive was examined and added manually as
necessary.
6. percpu.h was updated not to include slab.h.
7. Build test were done on the following configurations and failures
were fixed. CONFIG_GCOV_KERNEL was turned off for all tests (as my
distributed build env didn't work with gcov compiles) and a few
more options had to be turned off depending on archs to make things
build (like ipr on powerpc/64 which failed due to missing writeq).
* x86 and x86_64 UP and SMP allmodconfig and a custom test config.
* powerpc and powerpc64 SMP allmodconfig
* sparc and sparc64 SMP allmodconfig
* ia64 SMP allmodconfig
* s390 SMP allmodconfig
* alpha SMP allmodconfig
* um on x86_64 SMP allmodconfig
8. percpu.h modifications were reverted so that it could be applied as
a separate patch and serve as bisection point.
Given the fact that I had only a couple of failures from tests on step
6, I'm fairly confident about the coverage of this conversion patch.
If there is a breakage, it's likely to be something in one of the arch
headers which should be easily discoverable easily on most builds of
the specific arch.
Signed-off-by: Tejun Heo <tj@kernel.org>
Guess-its-ok-by: Christoph Lameter <cl@linux-foundation.org>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
2010-03-24 08:04:11 +00:00
|
|
|
#include <linux/slab.h>
|
2005-08-10 03:08:28 +00:00
|
|
|
#include <net/tcp_states.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
#include <linux/skbuff.h>
|
|
|
|
#include <linux/proc_fs.h>
|
|
|
|
#include <linux/seq_file.h>
|
2007-09-12 10:01:34 +00:00
|
|
|
#include <net/net_namespace.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
#include <net/icmp.h>
|
2013-10-07 16:01:39 +00:00
|
|
|
#include <net/inet_hashtables.h>
|
udp: Support for error handlers of tunnels with arbitrary destination port
ICMP error handling is currently not possible for UDP tunnels not
employing a receiving socket with local destination port matching the
remote one, because we have no way to look them up.
Add an err_handler tunnel encapsulation operation that can be exported by
tunnels in order to pass the error to the protocol implementing the
encapsulation. We can't easily use a lookup function as we did for VXLAN
and GENEVE, as protocol error handlers, which would be in turn called by
implementations of this new operation, handle the errors themselves,
together with the tunnel lookup.
Without a socket, we can't be sure which encapsulation error handler is
the appropriate one: encapsulation handlers (the ones for FoU and GUE
introduced in the next patch, e.g.) will need to check the new error codes
returned by protocol handlers to figure out if errors match the given
encapsulation, and, in turn, report this error back, so that we can try
all of them in __udp{4,6}_lib_err_encap_no_sk() until we have a match.
v2:
- Name all arguments in err_handler prototypes (David Miller)
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
Reviewed-by: Sabrina Dubroca <sd@queasysnail.net>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-11-08 11:19:22 +00:00
|
|
|
#include <net/ip_tunnels.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
#include <net/route.h>
|
|
|
|
#include <net/checksum.h>
|
2023-06-08 19:17:37 +00:00
|
|
|
#include <net/gso.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
#include <net/xfrm.h>
|
2011-06-17 11:58:39 +00:00
|
|
|
#include <trace/events/udp.h>
|
2012-04-11 23:05:28 +00:00
|
|
|
#include <linux/static_key.h>
|
2020-07-20 16:34:03 +00:00
|
|
|
#include <linux/btf_ids.h>
|
2012-06-27 00:23:44 +00:00
|
|
|
#include <trace/events/skb.h>
|
2013-07-10 14:13:17 +00:00
|
|
|
#include <net/busy_poll.h>
|
2006-11-27 19:10:57 +00:00
|
|
|
#include "udp_impl.h"
|
2016-01-04 22:41:46 +00:00
|
|
|
#include <net/sock_reuseport.h>
|
2016-08-18 16:59:12 +00:00
|
|
|
#include <net/addrconf.h>
|
2018-11-07 11:38:28 +00:00
|
|
|
#include <net/udp_tunnel.h>
|
2023-07-27 15:33:56 +00:00
|
|
|
#include <net/gro.h>
|
2020-04-27 15:59:34 +00:00
|
|
|
#if IS_ENABLED(CONFIG_IPV6)
|
|
|
|
#include <net/ipv6_stubs.h>
|
|
|
|
#endif
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2009-10-07 00:37:59 +00:00
|
|
|
struct udp_table udp_table __read_mostly;
|
2008-10-29 08:41:45 +00:00
|
|
|
EXPORT_SYMBOL(udp_table);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2010-11-09 23:24:26 +00:00
|
|
|
long sysctl_udp_mem[3] __read_mostly;
|
2007-12-31 08:29:24 +00:00
|
|
|
EXPORT_SYMBOL(sysctl_udp_mem);
|
2009-07-17 00:26:32 +00:00
|
|
|
|
2021-11-15 19:02:39 +00:00
|
|
|
atomic_long_t udp_memory_allocated ____cacheline_aligned_in_smp;
|
2007-12-31 08:29:24 +00:00
|
|
|
EXPORT_SYMBOL(udp_memory_allocated);
|
2022-06-09 06:34:08 +00:00
|
|
|
DEFINE_PER_CPU(int, udp_memory_per_cpu_fw_alloc);
|
|
|
|
EXPORT_PER_CPU_SYMBOL_GPL(udp_memory_per_cpu_fw_alloc);
|
2007-12-31 08:29:24 +00:00
|
|
|
|
2009-10-07 00:37:59 +00:00
|
|
|
#define MAX_UDP_PORTS 65536
|
udp: Introduce optional per-netns hash table.
The maximum hash table size is 64K due to the nature of the protocol. [0]
It's smaller than TCP, and fewer sockets can cause a performance drop.
On an EC2 c5.24xlarge instance (192 GiB memory), after running iperf3 in
different netns, creating 32Mi sockets without data transfer in the root
netns causes regression for the iperf3's connection.
uhash_entries sockets length Gbps
64K 1 1 5.69
1Mi 16 5.27
2Mi 32 4.90
4Mi 64 4.09
8Mi 128 2.96
16Mi 256 2.06
32Mi 512 1.12
The per-netns hash table breaks the lengthy lists into shorter ones. It is
useful on a multi-tenant system with thousands of netns. With smaller hash
tables, we can look up sockets faster, isolate noisy neighbours, and reduce
lock contention.
The max size of the per-netns table is 64K as well. This is because the
possible hash range by udp_hashfn() always fits in 64K within the same
netns and we cannot make full use of the whole buckets larger than 64K.
/* 0 < num < 64K -> X < hash < X + 64K */
(num + net_hash_mix(net)) & mask;
Also, the min size is 128. We use a bitmap to search for an available
port in udp_lib_get_port(). To keep the bitmap on the stack and not
fire the CONFIG_FRAME_WARN error at build time, we round up the table
size to 128.
The sysctl usage is the same with TCP:
$ dmesg | cut -d ' ' -f 6- | grep "UDP hash"
UDP hash table entries: 65536 (order: 9, 2097152 bytes, vmalloc)
# sysctl net.ipv4.udp_hash_entries
net.ipv4.udp_hash_entries = 65536 # can be changed by uhash_entries
# sysctl net.ipv4.udp_child_hash_entries
net.ipv4.udp_child_hash_entries = 0 # disabled by default
# ip netns add test1
# ip netns exec test1 sysctl net.ipv4.udp_hash_entries
net.ipv4.udp_hash_entries = -65536 # share the global table
# sysctl -w net.ipv4.udp_child_hash_entries=100
net.ipv4.udp_child_hash_entries = 100
# ip netns add test2
# ip netns exec test2 sysctl net.ipv4.udp_hash_entries
net.ipv4.udp_hash_entries = 128 # own a per-netns table with 2^n buckets
We could optimise the hash table lookup/iteration further by removing
the netns comparison for the per-netns one in the future. Also, we
could optimise the sparse udp_hslot layout by putting it in udp_table.
[0]: https://lore.kernel.org/netdev/4ACC2815.7010101@gmail.com/
Signed-off-by: Kuniyuki Iwashima <kuniyu@amazon.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2022-11-14 21:57:57 +00:00
|
|
|
#define PORTS_PER_CHAIN (MAX_UDP_PORTS / UDP_HTABLE_SIZE_MIN_PERNET)
|
2009-01-27 05:35:35 +00:00
|
|
|
|
2022-11-14 21:57:54 +00:00
|
|
|
static struct udp_table *udp_get_table_prot(struct sock *sk)
|
|
|
|
{
|
|
|
|
return sk->sk_prot->h.udp_table ? : sock_net(sk)->ipv4.udp_table;
|
|
|
|
}
|
|
|
|
|
2008-10-09 21:51:27 +00:00
|
|
|
static int udp_lib_lport_inuse(struct net *net, __u16 num,
|
2008-10-29 08:41:45 +00:00
|
|
|
const struct udp_hslot *hslot,
|
2009-01-27 05:35:35 +00:00
|
|
|
unsigned long *bitmap,
|
2017-01-17 15:51:01 +00:00
|
|
|
struct sock *sk, unsigned int log)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2008-10-09 21:51:27 +00:00
|
|
|
struct sock *sk2;
|
2013-01-22 09:50:32 +00:00
|
|
|
kuid_t uid = sock_i_uid(sk);
|
2006-08-27 03:06:05 +00:00
|
|
|
|
2016-04-01 15:52:13 +00:00
|
|
|
sk_for_each(sk2, &hslot->head) {
|
2009-11-23 18:41:23 +00:00
|
|
|
if (net_eq(sock_net(sk2), net) &&
|
|
|
|
sk2 != sk &&
|
2009-11-08 10:17:30 +00:00
|
|
|
(bitmap || udp_sk(sk2)->udp_port_hash == num) &&
|
2009-11-23 18:41:23 +00:00
|
|
|
(!sk2->sk_reuse || !sk->sk_reuse) &&
|
|
|
|
(!sk2->sk_bound_dev_if || !sk->sk_bound_dev_if ||
|
|
|
|
sk2->sk_bound_dev_if == sk->sk_bound_dev_if) &&
|
2017-01-17 15:51:01 +00:00
|
|
|
inet_rcv_saddr_equal(sk, sk2, true)) {
|
2017-01-06 01:22:36 +00:00
|
|
|
if (sk2->sk_reuseport && sk->sk_reuseport &&
|
|
|
|
!rcu_access_pointer(sk->sk_reuseport_cb) &&
|
|
|
|
uid_eq(uid, sock_i_uid(sk2))) {
|
|
|
|
if (!bitmap)
|
|
|
|
return 0;
|
|
|
|
} else {
|
|
|
|
if (!bitmap)
|
|
|
|
return 1;
|
|
|
|
__set_bit(udp_sk(sk2)->udp_port_hash >> log,
|
|
|
|
bitmap);
|
|
|
|
}
|
2009-01-27 05:35:35 +00:00
|
|
|
}
|
2014-11-12 05:59:20 +00:00
|
|
|
}
|
2006-08-27 03:06:05 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2009-11-09 05:26:33 +00:00
|
|
|
/*
|
|
|
|
* Note: we still hold spinlock of primary hash chain, so no other writer
|
|
|
|
* can insert/delete a socket with local_port == num
|
|
|
|
*/
|
|
|
|
static int udp_lib_lport_inuse2(struct net *net, __u16 num,
|
2014-11-12 05:59:20 +00:00
|
|
|
struct udp_hslot *hslot2,
|
2017-01-17 15:51:01 +00:00
|
|
|
struct sock *sk)
|
2009-11-09 05:26:33 +00:00
|
|
|
{
|
|
|
|
struct sock *sk2;
|
2013-01-22 09:50:32 +00:00
|
|
|
kuid_t uid = sock_i_uid(sk);
|
2009-11-09 05:26:33 +00:00
|
|
|
int res = 0;
|
|
|
|
|
|
|
|
spin_lock(&hslot2->lock);
|
2016-04-01 15:52:13 +00:00
|
|
|
udp_portaddr_for_each_entry(sk2, &hslot2->head) {
|
2009-11-23 18:41:23 +00:00
|
|
|
if (net_eq(sock_net(sk2), net) &&
|
|
|
|
sk2 != sk &&
|
|
|
|
(udp_sk(sk2)->udp_port_hash == num) &&
|
|
|
|
(!sk2->sk_reuse || !sk->sk_reuse) &&
|
|
|
|
(!sk2->sk_bound_dev_if || !sk->sk_bound_dev_if ||
|
|
|
|
sk2->sk_bound_dev_if == sk->sk_bound_dev_if) &&
|
2017-01-17 15:51:01 +00:00
|
|
|
inet_rcv_saddr_equal(sk, sk2, true)) {
|
2017-01-06 01:22:36 +00:00
|
|
|
if (sk2->sk_reuseport && sk->sk_reuseport &&
|
|
|
|
!rcu_access_pointer(sk->sk_reuseport_cb) &&
|
|
|
|
uid_eq(uid, sock_i_uid(sk2))) {
|
|
|
|
res = 0;
|
|
|
|
} else {
|
|
|
|
res = 1;
|
|
|
|
}
|
2009-11-09 05:26:33 +00:00
|
|
|
break;
|
|
|
|
}
|
2014-11-12 05:59:20 +00:00
|
|
|
}
|
2009-11-09 05:26:33 +00:00
|
|
|
spin_unlock(&hslot2->lock);
|
|
|
|
return res;
|
|
|
|
}
|
|
|
|
|
2017-01-17 15:51:01 +00:00
|
|
|
static int udp_reuseport_add_sock(struct sock *sk, struct udp_hslot *hslot)
|
2016-01-04 22:41:46 +00:00
|
|
|
{
|
|
|
|
struct net *net = sock_net(sk);
|
|
|
|
kuid_t uid = sock_i_uid(sk);
|
|
|
|
struct sock *sk2;
|
|
|
|
|
2016-04-01 15:52:13 +00:00
|
|
|
sk_for_each(sk2, &hslot->head) {
|
2016-01-04 22:41:46 +00:00
|
|
|
if (net_eq(sock_net(sk2), net) &&
|
|
|
|
sk2 != sk &&
|
|
|
|
sk2->sk_family == sk->sk_family &&
|
|
|
|
ipv6_only_sock(sk2) == ipv6_only_sock(sk) &&
|
|
|
|
(udp_sk(sk2)->udp_port_hash == udp_sk(sk)->udp_port_hash) &&
|
|
|
|
(sk2->sk_bound_dev_if == sk->sk_bound_dev_if) &&
|
|
|
|
sk2->sk_reuseport && uid_eq(uid, sock_i_uid(sk2)) &&
|
2017-01-17 15:51:01 +00:00
|
|
|
inet_rcv_saddr_equal(sk, sk2, false)) {
|
bpf: Introduce BPF_PROG_TYPE_SK_REUSEPORT
This patch adds a BPF_PROG_TYPE_SK_REUSEPORT which can select
a SO_REUSEPORT sk from a BPF_MAP_TYPE_REUSEPORT_ARRAY. Like other
non SK_FILTER/CGROUP_SKB program, it requires CAP_SYS_ADMIN.
BPF_PROG_TYPE_SK_REUSEPORT introduces "struct sk_reuseport_kern"
to store the bpf context instead of using the skb->cb[48].
At the SO_REUSEPORT sk lookup time, it is in the middle of transiting
from a lower layer (ipv4/ipv6) to a upper layer (udp/tcp). At this
point, it is not always clear where the bpf context can be appended
in the skb->cb[48] to avoid saving-and-restoring cb[]. Even putting
aside the difference between ipv4-vs-ipv6 and udp-vs-tcp. It is not
clear if the lower layer is only ipv4 and ipv6 in the future and
will it not touch the cb[] again before transiting to the upper
layer.
For example, in udp_gro_receive(), it uses the 48 byte NAPI_GRO_CB
instead of IP[6]CB and it may still modify the cb[] after calling
the udp[46]_lib_lookup_skb(). Because of the above reason, if
sk->cb is used for the bpf ctx, saving-and-restoring is needed
and likely the whole 48 bytes cb[] has to be saved and restored.
Instead of saving, setting and restoring the cb[], this patch opts
to create a new "struct sk_reuseport_kern" and setting the needed
values in there.
The new BPF_PROG_TYPE_SK_REUSEPORT and "struct sk_reuseport_(kern|md)"
will serve all ipv4/ipv6 + udp/tcp combinations. There is no protocol
specific usage at this point and it is also inline with the current
sock_reuseport.c implementation (i.e. no protocol specific requirement).
In "struct sk_reuseport_md", this patch exposes data/data_end/len
with semantic similar to other existing usages. Together
with "bpf_skb_load_bytes()" and "bpf_skb_load_bytes_relative()",
the bpf prog can peek anywhere in the skb. The "bind_inany" tells
the bpf prog that the reuseport group is bind-ed to a local
INANY address which cannot be learned from skb.
The new "bind_inany" is added to "struct sock_reuseport" which will be
used when running the new "BPF_PROG_TYPE_SK_REUSEPORT" bpf prog in order
to avoid repeating the "bind INANY" test on
"sk_v6_rcv_saddr/sk->sk_rcv_saddr" every time a bpf prog is run. It can
only be properly initialized when a "sk->sk_reuseport" enabled sk is
adding to a hashtable (i.e. during "reuseport_alloc()" and
"reuseport_add_sock()").
The new "sk_select_reuseport()" is the main helper that the
bpf prog will use to select a SO_REUSEPORT sk. It is the only function
that can use the new BPF_MAP_TYPE_REUSEPORT_ARRAY. As mentioned in
the earlier patch, the validity of a selected sk is checked in
run time in "sk_select_reuseport()". Doing the check in
verification time is difficult and inflexible (consider the map-in-map
use case). The runtime check is to compare the selected sk's reuseport_id
with the reuseport_id that we want. This helper will return -EXXX if the
selected sk cannot serve the incoming request (e.g. reuseport_id
not match). The bpf prog can decide if it wants to do SK_DROP as its
discretion.
When the bpf prog returns SK_PASS, the kernel will check if a
valid sk has been selected (i.e. "reuse_kern->selected_sk != NULL").
If it does , it will use the selected sk. If not, the kernel
will select one from "reuse->socks[]" (as before this patch).
The SK_DROP and SK_PASS handling logic will be in the next patch.
Signed-off-by: Martin KaFai Lau <kafai@fb.com>
Acked-by: Alexei Starovoitov <ast@kernel.org>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-08-08 08:01:25 +00:00
|
|
|
return reuseport_add_sock(sk, sk2,
|
|
|
|
inet_rcv_saddr_any(sk));
|
2016-01-04 22:41:46 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
bpf: Introduce BPF_PROG_TYPE_SK_REUSEPORT
This patch adds a BPF_PROG_TYPE_SK_REUSEPORT which can select
a SO_REUSEPORT sk from a BPF_MAP_TYPE_REUSEPORT_ARRAY. Like other
non SK_FILTER/CGROUP_SKB program, it requires CAP_SYS_ADMIN.
BPF_PROG_TYPE_SK_REUSEPORT introduces "struct sk_reuseport_kern"
to store the bpf context instead of using the skb->cb[48].
At the SO_REUSEPORT sk lookup time, it is in the middle of transiting
from a lower layer (ipv4/ipv6) to a upper layer (udp/tcp). At this
point, it is not always clear where the bpf context can be appended
in the skb->cb[48] to avoid saving-and-restoring cb[]. Even putting
aside the difference between ipv4-vs-ipv6 and udp-vs-tcp. It is not
clear if the lower layer is only ipv4 and ipv6 in the future and
will it not touch the cb[] again before transiting to the upper
layer.
For example, in udp_gro_receive(), it uses the 48 byte NAPI_GRO_CB
instead of IP[6]CB and it may still modify the cb[] after calling
the udp[46]_lib_lookup_skb(). Because of the above reason, if
sk->cb is used for the bpf ctx, saving-and-restoring is needed
and likely the whole 48 bytes cb[] has to be saved and restored.
Instead of saving, setting and restoring the cb[], this patch opts
to create a new "struct sk_reuseport_kern" and setting the needed
values in there.
The new BPF_PROG_TYPE_SK_REUSEPORT and "struct sk_reuseport_(kern|md)"
will serve all ipv4/ipv6 + udp/tcp combinations. There is no protocol
specific usage at this point and it is also inline with the current
sock_reuseport.c implementation (i.e. no protocol specific requirement).
In "struct sk_reuseport_md", this patch exposes data/data_end/len
with semantic similar to other existing usages. Together
with "bpf_skb_load_bytes()" and "bpf_skb_load_bytes_relative()",
the bpf prog can peek anywhere in the skb. The "bind_inany" tells
the bpf prog that the reuseport group is bind-ed to a local
INANY address which cannot be learned from skb.
The new "bind_inany" is added to "struct sock_reuseport" which will be
used when running the new "BPF_PROG_TYPE_SK_REUSEPORT" bpf prog in order
to avoid repeating the "bind INANY" test on
"sk_v6_rcv_saddr/sk->sk_rcv_saddr" every time a bpf prog is run. It can
only be properly initialized when a "sk->sk_reuseport" enabled sk is
adding to a hashtable (i.e. during "reuseport_alloc()" and
"reuseport_add_sock()").
The new "sk_select_reuseport()" is the main helper that the
bpf prog will use to select a SO_REUSEPORT sk. It is the only function
that can use the new BPF_MAP_TYPE_REUSEPORT_ARRAY. As mentioned in
the earlier patch, the validity of a selected sk is checked in
run time in "sk_select_reuseport()". Doing the check in
verification time is difficult and inflexible (consider the map-in-map
use case). The runtime check is to compare the selected sk's reuseport_id
with the reuseport_id that we want. This helper will return -EXXX if the
selected sk cannot serve the incoming request (e.g. reuseport_id
not match). The bpf prog can decide if it wants to do SK_DROP as its
discretion.
When the bpf prog returns SK_PASS, the kernel will check if a
valid sk has been selected (i.e. "reuse_kern->selected_sk != NULL").
If it does , it will use the selected sk. If not, the kernel
will select one from "reuse->socks[]" (as before this patch).
The SK_DROP and SK_PASS handling logic will be in the next patch.
Signed-off-by: Martin KaFai Lau <kafai@fb.com>
Acked-by: Alexei Starovoitov <ast@kernel.org>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2018-08-08 08:01:25 +00:00
|
|
|
return reuseport_alloc(sk, inet_rcv_saddr_any(sk));
|
2016-01-04 22:41:46 +00:00
|
|
|
}
|
|
|
|
|
2006-08-27 03:06:05 +00:00
|
|
|
/**
|
2008-03-22 23:51:21 +00:00
|
|
|
* udp_lib_get_port - UDP/-Lite port lookup for IPv4 and IPv6
|
2006-08-27 03:06:05 +00:00
|
|
|
*
|
|
|
|
* @sk: socket struct in question
|
|
|
|
* @snum: port number to look up
|
2011-03-31 01:57:33 +00:00
|
|
|
* @hash2_nulladdr: AF-dependent hash value in secondary hash chains,
|
2009-11-09 05:26:33 +00:00
|
|
|
* with NULL address
|
2006-08-27 03:06:05 +00:00
|
|
|
*/
|
2008-03-22 23:51:21 +00:00
|
|
|
int udp_lib_get_port(struct sock *sk, unsigned short snum,
|
2009-11-09 05:26:33 +00:00
|
|
|
unsigned int hash2_nulladdr)
|
2006-08-27 03:06:05 +00:00
|
|
|
{
|
2022-11-14 21:57:54 +00:00
|
|
|
struct udp_table *udptable = udp_get_table_prot(sk);
|
2022-11-14 21:57:53 +00:00
|
|
|
struct udp_hslot *hslot, *hslot2;
|
2008-03-25 17:26:21 +00:00
|
|
|
struct net *net = sock_net(sk);
|
2022-11-18 18:25:06 +00:00
|
|
|
int error = -EADDRINUSE;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2007-08-25 06:09:41 +00:00
|
|
|
if (!snum) {
|
2022-11-14 21:57:53 +00:00
|
|
|
DECLARE_BITMAP(bitmap, PORTS_PER_CHAIN);
|
|
|
|
unsigned short first, last;
|
2008-10-08 18:44:17 +00:00
|
|
|
int low, high, remaining;
|
2012-04-15 05:58:06 +00:00
|
|
|
unsigned int rand;
|
2007-08-25 06:09:41 +00:00
|
|
|
|
inet: Add IP_LOCAL_PORT_RANGE socket option
Users who want to share a single public IP address for outgoing connections
between several hosts traditionally reach for SNAT. However, SNAT requires
state keeping on the node(s) performing the NAT.
A stateless alternative exists, where a single IP address used for egress
can be shared between several hosts by partitioning the available ephemeral
port range. In such a setup:
1. Each host gets assigned a disjoint range of ephemeral ports.
2. Applications open connections from the host-assigned port range.
3. Return traffic gets routed to the host based on both, the destination IP
and the destination port.
An application which wants to open an outgoing connection (connect) from a
given port range today can choose between two solutions:
1. Manually pick the source port by bind()'ing to it before connect()'ing
the socket.
This approach has a couple of downsides:
a) Search for a free port has to be implemented in the user-space. If
the chosen 4-tuple happens to be busy, the application needs to retry
from a different local port number.
Detecting if 4-tuple is busy can be either easy (TCP) or hard
(UDP). In TCP case, the application simply has to check if connect()
returned an error (EADDRNOTAVAIL). That is assuming that the local
port sharing was enabled (REUSEADDR) by all the sockets.
# Assume desired local port range is 60_000-60_511
s = socket(AF_INET, SOCK_STREAM)
s.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
s.bind(("192.0.2.1", 60_000))
s.connect(("1.1.1.1", 53))
# Fails only if 192.0.2.1:60000 -> 1.1.1.1:53 is busy
# Application must retry with another local port
In case of UDP, the network stack allows binding more than one socket
to the same 4-tuple, when local port sharing is enabled
(REUSEADDR). Hence detecting the conflict is much harder and involves
querying sock_diag and toggling the REUSEADDR flag [1].
b) For TCP, bind()-ing to a port within the ephemeral port range means
that no connecting sockets, that is those which leave it to the
network stack to find a free local port at connect() time, can use
the this port.
IOW, the bind hash bucket tb->fastreuse will be 0 or 1, and the port
will be skipped during the free port search at connect() time.
2. Isolate the app in a dedicated netns and use the use the per-netns
ip_local_port_range sysctl to adjust the ephemeral port range bounds.
The per-netns setting affects all sockets, so this approach can be used
only if:
- there is just one egress IP address, or
- the desired egress port range is the same for all egress IP addresses
used by the application.
For TCP, this approach avoids the downsides of (1). Free port search and
4-tuple conflict detection is done by the network stack:
system("sysctl -w net.ipv4.ip_local_port_range='60000 60511'")
s = socket(AF_INET, SOCK_STREAM)
s.setsockopt(SOL_IP, IP_BIND_ADDRESS_NO_PORT, 1)
s.bind(("192.0.2.1", 0))
s.connect(("1.1.1.1", 53))
# Fails if all 4-tuples 192.0.2.1:60000-60511 -> 1.1.1.1:53 are busy
For UDP this approach has limited applicability. Setting the
IP_BIND_ADDRESS_NO_PORT socket option does not result in local source
port being shared with other connected UDP sockets.
Hence relying on the network stack to find a free source port, limits the
number of outgoing UDP flows from a single IP address down to the number
of available ephemeral ports.
To put it another way, partitioning the ephemeral port range between hosts
using the existing Linux networking API is cumbersome.
To address this use case, add a new socket option at the SOL_IP level,
named IP_LOCAL_PORT_RANGE. The new option can be used to clamp down the
ephemeral port range for each socket individually.
The option can be used only to narrow down the per-netns local port
range. If the per-socket range lies outside of the per-netns range, the
latter takes precedence.
UAPI-wise, the low and high range bounds are passed to the kernel as a pair
of u16 values in host byte order packed into a u32. This avoids pointer
passing.
PORT_LO = 40_000
PORT_HI = 40_511
s = socket(AF_INET, SOCK_STREAM)
v = struct.pack("I", PORT_HI << 16 | PORT_LO)
s.setsockopt(SOL_IP, IP_LOCAL_PORT_RANGE, v)
s.bind(("127.0.0.1", 0))
s.getsockname()
# Local address between ("127.0.0.1", 40_000) and ("127.0.0.1", 40_511),
# if there is a free port. EADDRINUSE otherwise.
[1] https://github.com/cloudflare/cloudflare-blog/blob/232b432c1d57/2022-02-connectx/connectx.py#L116
Reviewed-by: Marek Majkowski <marek@cloudflare.com>
Reviewed-by: Kuniyuki Iwashima <kuniyu@amazon.com>
Signed-off-by: Jakub Sitnicki <jakub@cloudflare.com>
Reviewed-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
2023-01-24 13:36:43 +00:00
|
|
|
inet_sk_get_local_port_range(sk, &low, &high);
|
2007-10-19 05:00:17 +00:00
|
|
|
remaining = (high - low) + 1;
|
2007-10-11 00:30:46 +00:00
|
|
|
|
2022-10-05 15:43:22 +00:00
|
|
|
rand = get_random_u32();
|
2014-08-23 18:58:54 +00:00
|
|
|
first = reciprocal_scale(rand, remaining) + low;
|
2009-01-27 05:35:35 +00:00
|
|
|
/*
|
|
|
|
* force rand to be an odd multiple of UDP_HTABLE_SIZE
|
|
|
|
*/
|
2009-10-07 00:37:59 +00:00
|
|
|
rand = (rand | 1) * (udptable->mask + 1);
|
2009-12-14 03:32:39 +00:00
|
|
|
last = first + udptable->mask + 1;
|
|
|
|
do {
|
2009-10-07 00:37:59 +00:00
|
|
|
hslot = udp_hashslot(udptable, net, first);
|
2009-01-27 05:35:35 +00:00
|
|
|
bitmap_zero(bitmap, PORTS_PER_CHAIN);
|
2008-10-29 08:41:45 +00:00
|
|
|
spin_lock_bh(&hslot->lock);
|
2009-01-27 05:35:35 +00:00
|
|
|
udp_lib_lport_inuse(net, snum, hslot, bitmap, sk,
|
2017-01-17 15:51:01 +00:00
|
|
|
udptable->log);
|
2009-01-27 05:35:35 +00:00
|
|
|
|
|
|
|
snum = first;
|
|
|
|
/*
|
|
|
|
* Iterate on all possible values of snum for this hash.
|
|
|
|
* Using steps of an odd multiple of UDP_HTABLE_SIZE
|
|
|
|
* give us randomization and full range coverage.
|
|
|
|
*/
|
2008-10-08 18:44:17 +00:00
|
|
|
do {
|
2009-01-27 05:35:35 +00:00
|
|
|
if (low <= snum && snum <= high &&
|
2010-05-05 00:27:06 +00:00
|
|
|
!test_bit(snum >> udptable->log, bitmap) &&
|
2014-05-12 23:04:53 +00:00
|
|
|
!inet_is_local_reserved_port(net, snum))
|
2009-01-27 05:35:35 +00:00
|
|
|
goto found;
|
|
|
|
snum += rand;
|
|
|
|
} while (snum != first);
|
|
|
|
spin_unlock_bh(&hslot->lock);
|
2017-01-06 01:22:36 +00:00
|
|
|
cond_resched();
|
2009-12-14 03:32:39 +00:00
|
|
|
} while (++first != last);
|
2009-01-27 05:35:35 +00:00
|
|
|
goto fail;
|
2008-10-29 08:41:45 +00:00
|
|
|
} else {
|
2009-10-07 00:37:59 +00:00
|
|
|
hslot = udp_hashslot(udptable, net, snum);
|
2008-10-29 08:41:45 +00:00
|
|
|
spin_lock_bh(&hslot->lock);
|
2009-11-09 05:26:33 +00:00
|
|
|
if (hslot->count > 10) {
|
|
|
|
int exist;
|
|
|
|
unsigned int slot2 = udp_sk(sk)->udp_portaddr_hash ^ snum;
|
|
|
|
|
|
|
|
slot2 &= udptable->mask;
|
|
|
|
hash2_nulladdr &= udptable->mask;
|
|
|
|
|
|
|
|
hslot2 = udp_hashslot2(udptable, slot2);
|
|
|
|
if (hslot->count < hslot2->count)
|
|
|
|
goto scan_primary_hash;
|
|
|
|
|
2017-01-17 15:51:01 +00:00
|
|
|
exist = udp_lib_lport_inuse2(net, snum, hslot2, sk);
|
2009-11-09 05:26:33 +00:00
|
|
|
if (!exist && (hash2_nulladdr != slot2)) {
|
|
|
|
hslot2 = udp_hashslot2(udptable, hash2_nulladdr);
|
|
|
|
exist = udp_lib_lport_inuse2(net, snum, hslot2,
|
2017-01-17 15:51:01 +00:00
|
|
|
sk);
|
2009-11-09 05:26:33 +00:00
|
|
|
}
|
|
|
|
if (exist)
|
|
|
|
goto fail_unlock;
|
|
|
|
else
|
|
|
|
goto found;
|
|
|
|
}
|
|
|
|
scan_primary_hash:
|
2017-01-17 15:51:01 +00:00
|
|
|
if (udp_lib_lport_inuse(net, snum, hslot, NULL, sk, 0))
|
2008-10-29 08:41:45 +00:00
|
|
|
goto fail_unlock;
|
|
|
|
}
|
2009-01-27 05:35:35 +00:00
|
|
|
found:
|
2009-10-15 06:30:45 +00:00
|
|
|
inet_sk(sk)->inet_num = snum;
|
2009-11-08 10:17:30 +00:00
|
|
|
udp_sk(sk)->udp_port_hash = snum;
|
|
|
|
udp_sk(sk)->udp_portaddr_hash ^= snum;
|
2005-04-16 22:20:36 +00:00
|
|
|
if (sk_unhashed(sk)) {
|
2016-01-04 22:41:46 +00:00
|
|
|
if (sk->sk_reuseport &&
|
2017-01-17 15:51:01 +00:00
|
|
|
udp_reuseport_add_sock(sk, hslot)) {
|
2016-01-04 22:41:46 +00:00
|
|
|
inet_sk(sk)->inet_num = 0;
|
|
|
|
udp_sk(sk)->udp_port_hash = 0;
|
|
|
|
udp_sk(sk)->udp_portaddr_hash ^= snum;
|
|
|
|
goto fail_unlock;
|
|
|
|
}
|
|
|
|
|
2016-04-01 15:52:13 +00:00
|
|
|
sk_add_node_rcu(sk, &hslot->head);
|
2009-11-08 10:17:05 +00:00
|
|
|
hslot->count++;
|
2008-04-01 02:41:46 +00:00
|
|
|
sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
|
2009-11-08 10:17:58 +00:00
|
|
|
|
|
|
|
hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash);
|
|
|
|
spin_lock(&hslot2->lock);
|
2016-04-12 17:11:25 +00:00
|
|
|
if (IS_ENABLED(CONFIG_IPV6) && sk->sk_reuseport &&
|
2016-04-23 22:26:24 +00:00
|
|
|
sk->sk_family == AF_INET6)
|
|
|
|
hlist_add_tail_rcu(&udp_sk(sk)->udp_portaddr_node,
|
|
|
|
&hslot2->head);
|
2016-04-12 17:11:25 +00:00
|
|
|
else
|
2016-04-23 22:26:24 +00:00
|
|
|
hlist_add_head_rcu(&udp_sk(sk)->udp_portaddr_node,
|
|
|
|
&hslot2->head);
|
2009-11-08 10:17:58 +00:00
|
|
|
hslot2->count++;
|
|
|
|
spin_unlock(&hslot2->lock);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2016-04-01 15:52:13 +00:00
|
|
|
sock_set_flag(sk, SOCK_RCU_FREE);
|
2006-08-27 03:06:05 +00:00
|
|
|
error = 0;
|
2008-10-29 08:41:45 +00:00
|
|
|
fail_unlock:
|
|
|
|
spin_unlock_bh(&hslot->lock);
|
2005-04-16 22:20:36 +00:00
|
|
|
fail:
|
2006-08-27 03:06:05 +00:00
|
|
|
return error;
|
|
|
|
}
|
2009-07-17 00:26:32 +00:00
|
|
|
EXPORT_SYMBOL(udp_lib_get_port);
|
2006-08-27 03:06:05 +00:00
|
|
|
|
2008-03-22 23:51:21 +00:00
|
|
|
int udp_v4_get_port(struct sock *sk, unsigned short snum)
|
2008-03-07 00:22:02 +00:00
|
|
|
{
|
2009-11-09 05:26:33 +00:00
|
|
|
unsigned int hash2_nulladdr =
|
2017-12-01 20:52:30 +00:00
|
|
|
ipv4_portaddr_hash(sock_net(sk), htonl(INADDR_ANY), snum);
|
2009-11-09 05:26:33 +00:00
|
|
|
unsigned int hash2_partial =
|
2017-12-01 20:52:30 +00:00
|
|
|
ipv4_portaddr_hash(sock_net(sk), inet_sk(sk)->inet_rcv_saddr, 0);
|
2009-11-09 05:26:33 +00:00
|
|
|
|
2009-11-08 10:17:30 +00:00
|
|
|
/* precompute partial secondary hash */
|
2009-11-09 05:26:33 +00:00
|
|
|
udp_sk(sk)->udp_portaddr_hash = hash2_partial;
|
2017-01-17 15:51:01 +00:00
|
|
|
return udp_lib_get_port(sk, snum, hash2_nulladdr);
|
2008-03-07 00:22:02 +00:00
|
|
|
}
|
|
|
|
|
udp reuseport: fix packet of same flow hashed to different socket
There is a corner case in which udp packets belonging to a same
flow are hashed to different socket when hslot->count changes from 10
to 11:
1) When hslot->count <= 10, __udp_lib_lookup() searches udp_table->hash,
and always passes 'daddr' to udp_ehashfn().
2) When hslot->count > 10, __udp_lib_lookup() searches udp_table->hash2,
but may pass 'INADDR_ANY' to udp_ehashfn() if the sockets are bound to
INADDR_ANY instead of some specific addr.
That means when hslot->count changes from 10 to 11, the hash calculated by
udp_ehashfn() is also changed, and the udp packets belonging to a same
flow will be hashed to different socket.
This is easily reproduced:
1) Create 10 udp sockets and bind all of them to 0.0.0.0:40000.
2) From the same host send udp packets to 127.0.0.1:40000, record the
socket index which receives the packets.
3) Create 1 more udp socket and bind it to 0.0.0.0:44096. The number 44096
is 40000 + UDP_HASH_SIZE(4096), this makes the new socket put into the
same hslot as the aformentioned 10 sockets, and makes the hslot->count
change from 10 to 11.
4) From the same host send udp packets to 127.0.0.1:40000, and the socket
index which receives the packets will be different from the one received
in step 2.
This should not happen as the socket bound to 0.0.0.0:44096 should not
change the behavior of the sockets bound to 0.0.0.0:40000.
It's the same case for IPv6, and this patch also fixes that.
Signed-off-by: Su, Xuemin <suxm@chinanetcenter.com>
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-06-13 03:02:50 +00:00
|
|
|
static int compute_score(struct sock *sk, struct net *net,
|
|
|
|
__be32 saddr, __be16 sport,
|
2017-08-07 15:44:16 +00:00
|
|
|
__be32 daddr, unsigned short hnum,
|
2019-06-14 04:41:26 +00:00
|
|
|
int dif, int sdif)
|
2008-10-29 08:41:45 +00:00
|
|
|
{
|
2014-12-02 04:29:06 +00:00
|
|
|
int score;
|
|
|
|
struct inet_sock *inet;
|
net: ensure unbound datagram socket to be chosen when not in a VRF
Ensure an unbound datagram skt is chosen when not in a VRF. The check
for a device match in compute_score() for UDP must be performed when
there is no device match. For this, a failure is returned when there is
no device match. This ensures that bound sockets are never selected,
even if there is no unbound socket.
Allow IPv6 packets to be sent over a datagram skt bound to a VRF. These
packets are currently blocked, as flowi6_oif was set to that of the
master vrf device, and the ipi6_ifindex is that of the slave device.
Allow these packets to be sent by checking the device with ipi6_ifindex
has the same L3 scope as that of the bound device of the skt, which is
the master vrf device. Note that this check always succeeds if the skt
is unbound.
Even though the right datagram skt is now selected by compute_score(),
a different skt is being returned that is bound to the wrong vrf. The
difference between these and stream sockets is the handling of the skt
option for SO_REUSEPORT. While the handling when adding a skt for reuse
correctly checks that the bound device of the skt is a match, the skts
in the hashslot are already incorrect. So for the same hash, a skt for
the wrong vrf may be selected for the required port. The root cause is
that the skt is immediately placed into a slot when it is created,
but when the skt is then bound using SO_BINDTODEVICE, it remains in the
same slot. The solution is to move the skt to the correct slot by
forcing a rehash.
Signed-off-by: Mike Manning <mmanning@vyatta.att-mail.com>
Reviewed-by: David Ahern <dsahern@gmail.com>
Tested-by: David Ahern <dsahern@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-11-07 15:36:04 +00:00
|
|
|
bool dev_match;
|
2008-10-29 08:41:45 +00:00
|
|
|
|
2014-12-02 04:29:06 +00:00
|
|
|
if (!net_eq(sock_net(sk), net) ||
|
|
|
|
udp_sk(sk)->udp_port_hash != hnum ||
|
|
|
|
ipv6_only_sock(sk))
|
|
|
|
return -1;
|
2008-10-29 08:41:45 +00:00
|
|
|
|
2018-12-12 21:15:33 +00:00
|
|
|
if (sk->sk_rcv_saddr != daddr)
|
|
|
|
return -1;
|
2014-12-02 04:29:06 +00:00
|
|
|
|
2018-12-12 21:15:33 +00:00
|
|
|
score = (sk->sk_family == PF_INET) ? 2 : 1;
|
2014-12-02 04:29:06 +00:00
|
|
|
|
2018-12-12 21:15:33 +00:00
|
|
|
inet = inet_sk(sk);
|
2014-12-02 04:29:06 +00:00
|
|
|
if (inet->inet_daddr) {
|
|
|
|
if (inet->inet_daddr != saddr)
|
|
|
|
return -1;
|
|
|
|
score += 4;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (inet->inet_dport) {
|
|
|
|
if (inet->inet_dport != sport)
|
|
|
|
return -1;
|
|
|
|
score += 4;
|
|
|
|
}
|
|
|
|
|
net: ensure unbound datagram socket to be chosen when not in a VRF
Ensure an unbound datagram skt is chosen when not in a VRF. The check
for a device match in compute_score() for UDP must be performed when
there is no device match. For this, a failure is returned when there is
no device match. This ensures that bound sockets are never selected,
even if there is no unbound socket.
Allow IPv6 packets to be sent over a datagram skt bound to a VRF. These
packets are currently blocked, as flowi6_oif was set to that of the
master vrf device, and the ipi6_ifindex is that of the slave device.
Allow these packets to be sent by checking the device with ipi6_ifindex
has the same L3 scope as that of the bound device of the skt, which is
the master vrf device. Note that this check always succeeds if the skt
is unbound.
Even though the right datagram skt is now selected by compute_score(),
a different skt is being returned that is bound to the wrong vrf. The
difference between these and stream sockets is the handling of the skt
option for SO_REUSEPORT. While the handling when adding a skt for reuse
correctly checks that the bound device of the skt is a match, the skts
in the hashslot are already incorrect. So for the same hash, a skt for
the wrong vrf may be selected for the required port. The root cause is
that the skt is immediately placed into a slot when it is created,
but when the skt is then bound using SO_BINDTODEVICE, it remains in the
same slot. The solution is to move the skt to the correct slot by
forcing a rehash.
Signed-off-by: Mike Manning <mmanning@vyatta.att-mail.com>
Reviewed-by: David Ahern <dsahern@gmail.com>
Tested-by: David Ahern <dsahern@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-11-07 15:36:04 +00:00
|
|
|
dev_match = udp_sk_bound_dev_eq(net, sk->sk_bound_dev_if,
|
|
|
|
dif, sdif);
|
|
|
|
if (!dev_match)
|
|
|
|
return -1;
|
2021-10-05 13:03:42 +00:00
|
|
|
if (sk->sk_bound_dev_if)
|
|
|
|
score += 4;
|
2017-08-07 15:44:16 +00:00
|
|
|
|
2019-10-30 20:00:04 +00:00
|
|
|
if (READ_ONCE(sk->sk_incoming_cpu) == raw_smp_processor_id())
|
2015-10-09 02:33:21 +00:00
|
|
|
score++;
|
2008-10-29 08:41:45 +00:00
|
|
|
return score;
|
|
|
|
}
|
|
|
|
|
2023-07-20 15:30:08 +00:00
|
|
|
INDIRECT_CALLABLE_SCOPE
|
|
|
|
u32 udp_ehashfn(const struct net *net, const __be32 laddr, const __u16 lport,
|
|
|
|
const __be32 faddr, const __be16 fport)
|
2013-10-19 19:48:51 +00:00
|
|
|
{
|
2013-10-19 19:48:57 +00:00
|
|
|
static u32 udp_ehash_secret __read_mostly;
|
|
|
|
|
|
|
|
net_get_random_once(&udp_ehash_secret, sizeof(udp_ehash_secret));
|
|
|
|
|
2013-10-19 19:48:51 +00:00
|
|
|
return __inet_ehashfn(laddr, lport, faddr, fport,
|
2013-10-19 19:48:57 +00:00
|
|
|
udp_ehash_secret + net_hash_mix(net));
|
2013-10-19 19:48:51 +00:00
|
|
|
}
|
|
|
|
|
udp reuseport: fix packet of same flow hashed to different socket
There is a corner case in which udp packets belonging to a same
flow are hashed to different socket when hslot->count changes from 10
to 11:
1) When hslot->count <= 10, __udp_lib_lookup() searches udp_table->hash,
and always passes 'daddr' to udp_ehashfn().
2) When hslot->count > 10, __udp_lib_lookup() searches udp_table->hash2,
but may pass 'INADDR_ANY' to udp_ehashfn() if the sockets are bound to
INADDR_ANY instead of some specific addr.
That means when hslot->count changes from 10 to 11, the hash calculated by
udp_ehashfn() is also changed, and the udp packets belonging to a same
flow will be hashed to different socket.
This is easily reproduced:
1) Create 10 udp sockets and bind all of them to 0.0.0.0:40000.
2) From the same host send udp packets to 127.0.0.1:40000, record the
socket index which receives the packets.
3) Create 1 more udp socket and bind it to 0.0.0.0:44096. The number 44096
is 40000 + UDP_HASH_SIZE(4096), this makes the new socket put into the
same hslot as the aformentioned 10 sockets, and makes the hslot->count
change from 10 to 11.
4) From the same host send udp packets to 127.0.0.1:40000, and the socket
index which receives the packets will be different from the one received
in step 2.
This should not happen as the socket bound to 0.0.0.0:44096 should not
change the behavior of the sockets bound to 0.0.0.0:40000.
It's the same case for IPv6, and this patch also fixes that.
Signed-off-by: Su, Xuemin <suxm@chinanetcenter.com>
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-06-13 03:02:50 +00:00
|
|
|
/* called with rcu_read_lock() */
|
ipv4: udp: optimize unicast RX path
We first locate the (local port) hash chain head
If few sockets are in this chain, we proceed with previous lookup algo.
If too many sockets are listed, we take a look at the secondary
(port, address) hash chain we added in previous patch.
We choose the shortest chain and proceed with a RCU lookup on the elected chain.
But, if we chose (port, address) chain, and fail to find a socket on given address,
we must try another lookup on (port, INADDR_ANY) chain to find socket not bound
to a particular IP.
-> No extra cost for typical setups, where the first lookup will probabbly
be performed.
RCU lookups everywhere, we dont acquire spinlock.
Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2009-11-08 10:18:11 +00:00
|
|
|
static struct sock *udp4_lib_lookup2(struct net *net,
|
2017-08-07 15:44:16 +00:00
|
|
|
__be32 saddr, __be16 sport,
|
|
|
|
__be32 daddr, unsigned int hnum,
|
2019-06-14 04:41:26 +00:00
|
|
|
int dif, int sdif,
|
2017-08-07 15:44:16 +00:00
|
|
|
struct udp_hslot *hslot2,
|
|
|
|
struct sk_buff *skb)
|
ipv4: udp: optimize unicast RX path
We first locate the (local port) hash chain head
If few sockets are in this chain, we proceed with previous lookup algo.
If too many sockets are listed, we take a look at the secondary
(port, address) hash chain we added in previous patch.
We choose the shortest chain and proceed with a RCU lookup on the elected chain.
But, if we chose (port, address) chain, and fail to find a socket on given address,
we must try another lookup on (port, INADDR_ANY) chain to find socket not bound
to a particular IP.
-> No extra cost for typical setups, where the first lookup will probabbly
be performed.
RCU lookups everywhere, we dont acquire spinlock.
Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2009-11-08 10:18:11 +00:00
|
|
|
{
|
|
|
|
struct sock *sk, *result;
|
2017-11-30 14:39:34 +00:00
|
|
|
int score, badness;
|
ipv4: udp: optimize unicast RX path
We first locate the (local port) hash chain head
If few sockets are in this chain, we proceed with previous lookup algo.
If too many sockets are listed, we take a look at the secondary
(port, address) hash chain we added in previous patch.
We choose the shortest chain and proceed with a RCU lookup on the elected chain.
But, if we chose (port, address) chain, and fail to find a socket on given address,
we must try another lookup on (port, INADDR_ANY) chain to find socket not bound
to a particular IP.
-> No extra cost for typical setups, where the first lookup will probabbly
be performed.
RCU lookups everywhere, we dont acquire spinlock.
Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2009-11-08 10:18:11 +00:00
|
|
|
|
|
|
|
result = NULL;
|
2013-01-22 09:50:32 +00:00
|
|
|
badness = 0;
|
2016-04-01 15:52:13 +00:00
|
|
|
udp_portaddr_for_each_entry_rcu(sk, &hslot2->head) {
|
udp reuseport: fix packet of same flow hashed to different socket
There is a corner case in which udp packets belonging to a same
flow are hashed to different socket when hslot->count changes from 10
to 11:
1) When hslot->count <= 10, __udp_lib_lookup() searches udp_table->hash,
and always passes 'daddr' to udp_ehashfn().
2) When hslot->count > 10, __udp_lib_lookup() searches udp_table->hash2,
but may pass 'INADDR_ANY' to udp_ehashfn() if the sockets are bound to
INADDR_ANY instead of some specific addr.
That means when hslot->count changes from 10 to 11, the hash calculated by
udp_ehashfn() is also changed, and the udp packets belonging to a same
flow will be hashed to different socket.
This is easily reproduced:
1) Create 10 udp sockets and bind all of them to 0.0.0.0:40000.
2) From the same host send udp packets to 127.0.0.1:40000, record the
socket index which receives the packets.
3) Create 1 more udp socket and bind it to 0.0.0.0:44096. The number 44096
is 40000 + UDP_HASH_SIZE(4096), this makes the new socket put into the
same hslot as the aformentioned 10 sockets, and makes the hslot->count
change from 10 to 11.
4) From the same host send udp packets to 127.0.0.1:40000, and the socket
index which receives the packets will be different from the one received
in step 2.
This should not happen as the socket bound to 0.0.0.0:44096 should not
change the behavior of the sockets bound to 0.0.0.0:40000.
It's the same case for IPv6, and this patch also fixes that.
Signed-off-by: Su, Xuemin <suxm@chinanetcenter.com>
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-06-13 03:02:50 +00:00
|
|
|
score = compute_score(sk, net, saddr, sport,
|
2019-06-14 04:41:26 +00:00
|
|
|
daddr, hnum, dif, sdif);
|
ipv4: udp: optimize unicast RX path
We first locate the (local port) hash chain head
If few sockets are in this chain, we proceed with previous lookup algo.
If too many sockets are listed, we take a look at the secondary
(port, address) hash chain we added in previous patch.
We choose the shortest chain and proceed with a RCU lookup on the elected chain.
But, if we chose (port, address) chain, and fail to find a socket on given address,
we must try another lookup on (port, INADDR_ANY) chain to find socket not bound
to a particular IP.
-> No extra cost for typical setups, where the first lookup will probabbly
be performed.
RCU lookups everywhere, we dont acquire spinlock.
Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2009-11-08 10:18:11 +00:00
|
|
|
if (score > badness) {
|
2023-07-20 15:30:05 +00:00
|
|
|
badness = score;
|
2023-07-20 15:30:08 +00:00
|
|
|
|
|
|
|
if (sk->sk_state == TCP_ESTABLISHED) {
|
|
|
|
result = sk;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
result = inet_lookup_reuseport(net, sk, skb, sizeof(struct udphdr),
|
|
|
|
saddr, sport, daddr, hnum, udp_ehashfn);
|
2023-07-20 15:30:05 +00:00
|
|
|
if (!result) {
|
|
|
|
result = sk;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
2020-07-26 00:49:04 +00:00
|
|
|
/* Fall back to scoring if group has connections */
|
2023-07-20 15:30:05 +00:00
|
|
|
if (!reuseport_has_conns(sk))
|
2020-07-17 10:35:28 +00:00
|
|
|
return result;
|
|
|
|
|
2023-07-20 15:30:05 +00:00
|
|
|
/* Reuseport logic returned an error, keep original score. */
|
|
|
|
if (IS_ERR(result))
|
|
|
|
continue;
|
|
|
|
|
|
|
|
badness = compute_score(result, net, saddr, sport,
|
|
|
|
daddr, hnum, dif, sdif);
|
|
|
|
|
ipv4: udp: optimize unicast RX path
We first locate the (local port) hash chain head
If few sockets are in this chain, we proceed with previous lookup algo.
If too many sockets are listed, we take a look at the secondary
(port, address) hash chain we added in previous patch.
We choose the shortest chain and proceed with a RCU lookup on the elected chain.
But, if we chose (port, address) chain, and fail to find a socket on given address,
we must try another lookup on (port, INADDR_ANY) chain to find socket not bound
to a particular IP.
-> No extra cost for typical setups, where the first lookup will probabbly
be performed.
RCU lookups everywhere, we dont acquire spinlock.
Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2009-11-08 10:18:11 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2008-03-07 00:22:02 +00:00
|
|
|
/* UDP is nearly always wildcards out the wazoo, it makes no sense to try
|
|
|
|
* harder than this. -DaveM
|
|
|
|
*/
|
2011-12-09 06:23:34 +00:00
|
|
|
struct sock *__udp4_lib_lookup(struct net *net, __be32 saddr,
|
2017-08-07 15:44:16 +00:00
|
|
|
__be16 sport, __be32 daddr, __be16 dport, int dif,
|
|
|
|
int sdif, struct udp_table *udptable, struct sk_buff *skb)
|
2008-03-07 00:22:02 +00:00
|
|
|
{
|
|
|
|
unsigned short hnum = ntohs(dport);
|
2018-12-12 21:15:33 +00:00
|
|
|
unsigned int hash2, slot2;
|
|
|
|
struct udp_hslot *hslot2;
|
2020-07-17 10:35:29 +00:00
|
|
|
struct sock *result, *sk;
|
2008-10-29 08:41:45 +00:00
|
|
|
|
2018-12-12 21:15:33 +00:00
|
|
|
hash2 = ipv4_portaddr_hash(net, daddr, hnum);
|
|
|
|
slot2 = hash2 & udptable->mask;
|
|
|
|
hslot2 = &udptable->hash2[slot2];
|
|
|
|
|
2020-07-17 10:35:29 +00:00
|
|
|
/* Lookup connected or non-wildcard socket */
|
2018-12-12 21:15:33 +00:00
|
|
|
result = udp4_lib_lookup2(net, saddr, sport,
|
|
|
|
daddr, hnum, dif, sdif,
|
2019-06-14 04:41:26 +00:00
|
|
|
hslot2, skb);
|
2020-07-17 10:35:29 +00:00
|
|
|
if (!IS_ERR_OR_NULL(result) && result->sk_state == TCP_ESTABLISHED)
|
|
|
|
goto done;
|
|
|
|
|
|
|
|
/* Lookup redirect from BPF */
|
2023-07-20 15:30:10 +00:00
|
|
|
if (static_branch_unlikely(&bpf_sk_lookup_enabled) &&
|
|
|
|
udptable == net->ipv4.udp_table) {
|
|
|
|
sk = inet_lookup_run_sk_lookup(net, IPPROTO_UDP, skb, sizeof(struct udphdr),
|
|
|
|
saddr, sport, daddr, hnum, dif,
|
|
|
|
udp_ehashfn);
|
2020-07-17 10:35:29 +00:00
|
|
|
if (sk) {
|
|
|
|
result = sk;
|
|
|
|
goto done;
|
|
|
|
}
|
2008-03-07 00:22:02 +00:00
|
|
|
}
|
2020-07-17 10:35:29 +00:00
|
|
|
|
|
|
|
/* Got non-wildcard socket or error on first lookup */
|
|
|
|
if (result)
|
|
|
|
goto done;
|
|
|
|
|
|
|
|
/* Lookup wildcard sockets */
|
|
|
|
hash2 = ipv4_portaddr_hash(net, htonl(INADDR_ANY), hnum);
|
|
|
|
slot2 = hash2 & udptable->mask;
|
|
|
|
hslot2 = &udptable->hash2[slot2];
|
|
|
|
|
|
|
|
result = udp4_lib_lookup2(net, saddr, sport,
|
|
|
|
htonl(INADDR_ANY), hnum, dif, sdif,
|
|
|
|
hslot2, skb);
|
|
|
|
done:
|
2019-06-05 21:09:05 +00:00
|
|
|
if (IS_ERR(result))
|
2018-12-12 21:15:33 +00:00
|
|
|
return NULL;
|
2008-03-07 00:22:02 +00:00
|
|
|
return result;
|
|
|
|
}
|
2011-12-09 06:23:34 +00:00
|
|
|
EXPORT_SYMBOL_GPL(__udp4_lib_lookup);
|
2008-03-07 00:22:02 +00:00
|
|
|
|
2008-10-07 19:38:32 +00:00
|
|
|
static inline struct sock *__udp4_lib_lookup_skb(struct sk_buff *skb,
|
|
|
|
__be16 sport, __be16 dport,
|
2008-10-29 08:41:45 +00:00
|
|
|
struct udp_table *udptable)
|
2008-10-07 19:38:32 +00:00
|
|
|
{
|
|
|
|
const struct iphdr *iph = ip_hdr(skb);
|
|
|
|
|
2016-05-12 23:23:44 +00:00
|
|
|
return __udp4_lib_lookup(dev_net(skb->dev), iph->saddr, sport,
|
2013-12-11 02:07:23 +00:00
|
|
|
iph->daddr, dport, inet_iif(skb),
|
2017-08-07 15:44:16 +00:00
|
|
|
inet_sdif(skb), udptable, skb);
|
2008-10-07 19:38:32 +00:00
|
|
|
}
|
|
|
|
|
2020-11-09 23:13:49 +00:00
|
|
|
struct sock *udp4_lib_lookup_skb(const struct sk_buff *skb,
|
2016-04-05 15:22:50 +00:00
|
|
|
__be16 sport, __be16 dport)
|
|
|
|
{
|
2019-05-31 22:29:13 +00:00
|
|
|
const struct iphdr *iph = ip_hdr(skb);
|
2022-11-14 21:57:56 +00:00
|
|
|
struct net *net = dev_net(skb->dev);
|
2023-07-27 15:33:56 +00:00
|
|
|
int iif, sdif;
|
|
|
|
|
|
|
|
inet_get_iif_sdif(skb, &iif, &sdif);
|
2019-05-31 22:29:13 +00:00
|
|
|
|
2022-11-14 21:57:56 +00:00
|
|
|
return __udp4_lib_lookup(net, iph->saddr, sport,
|
2023-07-27 15:33:56 +00:00
|
|
|
iph->daddr, dport, iif,
|
|
|
|
sdif, net->ipv4.udp_table, NULL);
|
2016-04-05 15:22:50 +00:00
|
|
|
}
|
|
|
|
|
2016-04-01 15:52:13 +00:00
|
|
|
/* Must be called under rcu_read_lock().
|
|
|
|
* Does increment socket refcount.
|
|
|
|
*/
|
2018-06-05 11:40:34 +00:00
|
|
|
#if IS_ENABLED(CONFIG_NF_TPROXY_IPV4) || IS_ENABLED(CONFIG_NF_SOCKET_IPV4)
|
2008-10-01 14:48:10 +00:00
|
|
|
struct sock *udp4_lib_lookup(struct net *net, __be32 saddr, __be16 sport,
|
|
|
|
__be32 daddr, __be16 dport, int dif)
|
|
|
|
{
|
2016-04-01 15:52:13 +00:00
|
|
|
struct sock *sk;
|
|
|
|
|
|
|
|
sk = __udp4_lib_lookup(net, saddr, sport, daddr, dport,
|
2022-11-14 21:57:56 +00:00
|
|
|
dif, 0, net->ipv4.udp_table, NULL);
|
2017-06-30 10:08:01 +00:00
|
|
|
if (sk && !refcount_inc_not_zero(&sk->sk_refcnt))
|
2016-04-01 15:52:13 +00:00
|
|
|
sk = NULL;
|
|
|
|
return sk;
|
2008-10-01 14:48:10 +00:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(udp4_lib_lookup);
|
2016-04-01 15:52:13 +00:00
|
|
|
#endif
|
2008-10-01 14:48:10 +00:00
|
|
|
|
2023-03-16 15:31:57 +00:00
|
|
|
static inline bool __udp_is_mcast_sock(struct net *net, const struct sock *sk,
|
2013-10-07 16:01:39 +00:00
|
|
|
__be16 loc_port, __be32 loc_addr,
|
|
|
|
__be16 rmt_port, __be32 rmt_addr,
|
2017-08-07 15:44:16 +00:00
|
|
|
int dif, int sdif, unsigned short hnum)
|
2013-10-07 16:01:39 +00:00
|
|
|
{
|
2023-03-16 15:31:57 +00:00
|
|
|
const struct inet_sock *inet = inet_sk(sk);
|
2013-10-07 16:01:39 +00:00
|
|
|
|
|
|
|
if (!net_eq(sock_net(sk), net) ||
|
|
|
|
udp_sk(sk)->udp_port_hash != hnum ||
|
|
|
|
(inet->inet_daddr && inet->inet_daddr != rmt_addr) ||
|
|
|
|
(inet->inet_dport != rmt_port && inet->inet_dport) ||
|
|
|
|
(inet->inet_rcv_saddr && inet->inet_rcv_saddr != loc_addr) ||
|
|
|
|
ipv6_only_sock(sk) ||
|
2019-06-04 01:56:23 +00:00
|
|
|
!udp_sk_bound_dev_eq(net, sk->sk_bound_dev_if, dif, sdif))
|
2013-10-07 16:01:39 +00:00
|
|
|
return false;
|
2017-08-07 15:44:19 +00:00
|
|
|
if (!ip_mc_sf_allow(sk, loc_addr, rmt_addr, dif, sdif))
|
2013-10-07 16:01:39 +00:00
|
|
|
return false;
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
udp: Handle ICMP errors for tunnels with same destination port on both endpoints
For both IPv4 and IPv6, if we can't match errors to a socket, try
tunnels before ignoring them. Look up a socket with the original source
and destination ports as found in the UDP packet inside the ICMP payload,
this will work for tunnels that force the same destination port for both
endpoints, i.e. VXLAN and GENEVE.
Actually, lwtunnels could break this assumption if they are configured by
an external control plane to have different destination ports on the
endpoints: in this case, we won't be able to trace ICMP messages back to
them.
For IPv6 redirect messages, call ip6_redirect() directly with the output
interface argument set to the interface we received the packet from (as
it's the very interface we should build the exception on), otherwise the
new nexthop will be rejected. There's no such need for IPv4.
Tunnels can now export an encap_err_lookup() operation that indicates a
match. Pass the packet to the lookup function, and if the tunnel driver
reports a matching association, continue with regular ICMP error handling.
v2:
- Added newline between network and transport header sets in
__udp{4,6}_lib_err_encap() (David Miller)
- Removed redundant skb_reset_network_header(skb); in
__udp4_lib_err_encap()
- Removed redundant reassignment of iph in __udp4_lib_err_encap()
(Sabrina Dubroca)
- Edited comment to __udp{4,6}_lib_err_encap() to reflect the fact this
won't work with lwtunnels configured to use asymmetric ports. By the way,
it's VXLAN, not VxLAN (Jiri Benc)
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
Reviewed-by: Sabrina Dubroca <sd@queasysnail.net>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-11-08 11:19:14 +00:00
|
|
|
DEFINE_STATIC_KEY_FALSE(udp_encap_needed_key);
|
|
|
|
void udp_encap_enable(void)
|
|
|
|
{
|
2018-11-15 01:34:50 +00:00
|
|
|
static_branch_inc(&udp_encap_needed_key);
|
udp: Handle ICMP errors for tunnels with same destination port on both endpoints
For both IPv4 and IPv6, if we can't match errors to a socket, try
tunnels before ignoring them. Look up a socket with the original source
and destination ports as found in the UDP packet inside the ICMP payload,
this will work for tunnels that force the same destination port for both
endpoints, i.e. VXLAN and GENEVE.
Actually, lwtunnels could break this assumption if they are configured by
an external control plane to have different destination ports on the
endpoints: in this case, we won't be able to trace ICMP messages back to
them.
For IPv6 redirect messages, call ip6_redirect() directly with the output
interface argument set to the interface we received the packet from (as
it's the very interface we should build the exception on), otherwise the
new nexthop will be rejected. There's no such need for IPv4.
Tunnels can now export an encap_err_lookup() operation that indicates a
match. Pass the packet to the lookup function, and if the tunnel driver
reports a matching association, continue with regular ICMP error handling.
v2:
- Added newline between network and transport header sets in
__udp{4,6}_lib_err_encap() (David Miller)
- Removed redundant skb_reset_network_header(skb); in
__udp4_lib_err_encap()
- Removed redundant reassignment of iph in __udp4_lib_err_encap()
(Sabrina Dubroca)
- Edited comment to __udp{4,6}_lib_err_encap() to reflect the fact this
won't work with lwtunnels configured to use asymmetric ports. By the way,
it's VXLAN, not VxLAN (Jiri Benc)
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
Reviewed-by: Sabrina Dubroca <sd@queasysnail.net>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-11-08 11:19:14 +00:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(udp_encap_enable);
|
|
|
|
|
2021-02-03 08:54:22 +00:00
|
|
|
void udp_encap_disable(void)
|
|
|
|
{
|
|
|
|
static_branch_dec(&udp_encap_needed_key);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(udp_encap_disable);
|
|
|
|
|
udp: Support for error handlers of tunnels with arbitrary destination port
ICMP error handling is currently not possible for UDP tunnels not
employing a receiving socket with local destination port matching the
remote one, because we have no way to look them up.
Add an err_handler tunnel encapsulation operation that can be exported by
tunnels in order to pass the error to the protocol implementing the
encapsulation. We can't easily use a lookup function as we did for VXLAN
and GENEVE, as protocol error handlers, which would be in turn called by
implementations of this new operation, handle the errors themselves,
together with the tunnel lookup.
Without a socket, we can't be sure which encapsulation error handler is
the appropriate one: encapsulation handlers (the ones for FoU and GUE
introduced in the next patch, e.g.) will need to check the new error codes
returned by protocol handlers to figure out if errors match the given
encapsulation, and, in turn, report this error back, so that we can try
all of them in __udp{4,6}_lib_err_encap_no_sk() until we have a match.
v2:
- Name all arguments in err_handler prototypes (David Miller)
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
Reviewed-by: Sabrina Dubroca <sd@queasysnail.net>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-11-08 11:19:22 +00:00
|
|
|
/* Handler for tunnels with arbitrary destination ports: no socket lookup, go
|
|
|
|
* through error handlers in encapsulations looking for a match.
|
|
|
|
*/
|
|
|
|
static int __udp4_lib_err_encap_no_sk(struct sk_buff *skb, u32 info)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = 0; i < MAX_IPTUN_ENCAP_OPS; i++) {
|
|
|
|
int (*handler)(struct sk_buff *skb, u32 info);
|
2019-02-21 16:44:00 +00:00
|
|
|
const struct ip_tunnel_encap_ops *encap;
|
udp: Support for error handlers of tunnels with arbitrary destination port
ICMP error handling is currently not possible for UDP tunnels not
employing a receiving socket with local destination port matching the
remote one, because we have no way to look them up.
Add an err_handler tunnel encapsulation operation that can be exported by
tunnels in order to pass the error to the protocol implementing the
encapsulation. We can't easily use a lookup function as we did for VXLAN
and GENEVE, as protocol error handlers, which would be in turn called by
implementations of this new operation, handle the errors themselves,
together with the tunnel lookup.
Without a socket, we can't be sure which encapsulation error handler is
the appropriate one: encapsulation handlers (the ones for FoU and GUE
introduced in the next patch, e.g.) will need to check the new error codes
returned by protocol handlers to figure out if errors match the given
encapsulation, and, in turn, report this error back, so that we can try
all of them in __udp{4,6}_lib_err_encap_no_sk() until we have a match.
v2:
- Name all arguments in err_handler prototypes (David Miller)
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
Reviewed-by: Sabrina Dubroca <sd@queasysnail.net>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-11-08 11:19:22 +00:00
|
|
|
|
2019-02-21 16:44:00 +00:00
|
|
|
encap = rcu_dereference(iptun_encaps[i]);
|
|
|
|
if (!encap)
|
udp: Support for error handlers of tunnels with arbitrary destination port
ICMP error handling is currently not possible for UDP tunnels not
employing a receiving socket with local destination port matching the
remote one, because we have no way to look them up.
Add an err_handler tunnel encapsulation operation that can be exported by
tunnels in order to pass the error to the protocol implementing the
encapsulation. We can't easily use a lookup function as we did for VXLAN
and GENEVE, as protocol error handlers, which would be in turn called by
implementations of this new operation, handle the errors themselves,
together with the tunnel lookup.
Without a socket, we can't be sure which encapsulation error handler is
the appropriate one: encapsulation handlers (the ones for FoU and GUE
introduced in the next patch, e.g.) will need to check the new error codes
returned by protocol handlers to figure out if errors match the given
encapsulation, and, in turn, report this error back, so that we can try
all of them in __udp{4,6}_lib_err_encap_no_sk() until we have a match.
v2:
- Name all arguments in err_handler prototypes (David Miller)
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
Reviewed-by: Sabrina Dubroca <sd@queasysnail.net>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-11-08 11:19:22 +00:00
|
|
|
continue;
|
2019-02-21 16:44:00 +00:00
|
|
|
handler = encap->err_handler;
|
udp: Support for error handlers of tunnels with arbitrary destination port
ICMP error handling is currently not possible for UDP tunnels not
employing a receiving socket with local destination port matching the
remote one, because we have no way to look them up.
Add an err_handler tunnel encapsulation operation that can be exported by
tunnels in order to pass the error to the protocol implementing the
encapsulation. We can't easily use a lookup function as we did for VXLAN
and GENEVE, as protocol error handlers, which would be in turn called by
implementations of this new operation, handle the errors themselves,
together with the tunnel lookup.
Without a socket, we can't be sure which encapsulation error handler is
the appropriate one: encapsulation handlers (the ones for FoU and GUE
introduced in the next patch, e.g.) will need to check the new error codes
returned by protocol handlers to figure out if errors match the given
encapsulation, and, in turn, report this error back, so that we can try
all of them in __udp{4,6}_lib_err_encap_no_sk() until we have a match.
v2:
- Name all arguments in err_handler prototypes (David Miller)
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
Reviewed-by: Sabrina Dubroca <sd@queasysnail.net>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-11-08 11:19:22 +00:00
|
|
|
if (handler && !handler(skb, info))
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
return -ENOENT;
|
|
|
|
}
|
|
|
|
|
udp: Handle ICMP errors for tunnels with same destination port on both endpoints
For both IPv4 and IPv6, if we can't match errors to a socket, try
tunnels before ignoring them. Look up a socket with the original source
and destination ports as found in the UDP packet inside the ICMP payload,
this will work for tunnels that force the same destination port for both
endpoints, i.e. VXLAN and GENEVE.
Actually, lwtunnels could break this assumption if they are configured by
an external control plane to have different destination ports on the
endpoints: in this case, we won't be able to trace ICMP messages back to
them.
For IPv6 redirect messages, call ip6_redirect() directly with the output
interface argument set to the interface we received the packet from (as
it's the very interface we should build the exception on), otherwise the
new nexthop will be rejected. There's no such need for IPv4.
Tunnels can now export an encap_err_lookup() operation that indicates a
match. Pass the packet to the lookup function, and if the tunnel driver
reports a matching association, continue with regular ICMP error handling.
v2:
- Added newline between network and transport header sets in
__udp{4,6}_lib_err_encap() (David Miller)
- Removed redundant skb_reset_network_header(skb); in
__udp4_lib_err_encap()
- Removed redundant reassignment of iph in __udp4_lib_err_encap()
(Sabrina Dubroca)
- Edited comment to __udp{4,6}_lib_err_encap() to reflect the fact this
won't work with lwtunnels configured to use asymmetric ports. By the way,
it's VXLAN, not VxLAN (Jiri Benc)
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
Reviewed-by: Sabrina Dubroca <sd@queasysnail.net>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-11-08 11:19:14 +00:00
|
|
|
/* Try to match ICMP errors to UDP tunnels by looking up a socket without
|
|
|
|
* reversing source and destination port: this will match tunnels that force the
|
|
|
|
* same destination port on both endpoints (e.g. VXLAN, GENEVE). Note that
|
|
|
|
* lwtunnels might actually break this assumption by being configured with
|
|
|
|
* different destination ports on endpoints, in this case we won't be able to
|
|
|
|
* trace ICMP messages back to them.
|
|
|
|
*
|
udp: Support for error handlers of tunnels with arbitrary destination port
ICMP error handling is currently not possible for UDP tunnels not
employing a receiving socket with local destination port matching the
remote one, because we have no way to look them up.
Add an err_handler tunnel encapsulation operation that can be exported by
tunnels in order to pass the error to the protocol implementing the
encapsulation. We can't easily use a lookup function as we did for VXLAN
and GENEVE, as protocol error handlers, which would be in turn called by
implementations of this new operation, handle the errors themselves,
together with the tunnel lookup.
Without a socket, we can't be sure which encapsulation error handler is
the appropriate one: encapsulation handlers (the ones for FoU and GUE
introduced in the next patch, e.g.) will need to check the new error codes
returned by protocol handlers to figure out if errors match the given
encapsulation, and, in turn, report this error back, so that we can try
all of them in __udp{4,6}_lib_err_encap_no_sk() until we have a match.
v2:
- Name all arguments in err_handler prototypes (David Miller)
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
Reviewed-by: Sabrina Dubroca <sd@queasysnail.net>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-11-08 11:19:22 +00:00
|
|
|
* If this doesn't match any socket, probe tunnels with arbitrary destination
|
|
|
|
* ports (e.g. FoU, GUE): there, the receiving socket is useless, as the port
|
|
|
|
* we've sent packets to won't necessarily match the local destination port.
|
|
|
|
*
|
udp: Handle ICMP errors for tunnels with same destination port on both endpoints
For both IPv4 and IPv6, if we can't match errors to a socket, try
tunnels before ignoring them. Look up a socket with the original source
and destination ports as found in the UDP packet inside the ICMP payload,
this will work for tunnels that force the same destination port for both
endpoints, i.e. VXLAN and GENEVE.
Actually, lwtunnels could break this assumption if they are configured by
an external control plane to have different destination ports on the
endpoints: in this case, we won't be able to trace ICMP messages back to
them.
For IPv6 redirect messages, call ip6_redirect() directly with the output
interface argument set to the interface we received the packet from (as
it's the very interface we should build the exception on), otherwise the
new nexthop will be rejected. There's no such need for IPv4.
Tunnels can now export an encap_err_lookup() operation that indicates a
match. Pass the packet to the lookup function, and if the tunnel driver
reports a matching association, continue with regular ICMP error handling.
v2:
- Added newline between network and transport header sets in
__udp{4,6}_lib_err_encap() (David Miller)
- Removed redundant skb_reset_network_header(skb); in
__udp4_lib_err_encap()
- Removed redundant reassignment of iph in __udp4_lib_err_encap()
(Sabrina Dubroca)
- Edited comment to __udp{4,6}_lib_err_encap() to reflect the fact this
won't work with lwtunnels configured to use asymmetric ports. By the way,
it's VXLAN, not VxLAN (Jiri Benc)
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
Reviewed-by: Sabrina Dubroca <sd@queasysnail.net>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-11-08 11:19:14 +00:00
|
|
|
* Then ask the tunnel implementation to match the error against a valid
|
|
|
|
* association.
|
|
|
|
*
|
udp: Support for error handlers of tunnels with arbitrary destination port
ICMP error handling is currently not possible for UDP tunnels not
employing a receiving socket with local destination port matching the
remote one, because we have no way to look them up.
Add an err_handler tunnel encapsulation operation that can be exported by
tunnels in order to pass the error to the protocol implementing the
encapsulation. We can't easily use a lookup function as we did for VXLAN
and GENEVE, as protocol error handlers, which would be in turn called by
implementations of this new operation, handle the errors themselves,
together with the tunnel lookup.
Without a socket, we can't be sure which encapsulation error handler is
the appropriate one: encapsulation handlers (the ones for FoU and GUE
introduced in the next patch, e.g.) will need to check the new error codes
returned by protocol handlers to figure out if errors match the given
encapsulation, and, in turn, report this error back, so that we can try
all of them in __udp{4,6}_lib_err_encap_no_sk() until we have a match.
v2:
- Name all arguments in err_handler prototypes (David Miller)
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
Reviewed-by: Sabrina Dubroca <sd@queasysnail.net>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-11-08 11:19:22 +00:00
|
|
|
* Return an error if we can't find a match, the socket if we need further
|
|
|
|
* processing, zero otherwise.
|
udp: Handle ICMP errors for tunnels with same destination port on both endpoints
For both IPv4 and IPv6, if we can't match errors to a socket, try
tunnels before ignoring them. Look up a socket with the original source
and destination ports as found in the UDP packet inside the ICMP payload,
this will work for tunnels that force the same destination port for both
endpoints, i.e. VXLAN and GENEVE.
Actually, lwtunnels could break this assumption if they are configured by
an external control plane to have different destination ports on the
endpoints: in this case, we won't be able to trace ICMP messages back to
them.
For IPv6 redirect messages, call ip6_redirect() directly with the output
interface argument set to the interface we received the packet from (as
it's the very interface we should build the exception on), otherwise the
new nexthop will be rejected. There's no such need for IPv4.
Tunnels can now export an encap_err_lookup() operation that indicates a
match. Pass the packet to the lookup function, and if the tunnel driver
reports a matching association, continue with regular ICMP error handling.
v2:
- Added newline between network and transport header sets in
__udp{4,6}_lib_err_encap() (David Miller)
- Removed redundant skb_reset_network_header(skb); in
__udp4_lib_err_encap()
- Removed redundant reassignment of iph in __udp4_lib_err_encap()
(Sabrina Dubroca)
- Edited comment to __udp{4,6}_lib_err_encap() to reflect the fact this
won't work with lwtunnels configured to use asymmetric ports. By the way,
it's VXLAN, not VxLAN (Jiri Benc)
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
Reviewed-by: Sabrina Dubroca <sd@queasysnail.net>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-11-08 11:19:14 +00:00
|
|
|
*/
|
|
|
|
static struct sock *__udp4_lib_err_encap(struct net *net,
|
|
|
|
const struct iphdr *iph,
|
|
|
|
struct udphdr *uh,
|
|
|
|
struct udp_table *udptable,
|
2021-07-20 20:35:28 +00:00
|
|
|
struct sock *sk,
|
udp: Support for error handlers of tunnels with arbitrary destination port
ICMP error handling is currently not possible for UDP tunnels not
employing a receiving socket with local destination port matching the
remote one, because we have no way to look them up.
Add an err_handler tunnel encapsulation operation that can be exported by
tunnels in order to pass the error to the protocol implementing the
encapsulation. We can't easily use a lookup function as we did for VXLAN
and GENEVE, as protocol error handlers, which would be in turn called by
implementations of this new operation, handle the errors themselves,
together with the tunnel lookup.
Without a socket, we can't be sure which encapsulation error handler is
the appropriate one: encapsulation handlers (the ones for FoU and GUE
introduced in the next patch, e.g.) will need to check the new error codes
returned by protocol handlers to figure out if errors match the given
encapsulation, and, in turn, report this error back, so that we can try
all of them in __udp{4,6}_lib_err_encap_no_sk() until we have a match.
v2:
- Name all arguments in err_handler prototypes (David Miller)
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
Reviewed-by: Sabrina Dubroca <sd@queasysnail.net>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-11-08 11:19:22 +00:00
|
|
|
struct sk_buff *skb, u32 info)
|
udp: Handle ICMP errors for tunnels with same destination port on both endpoints
For both IPv4 and IPv6, if we can't match errors to a socket, try
tunnels before ignoring them. Look up a socket with the original source
and destination ports as found in the UDP packet inside the ICMP payload,
this will work for tunnels that force the same destination port for both
endpoints, i.e. VXLAN and GENEVE.
Actually, lwtunnels could break this assumption if they are configured by
an external control plane to have different destination ports on the
endpoints: in this case, we won't be able to trace ICMP messages back to
them.
For IPv6 redirect messages, call ip6_redirect() directly with the output
interface argument set to the interface we received the packet from (as
it's the very interface we should build the exception on), otherwise the
new nexthop will be rejected. There's no such need for IPv4.
Tunnels can now export an encap_err_lookup() operation that indicates a
match. Pass the packet to the lookup function, and if the tunnel driver
reports a matching association, continue with regular ICMP error handling.
v2:
- Added newline between network and transport header sets in
__udp{4,6}_lib_err_encap() (David Miller)
- Removed redundant skb_reset_network_header(skb); in
__udp4_lib_err_encap()
- Removed redundant reassignment of iph in __udp4_lib_err_encap()
(Sabrina Dubroca)
- Edited comment to __udp{4,6}_lib_err_encap() to reflect the fact this
won't work with lwtunnels configured to use asymmetric ports. By the way,
it's VXLAN, not VxLAN (Jiri Benc)
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
Reviewed-by: Sabrina Dubroca <sd@queasysnail.net>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-11-08 11:19:14 +00:00
|
|
|
{
|
2021-07-20 20:35:28 +00:00
|
|
|
int (*lookup)(struct sock *sk, struct sk_buff *skb);
|
udp: Handle ICMP errors for tunnels with same destination port on both endpoints
For both IPv4 and IPv6, if we can't match errors to a socket, try
tunnels before ignoring them. Look up a socket with the original source
and destination ports as found in the UDP packet inside the ICMP payload,
this will work for tunnels that force the same destination port for both
endpoints, i.e. VXLAN and GENEVE.
Actually, lwtunnels could break this assumption if they are configured by
an external control plane to have different destination ports on the
endpoints: in this case, we won't be able to trace ICMP messages back to
them.
For IPv6 redirect messages, call ip6_redirect() directly with the output
interface argument set to the interface we received the packet from (as
it's the very interface we should build the exception on), otherwise the
new nexthop will be rejected. There's no such need for IPv4.
Tunnels can now export an encap_err_lookup() operation that indicates a
match. Pass the packet to the lookup function, and if the tunnel driver
reports a matching association, continue with regular ICMP error handling.
v2:
- Added newline between network and transport header sets in
__udp{4,6}_lib_err_encap() (David Miller)
- Removed redundant skb_reset_network_header(skb); in
__udp4_lib_err_encap()
- Removed redundant reassignment of iph in __udp4_lib_err_encap()
(Sabrina Dubroca)
- Edited comment to __udp{4,6}_lib_err_encap() to reflect the fact this
won't work with lwtunnels configured to use asymmetric ports. By the way,
it's VXLAN, not VxLAN (Jiri Benc)
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
Reviewed-by: Sabrina Dubroca <sd@queasysnail.net>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-11-08 11:19:14 +00:00
|
|
|
int network_offset, transport_offset;
|
2021-07-20 20:35:28 +00:00
|
|
|
struct udp_sock *up;
|
udp: Handle ICMP errors for tunnels with same destination port on both endpoints
For both IPv4 and IPv6, if we can't match errors to a socket, try
tunnels before ignoring them. Look up a socket with the original source
and destination ports as found in the UDP packet inside the ICMP payload,
this will work for tunnels that force the same destination port for both
endpoints, i.e. VXLAN and GENEVE.
Actually, lwtunnels could break this assumption if they are configured by
an external control plane to have different destination ports on the
endpoints: in this case, we won't be able to trace ICMP messages back to
them.
For IPv6 redirect messages, call ip6_redirect() directly with the output
interface argument set to the interface we received the packet from (as
it's the very interface we should build the exception on), otherwise the
new nexthop will be rejected. There's no such need for IPv4.
Tunnels can now export an encap_err_lookup() operation that indicates a
match. Pass the packet to the lookup function, and if the tunnel driver
reports a matching association, continue with regular ICMP error handling.
v2:
- Added newline between network and transport header sets in
__udp{4,6}_lib_err_encap() (David Miller)
- Removed redundant skb_reset_network_header(skb); in
__udp4_lib_err_encap()
- Removed redundant reassignment of iph in __udp4_lib_err_encap()
(Sabrina Dubroca)
- Edited comment to __udp{4,6}_lib_err_encap() to reflect the fact this
won't work with lwtunnels configured to use asymmetric ports. By the way,
it's VXLAN, not VxLAN (Jiri Benc)
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
Reviewed-by: Sabrina Dubroca <sd@queasysnail.net>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-11-08 11:19:14 +00:00
|
|
|
|
|
|
|
network_offset = skb_network_offset(skb);
|
|
|
|
transport_offset = skb_transport_offset(skb);
|
|
|
|
|
|
|
|
/* Network header needs to point to the outer IPv4 header inside ICMP */
|
|
|
|
skb_reset_network_header(skb);
|
|
|
|
|
|
|
|
/* Transport header needs to point to the UDP header */
|
|
|
|
skb_set_transport_header(skb, iph->ihl << 2);
|
|
|
|
|
2021-07-20 20:35:28 +00:00
|
|
|
if (sk) {
|
|
|
|
up = udp_sk(sk);
|
|
|
|
|
|
|
|
lookup = READ_ONCE(up->encap_err_lookup);
|
|
|
|
if (lookup && lookup(sk, skb))
|
|
|
|
sk = NULL;
|
|
|
|
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
udp: Support for error handlers of tunnels with arbitrary destination port
ICMP error handling is currently not possible for UDP tunnels not
employing a receiving socket with local destination port matching the
remote one, because we have no way to look them up.
Add an err_handler tunnel encapsulation operation that can be exported by
tunnels in order to pass the error to the protocol implementing the
encapsulation. We can't easily use a lookup function as we did for VXLAN
and GENEVE, as protocol error handlers, which would be in turn called by
implementations of this new operation, handle the errors themselves,
together with the tunnel lookup.
Without a socket, we can't be sure which encapsulation error handler is
the appropriate one: encapsulation handlers (the ones for FoU and GUE
introduced in the next patch, e.g.) will need to check the new error codes
returned by protocol handlers to figure out if errors match the given
encapsulation, and, in turn, report this error back, so that we can try
all of them in __udp{4,6}_lib_err_encap_no_sk() until we have a match.
v2:
- Name all arguments in err_handler prototypes (David Miller)
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
Reviewed-by: Sabrina Dubroca <sd@queasysnail.net>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-11-08 11:19:22 +00:00
|
|
|
sk = __udp4_lib_lookup(net, iph->daddr, uh->source,
|
|
|
|
iph->saddr, uh->dest, skb->dev->ifindex, 0,
|
|
|
|
udptable, NULL);
|
|
|
|
if (sk) {
|
2021-07-20 20:35:28 +00:00
|
|
|
up = udp_sk(sk);
|
udp: Support for error handlers of tunnels with arbitrary destination port
ICMP error handling is currently not possible for UDP tunnels not
employing a receiving socket with local destination port matching the
remote one, because we have no way to look them up.
Add an err_handler tunnel encapsulation operation that can be exported by
tunnels in order to pass the error to the protocol implementing the
encapsulation. We can't easily use a lookup function as we did for VXLAN
and GENEVE, as protocol error handlers, which would be in turn called by
implementations of this new operation, handle the errors themselves,
together with the tunnel lookup.
Without a socket, we can't be sure which encapsulation error handler is
the appropriate one: encapsulation handlers (the ones for FoU and GUE
introduced in the next patch, e.g.) will need to check the new error codes
returned by protocol handlers to figure out if errors match the given
encapsulation, and, in turn, report this error back, so that we can try
all of them in __udp{4,6}_lib_err_encap_no_sk() until we have a match.
v2:
- Name all arguments in err_handler prototypes (David Miller)
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
Reviewed-by: Sabrina Dubroca <sd@queasysnail.net>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-11-08 11:19:22 +00:00
|
|
|
|
|
|
|
lookup = READ_ONCE(up->encap_err_lookup);
|
|
|
|
if (!lookup || lookup(sk, skb))
|
|
|
|
sk = NULL;
|
|
|
|
}
|
|
|
|
|
2021-07-20 20:35:28 +00:00
|
|
|
out:
|
udp: Support for error handlers of tunnels with arbitrary destination port
ICMP error handling is currently not possible for UDP tunnels not
employing a receiving socket with local destination port matching the
remote one, because we have no way to look them up.
Add an err_handler tunnel encapsulation operation that can be exported by
tunnels in order to pass the error to the protocol implementing the
encapsulation. We can't easily use a lookup function as we did for VXLAN
and GENEVE, as protocol error handlers, which would be in turn called by
implementations of this new operation, handle the errors themselves,
together with the tunnel lookup.
Without a socket, we can't be sure which encapsulation error handler is
the appropriate one: encapsulation handlers (the ones for FoU and GUE
introduced in the next patch, e.g.) will need to check the new error codes
returned by protocol handlers to figure out if errors match the given
encapsulation, and, in turn, report this error back, so that we can try
all of them in __udp{4,6}_lib_err_encap_no_sk() until we have a match.
v2:
- Name all arguments in err_handler prototypes (David Miller)
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
Reviewed-by: Sabrina Dubroca <sd@queasysnail.net>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-11-08 11:19:22 +00:00
|
|
|
if (!sk)
|
|
|
|
sk = ERR_PTR(__udp4_lib_err_encap_no_sk(skb, info));
|
udp: Handle ICMP errors for tunnels with same destination port on both endpoints
For both IPv4 and IPv6, if we can't match errors to a socket, try
tunnels before ignoring them. Look up a socket with the original source
and destination ports as found in the UDP packet inside the ICMP payload,
this will work for tunnels that force the same destination port for both
endpoints, i.e. VXLAN and GENEVE.
Actually, lwtunnels could break this assumption if they are configured by
an external control plane to have different destination ports on the
endpoints: in this case, we won't be able to trace ICMP messages back to
them.
For IPv6 redirect messages, call ip6_redirect() directly with the output
interface argument set to the interface we received the packet from (as
it's the very interface we should build the exception on), otherwise the
new nexthop will be rejected. There's no such need for IPv4.
Tunnels can now export an encap_err_lookup() operation that indicates a
match. Pass the packet to the lookup function, and if the tunnel driver
reports a matching association, continue with regular ICMP error handling.
v2:
- Added newline between network and transport header sets in
__udp{4,6}_lib_err_encap() (David Miller)
- Removed redundant skb_reset_network_header(skb); in
__udp4_lib_err_encap()
- Removed redundant reassignment of iph in __udp4_lib_err_encap()
(Sabrina Dubroca)
- Edited comment to __udp{4,6}_lib_err_encap() to reflect the fact this
won't work with lwtunnels configured to use asymmetric ports. By the way,
it's VXLAN, not VxLAN (Jiri Benc)
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
Reviewed-by: Sabrina Dubroca <sd@queasysnail.net>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-11-08 11:19:14 +00:00
|
|
|
|
|
|
|
skb_set_transport_header(skb, transport_offset);
|
|
|
|
skb_set_network_header(skb, network_offset);
|
|
|
|
|
|
|
|
return sk;
|
|
|
|
}
|
|
|
|
|
2008-03-07 00:22:02 +00:00
|
|
|
/*
|
|
|
|
* This routine is called by the ICMP module when it gets some
|
|
|
|
* sort of error condition. If err < 0 then the socket should
|
|
|
|
* be closed and the error returned to the user. If err > 0
|
|
|
|
* it's just the icmp type << 8 | icmp code.
|
|
|
|
* Header points to the ip header of the error packet. We move
|
|
|
|
* on past this. Then (as it used to claim before adjustment)
|
|
|
|
* header points to the first 8 bytes of the udp header. We need
|
|
|
|
* to find the appropriate port.
|
|
|
|
*/
|
|
|
|
|
2018-11-08 11:19:21 +00:00
|
|
|
int __udp4_lib_err(struct sk_buff *skb, u32 info, struct udp_table *udptable)
|
2008-03-07 00:22:02 +00:00
|
|
|
{
|
|
|
|
struct inet_sock *inet;
|
2011-04-22 04:53:02 +00:00
|
|
|
const struct iphdr *iph = (const struct iphdr *)skb->data;
|
2009-07-17 00:26:32 +00:00
|
|
|
struct udphdr *uh = (struct udphdr *)(skb->data+(iph->ihl<<2));
|
2008-03-07 00:22:02 +00:00
|
|
|
const int type = icmp_hdr(skb)->type;
|
|
|
|
const int code = icmp_hdr(skb)->code;
|
udp: Handle ICMP errors for tunnels with same destination port on both endpoints
For both IPv4 and IPv6, if we can't match errors to a socket, try
tunnels before ignoring them. Look up a socket with the original source
and destination ports as found in the UDP packet inside the ICMP payload,
this will work for tunnels that force the same destination port for both
endpoints, i.e. VXLAN and GENEVE.
Actually, lwtunnels could break this assumption if they are configured by
an external control plane to have different destination ports on the
endpoints: in this case, we won't be able to trace ICMP messages back to
them.
For IPv6 redirect messages, call ip6_redirect() directly with the output
interface argument set to the interface we received the packet from (as
it's the very interface we should build the exception on), otherwise the
new nexthop will be rejected. There's no such need for IPv4.
Tunnels can now export an encap_err_lookup() operation that indicates a
match. Pass the packet to the lookup function, and if the tunnel driver
reports a matching association, continue with regular ICMP error handling.
v2:
- Added newline between network and transport header sets in
__udp{4,6}_lib_err_encap() (David Miller)
- Removed redundant skb_reset_network_header(skb); in
__udp4_lib_err_encap()
- Removed redundant reassignment of iph in __udp4_lib_err_encap()
(Sabrina Dubroca)
- Edited comment to __udp{4,6}_lib_err_encap() to reflect the fact this
won't work with lwtunnels configured to use asymmetric ports. By the way,
it's VXLAN, not VxLAN (Jiri Benc)
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
Reviewed-by: Sabrina Dubroca <sd@queasysnail.net>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-11-08 11:19:14 +00:00
|
|
|
bool tunnel = false;
|
2008-03-07 00:22:02 +00:00
|
|
|
struct sock *sk;
|
|
|
|
int harderr;
|
|
|
|
int err;
|
2008-07-15 06:01:40 +00:00
|
|
|
struct net *net = dev_net(skb->dev);
|
2008-03-07 00:22:02 +00:00
|
|
|
|
2008-07-15 06:01:40 +00:00
|
|
|
sk = __udp4_lib_lookup(net, iph->daddr, uh->dest,
|
2018-10-26 11:24:35 +00:00
|
|
|
iph->saddr, uh->source, skb->dev->ifindex,
|
|
|
|
inet_sdif(skb), udptable, NULL);
|
2021-07-20 20:35:28 +00:00
|
|
|
|
2023-09-12 09:17:28 +00:00
|
|
|
if (!sk || READ_ONCE(udp_sk(sk)->encap_type)) {
|
udp: Handle ICMP errors for tunnels with same destination port on both endpoints
For both IPv4 and IPv6, if we can't match errors to a socket, try
tunnels before ignoring them. Look up a socket with the original source
and destination ports as found in the UDP packet inside the ICMP payload,
this will work for tunnels that force the same destination port for both
endpoints, i.e. VXLAN and GENEVE.
Actually, lwtunnels could break this assumption if they are configured by
an external control plane to have different destination ports on the
endpoints: in this case, we won't be able to trace ICMP messages back to
them.
For IPv6 redirect messages, call ip6_redirect() directly with the output
interface argument set to the interface we received the packet from (as
it's the very interface we should build the exception on), otherwise the
new nexthop will be rejected. There's no such need for IPv4.
Tunnels can now export an encap_err_lookup() operation that indicates a
match. Pass the packet to the lookup function, and if the tunnel driver
reports a matching association, continue with regular ICMP error handling.
v2:
- Added newline between network and transport header sets in
__udp{4,6}_lib_err_encap() (David Miller)
- Removed redundant skb_reset_network_header(skb); in
__udp4_lib_err_encap()
- Removed redundant reassignment of iph in __udp4_lib_err_encap()
(Sabrina Dubroca)
- Edited comment to __udp{4,6}_lib_err_encap() to reflect the fact this
won't work with lwtunnels configured to use asymmetric ports. By the way,
it's VXLAN, not VxLAN (Jiri Benc)
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
Reviewed-by: Sabrina Dubroca <sd@queasysnail.net>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-11-08 11:19:14 +00:00
|
|
|
/* No socket for error: try tunnels before discarding */
|
udp: Support for error handlers of tunnels with arbitrary destination port
ICMP error handling is currently not possible for UDP tunnels not
employing a receiving socket with local destination port matching the
remote one, because we have no way to look them up.
Add an err_handler tunnel encapsulation operation that can be exported by
tunnels in order to pass the error to the protocol implementing the
encapsulation. We can't easily use a lookup function as we did for VXLAN
and GENEVE, as protocol error handlers, which would be in turn called by
implementations of this new operation, handle the errors themselves,
together with the tunnel lookup.
Without a socket, we can't be sure which encapsulation error handler is
the appropriate one: encapsulation handlers (the ones for FoU and GUE
introduced in the next patch, e.g.) will need to check the new error codes
returned by protocol handlers to figure out if errors match the given
encapsulation, and, in turn, report this error back, so that we can try
all of them in __udp{4,6}_lib_err_encap_no_sk() until we have a match.
v2:
- Name all arguments in err_handler prototypes (David Miller)
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
Reviewed-by: Sabrina Dubroca <sd@queasysnail.net>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-11-08 11:19:22 +00:00
|
|
|
if (static_branch_unlikely(&udp_encap_needed_key)) {
|
2021-07-20 20:35:28 +00:00
|
|
|
sk = __udp4_lib_err_encap(net, iph, uh, udptable, sk, skb,
|
udp: Support for error handlers of tunnels with arbitrary destination port
ICMP error handling is currently not possible for UDP tunnels not
employing a receiving socket with local destination port matching the
remote one, because we have no way to look them up.
Add an err_handler tunnel encapsulation operation that can be exported by
tunnels in order to pass the error to the protocol implementing the
encapsulation. We can't easily use a lookup function as we did for VXLAN
and GENEVE, as protocol error handlers, which would be in turn called by
implementations of this new operation, handle the errors themselves,
together with the tunnel lookup.
Without a socket, we can't be sure which encapsulation error handler is
the appropriate one: encapsulation handlers (the ones for FoU and GUE
introduced in the next patch, e.g.) will need to check the new error codes
returned by protocol handlers to figure out if errors match the given
encapsulation, and, in turn, report this error back, so that we can try
all of them in __udp{4,6}_lib_err_encap_no_sk() until we have a match.
v2:
- Name all arguments in err_handler prototypes (David Miller)
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
Reviewed-by: Sabrina Dubroca <sd@queasysnail.net>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-11-08 11:19:22 +00:00
|
|
|
info);
|
|
|
|
if (!sk)
|
|
|
|
return 0;
|
2021-07-20 20:35:28 +00:00
|
|
|
} else
|
|
|
|
sk = ERR_PTR(-ENOENT);
|
udp: Handle ICMP errors for tunnels with same destination port on both endpoints
For both IPv4 and IPv6, if we can't match errors to a socket, try
tunnels before ignoring them. Look up a socket with the original source
and destination ports as found in the UDP packet inside the ICMP payload,
this will work for tunnels that force the same destination port for both
endpoints, i.e. VXLAN and GENEVE.
Actually, lwtunnels could break this assumption if they are configured by
an external control plane to have different destination ports on the
endpoints: in this case, we won't be able to trace ICMP messages back to
them.
For IPv6 redirect messages, call ip6_redirect() directly with the output
interface argument set to the interface we received the packet from (as
it's the very interface we should build the exception on), otherwise the
new nexthop will be rejected. There's no such need for IPv4.
Tunnels can now export an encap_err_lookup() operation that indicates a
match. Pass the packet to the lookup function, and if the tunnel driver
reports a matching association, continue with regular ICMP error handling.
v2:
- Added newline between network and transport header sets in
__udp{4,6}_lib_err_encap() (David Miller)
- Removed redundant skb_reset_network_header(skb); in
__udp4_lib_err_encap()
- Removed redundant reassignment of iph in __udp4_lib_err_encap()
(Sabrina Dubroca)
- Edited comment to __udp{4,6}_lib_err_encap() to reflect the fact this
won't work with lwtunnels configured to use asymmetric ports. By the way,
it's VXLAN, not VxLAN (Jiri Benc)
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
Reviewed-by: Sabrina Dubroca <sd@queasysnail.net>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-11-08 11:19:14 +00:00
|
|
|
|
udp: Support for error handlers of tunnels with arbitrary destination port
ICMP error handling is currently not possible for UDP tunnels not
employing a receiving socket with local destination port matching the
remote one, because we have no way to look them up.
Add an err_handler tunnel encapsulation operation that can be exported by
tunnels in order to pass the error to the protocol implementing the
encapsulation. We can't easily use a lookup function as we did for VXLAN
and GENEVE, as protocol error handlers, which would be in turn called by
implementations of this new operation, handle the errors themselves,
together with the tunnel lookup.
Without a socket, we can't be sure which encapsulation error handler is
the appropriate one: encapsulation handlers (the ones for FoU and GUE
introduced in the next patch, e.g.) will need to check the new error codes
returned by protocol handlers to figure out if errors match the given
encapsulation, and, in turn, report this error back, so that we can try
all of them in __udp{4,6}_lib_err_encap_no_sk() until we have a match.
v2:
- Name all arguments in err_handler prototypes (David Miller)
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
Reviewed-by: Sabrina Dubroca <sd@queasysnail.net>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-11-08 11:19:22 +00:00
|
|
|
if (IS_ERR(sk)) {
|
udp: Handle ICMP errors for tunnels with same destination port on both endpoints
For both IPv4 and IPv6, if we can't match errors to a socket, try
tunnels before ignoring them. Look up a socket with the original source
and destination ports as found in the UDP packet inside the ICMP payload,
this will work for tunnels that force the same destination port for both
endpoints, i.e. VXLAN and GENEVE.
Actually, lwtunnels could break this assumption if they are configured by
an external control plane to have different destination ports on the
endpoints: in this case, we won't be able to trace ICMP messages back to
them.
For IPv6 redirect messages, call ip6_redirect() directly with the output
interface argument set to the interface we received the packet from (as
it's the very interface we should build the exception on), otherwise the
new nexthop will be rejected. There's no such need for IPv4.
Tunnels can now export an encap_err_lookup() operation that indicates a
match. Pass the packet to the lookup function, and if the tunnel driver
reports a matching association, continue with regular ICMP error handling.
v2:
- Added newline between network and transport header sets in
__udp{4,6}_lib_err_encap() (David Miller)
- Removed redundant skb_reset_network_header(skb); in
__udp4_lib_err_encap()
- Removed redundant reassignment of iph in __udp4_lib_err_encap()
(Sabrina Dubroca)
- Edited comment to __udp{4,6}_lib_err_encap() to reflect the fact this
won't work with lwtunnels configured to use asymmetric ports. By the way,
it's VXLAN, not VxLAN (Jiri Benc)
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
Reviewed-by: Sabrina Dubroca <sd@queasysnail.net>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-11-08 11:19:14 +00:00
|
|
|
__ICMP_INC_STATS(net, ICMP_MIB_INERRORS);
|
udp: Support for error handlers of tunnels with arbitrary destination port
ICMP error handling is currently not possible for UDP tunnels not
employing a receiving socket with local destination port matching the
remote one, because we have no way to look them up.
Add an err_handler tunnel encapsulation operation that can be exported by
tunnels in order to pass the error to the protocol implementing the
encapsulation. We can't easily use a lookup function as we did for VXLAN
and GENEVE, as protocol error handlers, which would be in turn called by
implementations of this new operation, handle the errors themselves,
together with the tunnel lookup.
Without a socket, we can't be sure which encapsulation error handler is
the appropriate one: encapsulation handlers (the ones for FoU and GUE
introduced in the next patch, e.g.) will need to check the new error codes
returned by protocol handlers to figure out if errors match the given
encapsulation, and, in turn, report this error back, so that we can try
all of them in __udp{4,6}_lib_err_encap_no_sk() until we have a match.
v2:
- Name all arguments in err_handler prototypes (David Miller)
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
Reviewed-by: Sabrina Dubroca <sd@queasysnail.net>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-11-08 11:19:22 +00:00
|
|
|
return PTR_ERR(sk);
|
udp: Handle ICMP errors for tunnels with same destination port on both endpoints
For both IPv4 and IPv6, if we can't match errors to a socket, try
tunnels before ignoring them. Look up a socket with the original source
and destination ports as found in the UDP packet inside the ICMP payload,
this will work for tunnels that force the same destination port for both
endpoints, i.e. VXLAN and GENEVE.
Actually, lwtunnels could break this assumption if they are configured by
an external control plane to have different destination ports on the
endpoints: in this case, we won't be able to trace ICMP messages back to
them.
For IPv6 redirect messages, call ip6_redirect() directly with the output
interface argument set to the interface we received the packet from (as
it's the very interface we should build the exception on), otherwise the
new nexthop will be rejected. There's no such need for IPv4.
Tunnels can now export an encap_err_lookup() operation that indicates a
match. Pass the packet to the lookup function, and if the tunnel driver
reports a matching association, continue with regular ICMP error handling.
v2:
- Added newline between network and transport header sets in
__udp{4,6}_lib_err_encap() (David Miller)
- Removed redundant skb_reset_network_header(skb); in
__udp4_lib_err_encap()
- Removed redundant reassignment of iph in __udp4_lib_err_encap()
(Sabrina Dubroca)
- Edited comment to __udp{4,6}_lib_err_encap() to reflect the fact this
won't work with lwtunnels configured to use asymmetric ports. By the way,
it's VXLAN, not VxLAN (Jiri Benc)
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
Reviewed-by: Sabrina Dubroca <sd@queasysnail.net>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-11-08 11:19:14 +00:00
|
|
|
}
|
udp: Support for error handlers of tunnels with arbitrary destination port
ICMP error handling is currently not possible for UDP tunnels not
employing a receiving socket with local destination port matching the
remote one, because we have no way to look them up.
Add an err_handler tunnel encapsulation operation that can be exported by
tunnels in order to pass the error to the protocol implementing the
encapsulation. We can't easily use a lookup function as we did for VXLAN
and GENEVE, as protocol error handlers, which would be in turn called by
implementations of this new operation, handle the errors themselves,
together with the tunnel lookup.
Without a socket, we can't be sure which encapsulation error handler is
the appropriate one: encapsulation handlers (the ones for FoU and GUE
introduced in the next patch, e.g.) will need to check the new error codes
returned by protocol handlers to figure out if errors match the given
encapsulation, and, in turn, report this error back, so that we can try
all of them in __udp{4,6}_lib_err_encap_no_sk() until we have a match.
v2:
- Name all arguments in err_handler prototypes (David Miller)
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
Reviewed-by: Sabrina Dubroca <sd@queasysnail.net>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-11-08 11:19:22 +00:00
|
|
|
|
udp: Handle ICMP errors for tunnels with same destination port on both endpoints
For both IPv4 and IPv6, if we can't match errors to a socket, try
tunnels before ignoring them. Look up a socket with the original source
and destination ports as found in the UDP packet inside the ICMP payload,
this will work for tunnels that force the same destination port for both
endpoints, i.e. VXLAN and GENEVE.
Actually, lwtunnels could break this assumption if they are configured by
an external control plane to have different destination ports on the
endpoints: in this case, we won't be able to trace ICMP messages back to
them.
For IPv6 redirect messages, call ip6_redirect() directly with the output
interface argument set to the interface we received the packet from (as
it's the very interface we should build the exception on), otherwise the
new nexthop will be rejected. There's no such need for IPv4.
Tunnels can now export an encap_err_lookup() operation that indicates a
match. Pass the packet to the lookup function, and if the tunnel driver
reports a matching association, continue with regular ICMP error handling.
v2:
- Added newline between network and transport header sets in
__udp{4,6}_lib_err_encap() (David Miller)
- Removed redundant skb_reset_network_header(skb); in
__udp4_lib_err_encap()
- Removed redundant reassignment of iph in __udp4_lib_err_encap()
(Sabrina Dubroca)
- Edited comment to __udp{4,6}_lib_err_encap() to reflect the fact this
won't work with lwtunnels configured to use asymmetric ports. By the way,
it's VXLAN, not VxLAN (Jiri Benc)
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
Reviewed-by: Sabrina Dubroca <sd@queasysnail.net>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-11-08 11:19:14 +00:00
|
|
|
tunnel = true;
|
2008-03-07 00:22:02 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
err = 0;
|
|
|
|
harderr = 0;
|
|
|
|
inet = inet_sk(sk);
|
|
|
|
|
|
|
|
switch (type) {
|
|
|
|
default:
|
|
|
|
case ICMP_TIME_EXCEEDED:
|
|
|
|
err = EHOSTUNREACH;
|
|
|
|
break;
|
|
|
|
case ICMP_SOURCE_QUENCH:
|
|
|
|
goto out;
|
|
|
|
case ICMP_PARAMETERPROB:
|
|
|
|
err = EPROTO;
|
|
|
|
harderr = 1;
|
|
|
|
break;
|
|
|
|
case ICMP_DEST_UNREACH:
|
|
|
|
if (code == ICMP_FRAG_NEEDED) { /* Path MTU discovery */
|
2012-06-15 05:21:46 +00:00
|
|
|
ipv4_sk_update_pmtu(skb, sk, info);
|
2023-09-22 03:42:15 +00:00
|
|
|
if (READ_ONCE(inet->pmtudisc) != IP_PMTUDISC_DONT) {
|
2008-03-07 00:22:02 +00:00
|
|
|
err = EMSGSIZE;
|
|
|
|
harderr = 1;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
err = EHOSTUNREACH;
|
|
|
|
if (code <= NR_ICMP_UNREACH) {
|
|
|
|
harderr = icmp_err_convert[code].fatal;
|
|
|
|
err = icmp_err_convert[code].errno;
|
|
|
|
}
|
|
|
|
break;
|
2012-07-12 04:27:49 +00:00
|
|
|
case ICMP_REDIRECT:
|
|
|
|
ipv4_sk_redirect(skb, sk);
|
2013-09-20 10:20:28 +00:00
|
|
|
goto out;
|
2008-03-07 00:22:02 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* RFC1122: OK. Passes ICMP errors back to application, as per
|
|
|
|
* 4.1.3.3.
|
|
|
|
*/
|
udp: Handle ICMP errors for tunnels with same destination port on both endpoints
For both IPv4 and IPv6, if we can't match errors to a socket, try
tunnels before ignoring them. Look up a socket with the original source
and destination ports as found in the UDP packet inside the ICMP payload,
this will work for tunnels that force the same destination port for both
endpoints, i.e. VXLAN and GENEVE.
Actually, lwtunnels could break this assumption if they are configured by
an external control plane to have different destination ports on the
endpoints: in this case, we won't be able to trace ICMP messages back to
them.
For IPv6 redirect messages, call ip6_redirect() directly with the output
interface argument set to the interface we received the packet from (as
it's the very interface we should build the exception on), otherwise the
new nexthop will be rejected. There's no such need for IPv4.
Tunnels can now export an encap_err_lookup() operation that indicates a
match. Pass the packet to the lookup function, and if the tunnel driver
reports a matching association, continue with regular ICMP error handling.
v2:
- Added newline between network and transport header sets in
__udp{4,6}_lib_err_encap() (David Miller)
- Removed redundant skb_reset_network_header(skb); in
__udp4_lib_err_encap()
- Removed redundant reassignment of iph in __udp4_lib_err_encap()
(Sabrina Dubroca)
- Edited comment to __udp{4,6}_lib_err_encap() to reflect the fact this
won't work with lwtunnels configured to use asymmetric ports. By the way,
it's VXLAN, not VxLAN (Jiri Benc)
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
Reviewed-by: Sabrina Dubroca <sd@queasysnail.net>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-11-08 11:19:14 +00:00
|
|
|
if (tunnel) {
|
|
|
|
/* ...not for tunnels though: we don't have a sending socket */
|
2022-08-26 14:39:28 +00:00
|
|
|
if (udp_sk(sk)->encap_err_rcv)
|
2022-10-12 07:49:29 +00:00
|
|
|
udp_sk(sk)->encap_err_rcv(sk, skb, err, uh->dest, info,
|
|
|
|
(u8 *)(uh+1));
|
udp: Handle ICMP errors for tunnels with same destination port on both endpoints
For both IPv4 and IPv6, if we can't match errors to a socket, try
tunnels before ignoring them. Look up a socket with the original source
and destination ports as found in the UDP packet inside the ICMP payload,
this will work for tunnels that force the same destination port for both
endpoints, i.e. VXLAN and GENEVE.
Actually, lwtunnels could break this assumption if they are configured by
an external control plane to have different destination ports on the
endpoints: in this case, we won't be able to trace ICMP messages back to
them.
For IPv6 redirect messages, call ip6_redirect() directly with the output
interface argument set to the interface we received the packet from (as
it's the very interface we should build the exception on), otherwise the
new nexthop will be rejected. There's no such need for IPv4.
Tunnels can now export an encap_err_lookup() operation that indicates a
match. Pass the packet to the lookup function, and if the tunnel driver
reports a matching association, continue with regular ICMP error handling.
v2:
- Added newline between network and transport header sets in
__udp{4,6}_lib_err_encap() (David Miller)
- Removed redundant skb_reset_network_header(skb); in
__udp4_lib_err_encap()
- Removed redundant reassignment of iph in __udp4_lib_err_encap()
(Sabrina Dubroca)
- Edited comment to __udp{4,6}_lib_err_encap() to reflect the fact this
won't work with lwtunnels configured to use asymmetric ports. By the way,
it's VXLAN, not VxLAN (Jiri Benc)
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
Reviewed-by: Sabrina Dubroca <sd@queasysnail.net>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-11-08 11:19:14 +00:00
|
|
|
goto out;
|
|
|
|
}
|
2023-08-16 08:15:35 +00:00
|
|
|
if (!inet_test_bit(RECVERR, sk)) {
|
2008-03-07 00:22:02 +00:00
|
|
|
if (!harderr || sk->sk_state != TCP_ESTABLISHED)
|
|
|
|
goto out;
|
2010-06-01 06:44:05 +00:00
|
|
|
} else
|
2009-07-17 00:26:32 +00:00
|
|
|
ip_icmp_error(sk, skb, err, uh->dest, info, (u8 *)(uh+1));
|
2010-06-01 06:44:05 +00:00
|
|
|
|
2008-03-07 00:22:02 +00:00
|
|
|
sk->sk_err = err;
|
2021-06-27 22:48:21 +00:00
|
|
|
sk_error_report(sk);
|
2008-03-07 00:22:02 +00:00
|
|
|
out:
|
2018-11-08 11:19:21 +00:00
|
|
|
return 0;
|
2008-03-07 00:22:02 +00:00
|
|
|
}
|
|
|
|
|
2018-11-08 11:19:21 +00:00
|
|
|
int udp_err(struct sk_buff *skb, u32 info)
|
2008-03-07 00:22:02 +00:00
|
|
|
{
|
2022-11-14 21:57:56 +00:00
|
|
|
return __udp4_lib_err(skb, info, dev_net(skb->dev)->ipv4.udp_table);
|
2008-03-07 00:22:02 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Throw away all pending data and cancel the corking. Socket is locked.
|
|
|
|
*/
|
2008-06-04 11:49:07 +00:00
|
|
|
void udp_flush_pending_frames(struct sock *sk)
|
2008-03-07 00:22:02 +00:00
|
|
|
{
|
|
|
|
struct udp_sock *up = udp_sk(sk);
|
|
|
|
|
|
|
|
if (up->pending) {
|
|
|
|
up->len = 0;
|
|
|
|
up->pending = 0;
|
|
|
|
ip_flush_pending_frames(sk);
|
|
|
|
}
|
|
|
|
}
|
2008-06-04 11:49:07 +00:00
|
|
|
EXPORT_SYMBOL(udp_flush_pending_frames);
|
2008-03-07 00:22:02 +00:00
|
|
|
|
|
|
|
/**
|
2011-03-01 02:36:48 +00:00
|
|
|
* udp4_hwcsum - handle outgoing HW checksumming
|
2008-03-07 00:22:02 +00:00
|
|
|
* @skb: sk_buff containing the filled-in UDP header
|
|
|
|
* (checksum field must be zeroed out)
|
2011-03-01 02:36:48 +00:00
|
|
|
* @src: source IP address
|
|
|
|
* @dst: destination IP address
|
2008-03-07 00:22:02 +00:00
|
|
|
*/
|
2013-07-25 16:12:18 +00:00
|
|
|
void udp4_hwcsum(struct sk_buff *skb, __be32 src, __be32 dst)
|
2008-03-07 00:22:02 +00:00
|
|
|
{
|
|
|
|
struct udphdr *uh = udp_hdr(skb);
|
2011-03-01 02:36:48 +00:00
|
|
|
int offset = skb_transport_offset(skb);
|
|
|
|
int len = skb->len - offset;
|
|
|
|
int hlen = len;
|
2008-03-07 00:22:02 +00:00
|
|
|
__wsum csum = 0;
|
|
|
|
|
2014-06-02 23:12:02 +00:00
|
|
|
if (!skb_has_frag_list(skb)) {
|
2008-03-07 00:22:02 +00:00
|
|
|
/*
|
|
|
|
* Only one fragment on the socket.
|
|
|
|
*/
|
|
|
|
skb->csum_start = skb_transport_header(skb) - skb->head;
|
|
|
|
skb->csum_offset = offsetof(struct udphdr, check);
|
2011-03-01 02:36:48 +00:00
|
|
|
uh->check = ~csum_tcpudp_magic(src, dst, len,
|
|
|
|
IPPROTO_UDP, 0);
|
2008-03-07 00:22:02 +00:00
|
|
|
} else {
|
2014-06-02 23:12:02 +00:00
|
|
|
struct sk_buff *frags;
|
|
|
|
|
2008-03-07 00:22:02 +00:00
|
|
|
/*
|
|
|
|
* HW-checksum won't work as there are two or more
|
|
|
|
* fragments on the socket so that all csums of sk_buffs
|
|
|
|
* should be together
|
|
|
|
*/
|
2014-06-02 23:12:02 +00:00
|
|
|
skb_walk_frags(skb, frags) {
|
2011-03-01 02:36:48 +00:00
|
|
|
csum = csum_add(csum, frags->csum);
|
|
|
|
hlen -= frags->len;
|
2014-06-02 23:12:02 +00:00
|
|
|
}
|
2008-03-07 00:22:02 +00:00
|
|
|
|
2011-03-01 02:36:48 +00:00
|
|
|
csum = skb_checksum(skb, offset, hlen, csum);
|
2008-03-07 00:22:02 +00:00
|
|
|
skb->ip_summed = CHECKSUM_NONE;
|
|
|
|
|
|
|
|
uh->check = csum_tcpudp_magic(src, dst, len, IPPROTO_UDP, csum);
|
|
|
|
if (uh->check == 0)
|
|
|
|
uh->check = CSUM_MANGLED_0;
|
|
|
|
}
|
|
|
|
}
|
2013-07-25 16:12:18 +00:00
|
|
|
EXPORT_SYMBOL_GPL(udp4_hwcsum);
|
2008-03-07 00:22:02 +00:00
|
|
|
|
2014-06-05 00:19:48 +00:00
|
|
|
/* Function to set UDP checksum for an IPv4 UDP packet. This is intended
|
|
|
|
* for the simple case like when setting the checksum for a UDP tunnel.
|
|
|
|
*/
|
|
|
|
void udp_set_csum(bool nocheck, struct sk_buff *skb,
|
|
|
|
__be32 saddr, __be32 daddr, int len)
|
|
|
|
{
|
|
|
|
struct udphdr *uh = udp_hdr(skb);
|
|
|
|
|
2016-02-11 20:48:04 +00:00
|
|
|
if (nocheck) {
|
2014-06-05 00:19:48 +00:00
|
|
|
uh->check = 0;
|
2016-02-11 20:48:04 +00:00
|
|
|
} else if (skb_is_gso(skb)) {
|
2014-06-05 00:19:48 +00:00
|
|
|
uh->check = ~udp_v4_check(len, saddr, daddr, 0);
|
2016-02-11 20:48:04 +00:00
|
|
|
} else if (skb->ip_summed == CHECKSUM_PARTIAL) {
|
|
|
|
uh->check = 0;
|
|
|
|
uh->check = udp_v4_check(len, saddr, daddr, lco_csum(skb));
|
|
|
|
if (uh->check == 0)
|
|
|
|
uh->check = CSUM_MANGLED_0;
|
2016-02-11 20:49:40 +00:00
|
|
|
} else {
|
2014-06-05 00:19:48 +00:00
|
|
|
skb->ip_summed = CHECKSUM_PARTIAL;
|
|
|
|
skb->csum_start = skb_transport_header(skb) - skb->head;
|
|
|
|
skb->csum_offset = offsetof(struct udphdr, check);
|
|
|
|
uh->check = ~udp_v4_check(len, saddr, daddr, 0);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(udp_set_csum);
|
|
|
|
|
udp: generate gso with UDP_SEGMENT
Support generic segmentation offload for udp datagrams. Callers can
concatenate and send at once the payload of multiple datagrams with
the same destination.
To set segment size, the caller sets socket option UDP_SEGMENT to the
length of each discrete payload. This value must be smaller than or
equal to the relevant MTU.
A follow-up patch adds cmsg UDP_SEGMENT to specify segment size on a
per send call basis.
Total byte length may then exceed MTU. If not an exact multiple of
segment size, the last segment will be shorter.
The implementation adds a gso_size field to the udp socket, ip(v6)
cmsg cookie and inet_cork structure to be able to set the value at
setsockopt or cmsg time and to work with both lockless and corked
paths.
Initial benchmark numbers show UDP GSO about as expensive as TCP GSO.
tcp tso
3197 MB/s 54232 msg/s 54232 calls/s
6,457,754,262 cycles
tcp gso
1765 MB/s 29939 msg/s 29939 calls/s
11,203,021,806 cycles
tcp without tso/gso *
739 MB/s 12548 msg/s 12548 calls/s
11,205,483,630 cycles
udp
876 MB/s 14873 msg/s 624666 calls/s
11,205,777,429 cycles
udp gso
2139 MB/s 36282 msg/s 36282 calls/s
11,204,374,561 cycles
[*] after reverting commit 0a6b2a1dc2a2
("tcp: switch to GSO being always on")
Measured total system cycles ('-a') for one core while pinning both
the network receive path and benchmark process to that core:
perf stat -a -C 12 -e cycles \
./udpgso_bench_tx -C 12 -4 -D "$DST" -l 4
Note the reduction in calls/s with GSO. Bytes per syscall drops
increases from 1470 to 61818.
Signed-off-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-04-26 17:42:17 +00:00
|
|
|
static int udp_send_skb(struct sk_buff *skb, struct flowi4 *fl4,
|
|
|
|
struct inet_cork *cork)
|
2008-03-07 00:22:02 +00:00
|
|
|
{
|
2011-03-01 02:36:48 +00:00
|
|
|
struct sock *sk = skb->sk;
|
2008-03-07 00:22:02 +00:00
|
|
|
struct inet_sock *inet = inet_sk(sk);
|
|
|
|
struct udphdr *uh;
|
2020-11-06 06:42:40 +00:00
|
|
|
int err;
|
2008-03-07 00:22:02 +00:00
|
|
|
int is_udplite = IS_UDPLITE(sk);
|
2011-03-01 02:36:48 +00:00
|
|
|
int offset = skb_transport_offset(skb);
|
|
|
|
int len = skb->len - offset;
|
2019-10-02 17:29:23 +00:00
|
|
|
int datalen = len - sizeof(*uh);
|
2008-03-07 00:22:02 +00:00
|
|
|
__wsum csum = 0;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Create a UDP header
|
|
|
|
*/
|
|
|
|
uh = udp_hdr(skb);
|
2011-03-01 02:36:48 +00:00
|
|
|
uh->source = inet->inet_sport;
|
2011-05-09 20:31:04 +00:00
|
|
|
uh->dest = fl4->fl4_dport;
|
2011-03-01 02:36:48 +00:00
|
|
|
uh->len = htons(len);
|
2008-03-07 00:22:02 +00:00
|
|
|
uh->check = 0;
|
|
|
|
|
udp: generate gso with UDP_SEGMENT
Support generic segmentation offload for udp datagrams. Callers can
concatenate and send at once the payload of multiple datagrams with
the same destination.
To set segment size, the caller sets socket option UDP_SEGMENT to the
length of each discrete payload. This value must be smaller than or
equal to the relevant MTU.
A follow-up patch adds cmsg UDP_SEGMENT to specify segment size on a
per send call basis.
Total byte length may then exceed MTU. If not an exact multiple of
segment size, the last segment will be shorter.
The implementation adds a gso_size field to the udp socket, ip(v6)
cmsg cookie and inet_cork structure to be able to set the value at
setsockopt or cmsg time and to work with both lockless and corked
paths.
Initial benchmark numbers show UDP GSO about as expensive as TCP GSO.
tcp tso
3197 MB/s 54232 msg/s 54232 calls/s
6,457,754,262 cycles
tcp gso
1765 MB/s 29939 msg/s 29939 calls/s
11,203,021,806 cycles
tcp without tso/gso *
739 MB/s 12548 msg/s 12548 calls/s
11,205,483,630 cycles
udp
876 MB/s 14873 msg/s 624666 calls/s
11,205,777,429 cycles
udp gso
2139 MB/s 36282 msg/s 36282 calls/s
11,204,374,561 cycles
[*] after reverting commit 0a6b2a1dc2a2
("tcp: switch to GSO being always on")
Measured total system cycles ('-a') for one core while pinning both
the network receive path and benchmark process to that core:
perf stat -a -C 12 -e cycles \
./udpgso_bench_tx -C 12 -4 -D "$DST" -l 4
Note the reduction in calls/s with GSO. Bytes per syscall drops
increases from 1470 to 61818.
Signed-off-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-04-26 17:42:17 +00:00
|
|
|
if (cork->gso_size) {
|
|
|
|
const int hlen = skb_network_header_len(skb) +
|
|
|
|
sizeof(struct udphdr);
|
|
|
|
|
2019-01-15 16:40:02 +00:00
|
|
|
if (hlen + cork->gso_size > cork->fragsize) {
|
|
|
|
kfree_skb(skb);
|
udp: generate gso with UDP_SEGMENT
Support generic segmentation offload for udp datagrams. Callers can
concatenate and send at once the payload of multiple datagrams with
the same destination.
To set segment size, the caller sets socket option UDP_SEGMENT to the
length of each discrete payload. This value must be smaller than or
equal to the relevant MTU.
A follow-up patch adds cmsg UDP_SEGMENT to specify segment size on a
per send call basis.
Total byte length may then exceed MTU. If not an exact multiple of
segment size, the last segment will be shorter.
The implementation adds a gso_size field to the udp socket, ip(v6)
cmsg cookie and inet_cork structure to be able to set the value at
setsockopt or cmsg time and to work with both lockless and corked
paths.
Initial benchmark numbers show UDP GSO about as expensive as TCP GSO.
tcp tso
3197 MB/s 54232 msg/s 54232 calls/s
6,457,754,262 cycles
tcp gso
1765 MB/s 29939 msg/s 29939 calls/s
11,203,021,806 cycles
tcp without tso/gso *
739 MB/s 12548 msg/s 12548 calls/s
11,205,483,630 cycles
udp
876 MB/s 14873 msg/s 624666 calls/s
11,205,777,429 cycles
udp gso
2139 MB/s 36282 msg/s 36282 calls/s
11,204,374,561 cycles
[*] after reverting commit 0a6b2a1dc2a2
("tcp: switch to GSO being always on")
Measured total system cycles ('-a') for one core while pinning both
the network receive path and benchmark process to that core:
perf stat -a -C 12 -e cycles \
./udpgso_bench_tx -C 12 -4 -D "$DST" -l 4
Note the reduction in calls/s with GSO. Bytes per syscall drops
increases from 1470 to 61818.
Signed-off-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-04-26 17:42:17 +00:00
|
|
|
return -EINVAL;
|
2019-01-15 16:40:02 +00:00
|
|
|
}
|
2021-12-08 10:03:33 +00:00
|
|
|
if (datalen > cork->gso_size * UDP_MAX_SEGMENTS) {
|
2019-01-15 16:40:02 +00:00
|
|
|
kfree_skb(skb);
|
udp: generate gso with UDP_SEGMENT
Support generic segmentation offload for udp datagrams. Callers can
concatenate and send at once the payload of multiple datagrams with
the same destination.
To set segment size, the caller sets socket option UDP_SEGMENT to the
length of each discrete payload. This value must be smaller than or
equal to the relevant MTU.
A follow-up patch adds cmsg UDP_SEGMENT to specify segment size on a
per send call basis.
Total byte length may then exceed MTU. If not an exact multiple of
segment size, the last segment will be shorter.
The implementation adds a gso_size field to the udp socket, ip(v6)
cmsg cookie and inet_cork structure to be able to set the value at
setsockopt or cmsg time and to work with both lockless and corked
paths.
Initial benchmark numbers show UDP GSO about as expensive as TCP GSO.
tcp tso
3197 MB/s 54232 msg/s 54232 calls/s
6,457,754,262 cycles
tcp gso
1765 MB/s 29939 msg/s 29939 calls/s
11,203,021,806 cycles
tcp without tso/gso *
739 MB/s 12548 msg/s 12548 calls/s
11,205,483,630 cycles
udp
876 MB/s 14873 msg/s 624666 calls/s
11,205,777,429 cycles
udp gso
2139 MB/s 36282 msg/s 36282 calls/s
11,204,374,561 cycles
[*] after reverting commit 0a6b2a1dc2a2
("tcp: switch to GSO being always on")
Measured total system cycles ('-a') for one core while pinning both
the network receive path and benchmark process to that core:
perf stat -a -C 12 -e cycles \
./udpgso_bench_tx -C 12 -4 -D "$DST" -l 4
Note the reduction in calls/s with GSO. Bytes per syscall drops
increases from 1470 to 61818.
Signed-off-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-04-26 17:42:17 +00:00
|
|
|
return -EINVAL;
|
2019-01-15 16:40:02 +00:00
|
|
|
}
|
|
|
|
if (sk->sk_no_check_tx) {
|
|
|
|
kfree_skb(skb);
|
2018-04-30 19:58:36 +00:00
|
|
|
return -EINVAL;
|
2019-01-15 16:40:02 +00:00
|
|
|
}
|
2018-05-22 15:34:39 +00:00
|
|
|
if (skb->ip_summed != CHECKSUM_PARTIAL || is_udplite ||
|
2019-01-15 16:40:02 +00:00
|
|
|
dst_xfrm(skb_dst(skb))) {
|
|
|
|
kfree_skb(skb);
|
udp: generate gso with UDP_SEGMENT
Support generic segmentation offload for udp datagrams. Callers can
concatenate and send at once the payload of multiple datagrams with
the same destination.
To set segment size, the caller sets socket option UDP_SEGMENT to the
length of each discrete payload. This value must be smaller than or
equal to the relevant MTU.
A follow-up patch adds cmsg UDP_SEGMENT to specify segment size on a
per send call basis.
Total byte length may then exceed MTU. If not an exact multiple of
segment size, the last segment will be shorter.
The implementation adds a gso_size field to the udp socket, ip(v6)
cmsg cookie and inet_cork structure to be able to set the value at
setsockopt or cmsg time and to work with both lockless and corked
paths.
Initial benchmark numbers show UDP GSO about as expensive as TCP GSO.
tcp tso
3197 MB/s 54232 msg/s 54232 calls/s
6,457,754,262 cycles
tcp gso
1765 MB/s 29939 msg/s 29939 calls/s
11,203,021,806 cycles
tcp without tso/gso *
739 MB/s 12548 msg/s 12548 calls/s
11,205,483,630 cycles
udp
876 MB/s 14873 msg/s 624666 calls/s
11,205,777,429 cycles
udp gso
2139 MB/s 36282 msg/s 36282 calls/s
11,204,374,561 cycles
[*] after reverting commit 0a6b2a1dc2a2
("tcp: switch to GSO being always on")
Measured total system cycles ('-a') for one core while pinning both
the network receive path and benchmark process to that core:
perf stat -a -C 12 -e cycles \
./udpgso_bench_tx -C 12 -4 -D "$DST" -l 4
Note the reduction in calls/s with GSO. Bytes per syscall drops
increases from 1470 to 61818.
Signed-off-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-04-26 17:42:17 +00:00
|
|
|
return -EIO;
|
2019-01-15 16:40:02 +00:00
|
|
|
}
|
udp: generate gso with UDP_SEGMENT
Support generic segmentation offload for udp datagrams. Callers can
concatenate and send at once the payload of multiple datagrams with
the same destination.
To set segment size, the caller sets socket option UDP_SEGMENT to the
length of each discrete payload. This value must be smaller than or
equal to the relevant MTU.
A follow-up patch adds cmsg UDP_SEGMENT to specify segment size on a
per send call basis.
Total byte length may then exceed MTU. If not an exact multiple of
segment size, the last segment will be shorter.
The implementation adds a gso_size field to the udp socket, ip(v6)
cmsg cookie and inet_cork structure to be able to set the value at
setsockopt or cmsg time and to work with both lockless and corked
paths.
Initial benchmark numbers show UDP GSO about as expensive as TCP GSO.
tcp tso
3197 MB/s 54232 msg/s 54232 calls/s
6,457,754,262 cycles
tcp gso
1765 MB/s 29939 msg/s 29939 calls/s
11,203,021,806 cycles
tcp without tso/gso *
739 MB/s 12548 msg/s 12548 calls/s
11,205,483,630 cycles
udp
876 MB/s 14873 msg/s 624666 calls/s
11,205,777,429 cycles
udp gso
2139 MB/s 36282 msg/s 36282 calls/s
11,204,374,561 cycles
[*] after reverting commit 0a6b2a1dc2a2
("tcp: switch to GSO being always on")
Measured total system cycles ('-a') for one core while pinning both
the network receive path and benchmark process to that core:
perf stat -a -C 12 -e cycles \
./udpgso_bench_tx -C 12 -4 -D "$DST" -l 4
Note the reduction in calls/s with GSO. Bytes per syscall drops
increases from 1470 to 61818.
Signed-off-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-04-26 17:42:17 +00:00
|
|
|
|
2019-10-02 17:29:23 +00:00
|
|
|
if (datalen > cork->gso_size) {
|
|
|
|
skb_shinfo(skb)->gso_size = cork->gso_size;
|
|
|
|
skb_shinfo(skb)->gso_type = SKB_GSO_UDP_L4;
|
|
|
|
skb_shinfo(skb)->gso_segs = DIV_ROUND_UP(datalen,
|
|
|
|
cork->gso_size);
|
|
|
|
}
|
2018-04-30 19:58:36 +00:00
|
|
|
goto csum_partial;
|
udp: generate gso with UDP_SEGMENT
Support generic segmentation offload for udp datagrams. Callers can
concatenate and send at once the payload of multiple datagrams with
the same destination.
To set segment size, the caller sets socket option UDP_SEGMENT to the
length of each discrete payload. This value must be smaller than or
equal to the relevant MTU.
A follow-up patch adds cmsg UDP_SEGMENT to specify segment size on a
per send call basis.
Total byte length may then exceed MTU. If not an exact multiple of
segment size, the last segment will be shorter.
The implementation adds a gso_size field to the udp socket, ip(v6)
cmsg cookie and inet_cork structure to be able to set the value at
setsockopt or cmsg time and to work with both lockless and corked
paths.
Initial benchmark numbers show UDP GSO about as expensive as TCP GSO.
tcp tso
3197 MB/s 54232 msg/s 54232 calls/s
6,457,754,262 cycles
tcp gso
1765 MB/s 29939 msg/s 29939 calls/s
11,203,021,806 cycles
tcp without tso/gso *
739 MB/s 12548 msg/s 12548 calls/s
11,205,483,630 cycles
udp
876 MB/s 14873 msg/s 624666 calls/s
11,205,777,429 cycles
udp gso
2139 MB/s 36282 msg/s 36282 calls/s
11,204,374,561 cycles
[*] after reverting commit 0a6b2a1dc2a2
("tcp: switch to GSO being always on")
Measured total system cycles ('-a') for one core while pinning both
the network receive path and benchmark process to that core:
perf stat -a -C 12 -e cycles \
./udpgso_bench_tx -C 12 -4 -D "$DST" -l 4
Note the reduction in calls/s with GSO. Bytes per syscall drops
increases from 1470 to 61818.
Signed-off-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-04-26 17:42:17 +00:00
|
|
|
}
|
|
|
|
|
2008-03-07 00:22:02 +00:00
|
|
|
if (is_udplite) /* UDP-Lite */
|
2011-03-01 02:36:48 +00:00
|
|
|
csum = udplite_csum(skb);
|
2008-03-07 00:22:02 +00:00
|
|
|
|
2017-08-22 15:39:57 +00:00
|
|
|
else if (sk->sk_no_check_tx) { /* UDP csum off */
|
2008-03-07 00:22:02 +00:00
|
|
|
|
|
|
|
skb->ip_summed = CHECKSUM_NONE;
|
|
|
|
goto send;
|
|
|
|
|
|
|
|
} else if (skb->ip_summed == CHECKSUM_PARTIAL) { /* UDP hardware csum */
|
2018-04-30 19:58:36 +00:00
|
|
|
csum_partial:
|
2008-03-07 00:22:02 +00:00
|
|
|
|
2011-05-09 20:31:04 +00:00
|
|
|
udp4_hwcsum(skb, fl4->saddr, fl4->daddr);
|
2008-03-07 00:22:02 +00:00
|
|
|
goto send;
|
|
|
|
|
2011-03-01 02:36:48 +00:00
|
|
|
} else
|
|
|
|
csum = udp_csum(skb);
|
2008-03-07 00:22:02 +00:00
|
|
|
|
|
|
|
/* add protocol-dependent pseudo-header */
|
2011-05-09 20:31:04 +00:00
|
|
|
uh->check = csum_tcpudp_magic(fl4->saddr, fl4->daddr, len,
|
2009-07-17 00:26:32 +00:00
|
|
|
sk->sk_protocol, csum);
|
2008-03-07 00:22:02 +00:00
|
|
|
if (uh->check == 0)
|
|
|
|
uh->check = CSUM_MANGLED_0;
|
|
|
|
|
|
|
|
send:
|
2012-08-10 02:22:47 +00:00
|
|
|
err = ip_send_skb(sock_net(sk), skb);
|
ip: Report qdisc packet drops
Christoph Lameter pointed out that packet drops at qdisc level where not
accounted in SNMP counters. Only if application sets IP_RECVERR, drops
are reported to user (-ENOBUFS errors) and SNMP counters updated.
IP_RECVERR is used to enable extended reliable error message passing,
but these are not needed to update system wide SNMP stats.
This patch changes things a bit to allow SNMP counters to be updated,
regardless of IP_RECVERR being set or not on the socket.
Example after an UDP tx flood
# netstat -s
...
IP:
1487048 outgoing packets dropped
...
Udp:
...
SndbufErrors: 1487048
send() syscalls, do however still return an OK status, to not
break applications.
Note : send() manual page explicitly says for -ENOBUFS error :
"The output queue for a network interface was full.
This generally indicates that the interface has stopped sending,
but may be caused by transient congestion.
(Normally, this does not occur in Linux. Packets are just silently
dropped when a device queue overflows.) "
This is not true for IP_RECVERR enabled sockets : a send() syscall
that hit a qdisc drop returns an ENOBUFS error.
Many thanks to Christoph, David, and last but not least, Alexey !
Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2009-09-03 01:05:33 +00:00
|
|
|
if (err) {
|
2023-08-16 08:15:35 +00:00
|
|
|
if (err == -ENOBUFS &&
|
|
|
|
!inet_test_bit(RECVERR, sk)) {
|
net: snmp: kill various STATS_USER() helpers
In the old days (before linux-3.0), SNMP counters were duplicated,
one for user context, and one for BH context.
After commit 8f0ea0fe3a03 ("snmp: reduce percpu needs by 50%")
we have a single copy, and what really matters is preemption being
enabled or disabled, since we use this_cpu_inc() or __this_cpu_inc()
respectively.
We therefore kill SNMP_INC_STATS_USER(), SNMP_ADD_STATS_USER(),
NET_INC_STATS_USER(), NET_ADD_STATS_USER(), SCTP_INC_STATS_USER(),
SNMP_INC_STATS64_USER(), SNMP_ADD_STATS64_USER(), TCP_ADD_STATS_USER(),
UDP_INC_STATS_USER(), UDP6_INC_STATS_USER(), and XFRM_INC_STATS_USER()
Following patches will rename __BH helpers to make clear their
usage is not tied to BH being disabled.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-04-27 23:44:27 +00:00
|
|
|
UDP_INC_STATS(sock_net(sk),
|
|
|
|
UDP_MIB_SNDBUFERRORS, is_udplite);
|
ip: Report qdisc packet drops
Christoph Lameter pointed out that packet drops at qdisc level where not
accounted in SNMP counters. Only if application sets IP_RECVERR, drops
are reported to user (-ENOBUFS errors) and SNMP counters updated.
IP_RECVERR is used to enable extended reliable error message passing,
but these are not needed to update system wide SNMP stats.
This patch changes things a bit to allow SNMP counters to be updated,
regardless of IP_RECVERR being set or not on the socket.
Example after an UDP tx flood
# netstat -s
...
IP:
1487048 outgoing packets dropped
...
Udp:
...
SndbufErrors: 1487048
send() syscalls, do however still return an OK status, to not
break applications.
Note : send() manual page explicitly says for -ENOBUFS error :
"The output queue for a network interface was full.
This generally indicates that the interface has stopped sending,
but may be caused by transient congestion.
(Normally, this does not occur in Linux. Packets are just silently
dropped when a device queue overflows.) "
This is not true for IP_RECVERR enabled sockets : a send() syscall
that hit a qdisc drop returns an ENOBUFS error.
Many thanks to Christoph, David, and last but not least, Alexey !
Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2009-09-03 01:05:33 +00:00
|
|
|
err = 0;
|
|
|
|
}
|
|
|
|
} else
|
net: snmp: kill various STATS_USER() helpers
In the old days (before linux-3.0), SNMP counters were duplicated,
one for user context, and one for BH context.
After commit 8f0ea0fe3a03 ("snmp: reduce percpu needs by 50%")
we have a single copy, and what really matters is preemption being
enabled or disabled, since we use this_cpu_inc() or __this_cpu_inc()
respectively.
We therefore kill SNMP_INC_STATS_USER(), SNMP_ADD_STATS_USER(),
NET_INC_STATS_USER(), NET_ADD_STATS_USER(), SCTP_INC_STATS_USER(),
SNMP_INC_STATS64_USER(), SNMP_ADD_STATS64_USER(), TCP_ADD_STATS_USER(),
UDP_INC_STATS_USER(), UDP6_INC_STATS_USER(), and XFRM_INC_STATS_USER()
Following patches will rename __BH helpers to make clear their
usage is not tied to BH being disabled.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-04-27 23:44:27 +00:00
|
|
|
UDP_INC_STATS(sock_net(sk),
|
|
|
|
UDP_MIB_OUTDATAGRAMS, is_udplite);
|
2011-03-01 02:36:48 +00:00
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Push out all pending data as one UDP datagram. Socket is locked.
|
|
|
|
*/
|
2013-07-01 18:21:30 +00:00
|
|
|
int udp_push_pending_frames(struct sock *sk)
|
2011-03-01 02:36:48 +00:00
|
|
|
{
|
|
|
|
struct udp_sock *up = udp_sk(sk);
|
|
|
|
struct inet_sock *inet = inet_sk(sk);
|
2011-03-12 07:09:18 +00:00
|
|
|
struct flowi4 *fl4 = &inet->cork.fl.u.ip4;
|
2011-03-01 02:36:48 +00:00
|
|
|
struct sk_buff *skb;
|
|
|
|
int err = 0;
|
|
|
|
|
2011-05-09 00:12:19 +00:00
|
|
|
skb = ip_finish_skb(sk, fl4);
|
2011-03-01 02:36:48 +00:00
|
|
|
if (!skb)
|
|
|
|
goto out;
|
|
|
|
|
udp: generate gso with UDP_SEGMENT
Support generic segmentation offload for udp datagrams. Callers can
concatenate and send at once the payload of multiple datagrams with
the same destination.
To set segment size, the caller sets socket option UDP_SEGMENT to the
length of each discrete payload. This value must be smaller than or
equal to the relevant MTU.
A follow-up patch adds cmsg UDP_SEGMENT to specify segment size on a
per send call basis.
Total byte length may then exceed MTU. If not an exact multiple of
segment size, the last segment will be shorter.
The implementation adds a gso_size field to the udp socket, ip(v6)
cmsg cookie and inet_cork structure to be able to set the value at
setsockopt or cmsg time and to work with both lockless and corked
paths.
Initial benchmark numbers show UDP GSO about as expensive as TCP GSO.
tcp tso
3197 MB/s 54232 msg/s 54232 calls/s
6,457,754,262 cycles
tcp gso
1765 MB/s 29939 msg/s 29939 calls/s
11,203,021,806 cycles
tcp without tso/gso *
739 MB/s 12548 msg/s 12548 calls/s
11,205,483,630 cycles
udp
876 MB/s 14873 msg/s 624666 calls/s
11,205,777,429 cycles
udp gso
2139 MB/s 36282 msg/s 36282 calls/s
11,204,374,561 cycles
[*] after reverting commit 0a6b2a1dc2a2
("tcp: switch to GSO being always on")
Measured total system cycles ('-a') for one core while pinning both
the network receive path and benchmark process to that core:
perf stat -a -C 12 -e cycles \
./udpgso_bench_tx -C 12 -4 -D "$DST" -l 4
Note the reduction in calls/s with GSO. Bytes per syscall drops
increases from 1470 to 61818.
Signed-off-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-04-26 17:42:17 +00:00
|
|
|
err = udp_send_skb(skb, fl4, &inet->cork.base);
|
2011-03-01 02:36:48 +00:00
|
|
|
|
2008-03-07 00:22:02 +00:00
|
|
|
out:
|
|
|
|
up->len = 0;
|
|
|
|
up->pending = 0;
|
|
|
|
return err;
|
|
|
|
}
|
2013-07-01 18:21:30 +00:00
|
|
|
EXPORT_SYMBOL(udp_push_pending_frames);
|
2008-03-07 00:22:02 +00:00
|
|
|
|
2018-04-26 17:42:20 +00:00
|
|
|
static int __udp_cmsg_send(struct cmsghdr *cmsg, u16 *gso_size)
|
|
|
|
{
|
|
|
|
switch (cmsg->cmsg_type) {
|
|
|
|
case UDP_SEGMENT:
|
|
|
|
if (cmsg->cmsg_len != CMSG_LEN(sizeof(__u16)))
|
|
|
|
return -EINVAL;
|
|
|
|
*gso_size = *(__u16 *)CMSG_DATA(cmsg);
|
|
|
|
return 0;
|
|
|
|
default:
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
int udp_cmsg_send(struct sock *sk, struct msghdr *msg, u16 *gso_size)
|
|
|
|
{
|
|
|
|
struct cmsghdr *cmsg;
|
|
|
|
bool need_ip = false;
|
|
|
|
int err;
|
|
|
|
|
|
|
|
for_each_cmsghdr(cmsg, msg) {
|
|
|
|
if (!CMSG_OK(msg, cmsg))
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
if (cmsg->cmsg_level != SOL_UDP) {
|
|
|
|
need_ip = true;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
err = __udp_cmsg_send(cmsg, gso_size);
|
|
|
|
if (err)
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
|
|
|
return need_ip;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(udp_cmsg_send);
|
|
|
|
|
2015-03-02 07:37:48 +00:00
|
|
|
int udp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)
|
2008-03-07 00:22:02 +00:00
|
|
|
{
|
|
|
|
struct inet_sock *inet = inet_sk(sk);
|
|
|
|
struct udp_sock *up = udp_sk(sk);
|
2018-05-25 15:55:23 +00:00
|
|
|
DECLARE_SOCKADDR(struct sockaddr_in *, usin, msg->msg_name);
|
2011-05-08 23:38:45 +00:00
|
|
|
struct flowi4 fl4_stack;
|
2011-03-12 07:09:18 +00:00
|
|
|
struct flowi4 *fl4;
|
2008-03-07 00:22:02 +00:00
|
|
|
int ulen = len;
|
|
|
|
struct ipcm_cookie ipc;
|
|
|
|
struct rtable *rt = NULL;
|
|
|
|
int free = 0;
|
|
|
|
int connected = 0;
|
|
|
|
__be32 daddr, faddr, saddr;
|
2023-05-22 14:38:07 +00:00
|
|
|
u8 tos, scope;
|
2008-03-07 00:22:02 +00:00
|
|
|
__be16 dport;
|
|
|
|
int err, is_udplite = IS_UDPLITE(sk);
|
2023-09-12 09:17:21 +00:00
|
|
|
int corkreq = udp_test_bit(CORK, sk) || msg->msg_flags & MSG_MORE;
|
2008-03-07 00:22:02 +00:00
|
|
|
int (*getfrag)(void *, char *, int, int, int, struct sk_buff *);
|
2011-03-01 02:36:48 +00:00
|
|
|
struct sk_buff *skb;
|
2011-04-21 09:45:37 +00:00
|
|
|
struct ip_options_data opt_copy;
|
2023-09-22 03:42:19 +00:00
|
|
|
int uc_index;
|
2008-03-07 00:22:02 +00:00
|
|
|
|
|
|
|
if (len > 0xFFFF)
|
|
|
|
return -EMSGSIZE;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Check the flags.
|
|
|
|
*/
|
|
|
|
|
2009-07-17 00:26:32 +00:00
|
|
|
if (msg->msg_flags & MSG_OOB) /* Mirror BSD error message compatibility */
|
2008-03-07 00:22:02 +00:00
|
|
|
return -EOPNOTSUPP;
|
|
|
|
|
2011-03-01 02:36:48 +00:00
|
|
|
getfrag = is_udplite ? udplite_getfrag : ip_generic_getfrag;
|
|
|
|
|
2011-05-09 00:24:10 +00:00
|
|
|
fl4 = &inet->cork.fl.u.ip4;
|
2008-03-07 00:22:02 +00:00
|
|
|
if (up->pending) {
|
|
|
|
/*
|
|
|
|
* There are pending frames.
|
|
|
|
* The socket lock must be held while it's corked.
|
|
|
|
*/
|
|
|
|
lock_sock(sk);
|
|
|
|
if (likely(up->pending)) {
|
|
|
|
if (unlikely(up->pending != AF_INET)) {
|
|
|
|
release_sock(sk);
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
goto do_append_data;
|
|
|
|
}
|
|
|
|
release_sock(sk);
|
|
|
|
}
|
|
|
|
ulen += sizeof(struct udphdr);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Get and verify the address.
|
|
|
|
*/
|
2018-05-25 15:55:23 +00:00
|
|
|
if (usin) {
|
2008-03-07 00:22:02 +00:00
|
|
|
if (msg->msg_namelen < sizeof(*usin))
|
|
|
|
return -EINVAL;
|
|
|
|
if (usin->sin_family != AF_INET) {
|
|
|
|
if (usin->sin_family != AF_UNSPEC)
|
|
|
|
return -EAFNOSUPPORT;
|
|
|
|
}
|
|
|
|
|
|
|
|
daddr = usin->sin_addr.s_addr;
|
|
|
|
dport = usin->sin_port;
|
|
|
|
if (dport == 0)
|
|
|
|
return -EINVAL;
|
|
|
|
} else {
|
|
|
|
if (sk->sk_state != TCP_ESTABLISHED)
|
|
|
|
return -EDESTADDRREQ;
|
2009-10-15 06:30:45 +00:00
|
|
|
daddr = inet->inet_daddr;
|
|
|
|
dport = inet->inet_dport;
|
2008-03-07 00:22:02 +00:00
|
|
|
/* Open fast path for connected socket.
|
|
|
|
Route will not be used, if at least one option is set.
|
|
|
|
*/
|
|
|
|
connected = 1;
|
|
|
|
}
|
|
|
|
|
2018-07-06 14:12:54 +00:00
|
|
|
ipcm_init_sk(&ipc, inet);
|
2021-06-30 16:42:44 +00:00
|
|
|
ipc.gso_size = READ_ONCE(up->gso_size);
|
2013-04-14 08:08:13 +00:00
|
|
|
|
2008-03-07 00:22:02 +00:00
|
|
|
if (msg->msg_controllen) {
|
2018-04-26 17:42:20 +00:00
|
|
|
err = udp_cmsg_send(sk, msg, &ipc.gso_size);
|
|
|
|
if (err > 0)
|
|
|
|
err = ip_cmsg_send(sk, msg, &ipc,
|
|
|
|
sk->sk_family == AF_INET6);
|
|
|
|
if (unlikely(err < 0)) {
|
2016-02-04 14:23:28 +00:00
|
|
|
kfree(ipc.opt);
|
2008-03-07 00:22:02 +00:00
|
|
|
return err;
|
2016-02-04 14:23:28 +00:00
|
|
|
}
|
2008-03-07 00:22:02 +00:00
|
|
|
if (ipc.opt)
|
|
|
|
free = 1;
|
|
|
|
connected = 0;
|
|
|
|
}
|
2011-04-21 09:45:37 +00:00
|
|
|
if (!ipc.opt) {
|
|
|
|
struct ip_options_rcu *inet_opt;
|
|
|
|
|
|
|
|
rcu_read_lock();
|
|
|
|
inet_opt = rcu_dereference(inet->inet_opt);
|
|
|
|
if (inet_opt) {
|
|
|
|
memcpy(&opt_copy, inet_opt,
|
|
|
|
sizeof(*inet_opt) + inet_opt->opt.optlen);
|
|
|
|
ipc.opt = &opt_copy.opt;
|
|
|
|
}
|
|
|
|
rcu_read_unlock();
|
|
|
|
}
|
2008-03-07 00:22:02 +00:00
|
|
|
|
2021-08-19 09:24:20 +00:00
|
|
|
if (cgroup_bpf_enabled(CGROUP_UDP4_SENDMSG) && !connected) {
|
2018-05-25 15:55:23 +00:00
|
|
|
err = BPF_CGROUP_RUN_PROG_UDP4_SENDMSG_LOCK(sk,
|
2023-10-11 18:51:04 +00:00
|
|
|
(struct sockaddr *)usin,
|
|
|
|
&msg->msg_namelen,
|
|
|
|
&ipc.addr);
|
2018-05-25 15:55:23 +00:00
|
|
|
if (err)
|
|
|
|
goto out_free;
|
|
|
|
if (usin) {
|
|
|
|
if (usin->sin_port == 0) {
|
|
|
|
/* BPF program set invalid port. Reject it. */
|
|
|
|
err = -EINVAL;
|
|
|
|
goto out_free;
|
|
|
|
}
|
|
|
|
daddr = usin->sin_addr.s_addr;
|
|
|
|
dport = usin->sin_port;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2008-03-07 00:22:02 +00:00
|
|
|
saddr = ipc.addr;
|
|
|
|
ipc.addr = faddr = daddr;
|
|
|
|
|
2011-04-21 09:45:37 +00:00
|
|
|
if (ipc.opt && ipc.opt->opt.srr) {
|
2018-05-10 17:59:34 +00:00
|
|
|
if (!daddr) {
|
|
|
|
err = -EINVAL;
|
|
|
|
goto out_free;
|
|
|
|
}
|
2011-04-21 09:45:37 +00:00
|
|
|
faddr = ipc.opt->opt.faddr;
|
2008-03-07 00:22:02 +00:00
|
|
|
connected = 0;
|
|
|
|
}
|
2013-09-24 13:43:09 +00:00
|
|
|
tos = get_rttos(&ipc, inet);
|
2023-05-22 14:38:07 +00:00
|
|
|
scope = ip_sendmsg_scope(inet, &ipc, msg);
|
|
|
|
if (scope == RT_SCOPE_LINK)
|
2008-03-07 00:22:02 +00:00
|
|
|
connected = 0;
|
|
|
|
|
2023-09-22 03:42:19 +00:00
|
|
|
uc_index = READ_ONCE(inet->uc_index);
|
2008-03-07 00:22:02 +00:00
|
|
|
if (ipv4_is_multicast(daddr)) {
|
2018-10-01 08:40:23 +00:00
|
|
|
if (!ipc.oif || netif_index_is_l3_master(sock_net(sk), ipc.oif))
|
2023-09-22 03:42:21 +00:00
|
|
|
ipc.oif = READ_ONCE(inet->mc_index);
|
2008-03-07 00:22:02 +00:00
|
|
|
if (!saddr)
|
2023-09-22 03:42:21 +00:00
|
|
|
saddr = READ_ONCE(inet->mc_addr);
|
2008-03-07 00:22:02 +00:00
|
|
|
connected = 0;
|
2018-01-25 03:37:38 +00:00
|
|
|
} else if (!ipc.oif) {
|
2023-09-22 03:42:19 +00:00
|
|
|
ipc.oif = uc_index;
|
|
|
|
} else if (ipv4_is_lbcast(daddr) && uc_index) {
|
2018-01-25 03:37:38 +00:00
|
|
|
/* oif is set, packet is to local broadcast and
|
2020-08-22 23:31:41 +00:00
|
|
|
* uc_index is set. oif is most likely set
|
2018-01-25 03:37:38 +00:00
|
|
|
* by sk_bound_dev_if. If uc_index != oif check if the
|
|
|
|
* oif is an L3 master and uc_index is an L3 slave.
|
|
|
|
* If so, we want to allow the send using the uc_index.
|
|
|
|
*/
|
2023-09-22 03:42:19 +00:00
|
|
|
if (ipc.oif != uc_index &&
|
2018-01-25 03:37:38 +00:00
|
|
|
ipc.oif == l3mdev_master_ifindex_by_index(sock_net(sk),
|
2023-09-22 03:42:19 +00:00
|
|
|
uc_index)) {
|
|
|
|
ipc.oif = uc_index;
|
2018-01-25 03:37:38 +00:00
|
|
|
}
|
|
|
|
}
|
2008-03-07 00:22:02 +00:00
|
|
|
|
|
|
|
if (connected)
|
2009-07-17 00:26:32 +00:00
|
|
|
rt = (struct rtable *)sk_dst_check(sk, 0);
|
2008-03-07 00:22:02 +00:00
|
|
|
|
2015-04-03 08:17:26 +00:00
|
|
|
if (!rt) {
|
2008-07-17 03:19:08 +00:00
|
|
|
struct net *net = sock_net(sk);
|
2015-08-13 20:59:03 +00:00
|
|
|
__u8 flow_flags = inet_sk_flowi_flags(sk);
|
2008-07-17 03:19:08 +00:00
|
|
|
|
2011-05-08 23:38:45 +00:00
|
|
|
fl4 = &fl4_stack;
|
2015-08-13 20:59:03 +00:00
|
|
|
|
2023-05-22 14:38:07 +00:00
|
|
|
flowi4_init_output(fl4, ipc.oif, ipc.sockc.mark, tos, scope,
|
|
|
|
sk->sk_protocol, flow_flags, faddr, saddr,
|
|
|
|
dport, inet->inet_sport, sk->sk_uid);
|
2011-03-31 11:54:27 +00:00
|
|
|
|
2020-09-28 02:38:26 +00:00
|
|
|
security_sk_classify_flow(sk, flowi4_to_flowi_common(fl4));
|
2011-05-08 23:38:45 +00:00
|
|
|
rt = ip_route_output_flow(net, fl4, sk);
|
2011-03-02 22:31:35 +00:00
|
|
|
if (IS_ERR(rt)) {
|
|
|
|
err = PTR_ERR(rt);
|
2011-03-03 18:38:01 +00:00
|
|
|
rt = NULL;
|
2008-03-07 00:22:02 +00:00
|
|
|
if (err == -ENETUNREACH)
|
2013-11-28 17:51:22 +00:00
|
|
|
IP_INC_STATS(net, IPSTATS_MIB_OUTNOROUTES);
|
2008-03-07 00:22:02 +00:00
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
|
|
|
err = -EACCES;
|
|
|
|
if ((rt->rt_flags & RTCF_BROADCAST) &&
|
|
|
|
!sock_flag(sk, SOCK_BROADCAST))
|
|
|
|
goto out;
|
|
|
|
if (connected)
|
2010-06-11 06:31:35 +00:00
|
|
|
sk_dst_set(sk, dst_clone(&rt->dst));
|
2008-03-07 00:22:02 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if (msg->msg_flags&MSG_CONFIRM)
|
|
|
|
goto do_confirm;
|
|
|
|
back_from_confirm:
|
|
|
|
|
2011-05-08 23:38:45 +00:00
|
|
|
saddr = fl4->saddr;
|
2008-03-07 00:22:02 +00:00
|
|
|
if (!ipc.addr)
|
2011-05-08 23:38:45 +00:00
|
|
|
daddr = ipc.addr = fl4->daddr;
|
2008-03-07 00:22:02 +00:00
|
|
|
|
2011-03-01 02:36:48 +00:00
|
|
|
/* Lockless fast path for the non-corking case. */
|
|
|
|
if (!corkreq) {
|
2018-04-26 17:42:15 +00:00
|
|
|
struct inet_cork cork;
|
|
|
|
|
2014-11-24 18:23:40 +00:00
|
|
|
skb = ip_make_skb(sk, fl4, getfrag, msg, ulen,
|
2011-03-01 02:36:48 +00:00
|
|
|
sizeof(struct udphdr), &ipc, &rt,
|
2018-04-26 17:42:15 +00:00
|
|
|
&cork, msg->msg_flags);
|
2011-03-01 02:36:48 +00:00
|
|
|
err = PTR_ERR(skb);
|
2013-01-22 06:32:49 +00:00
|
|
|
if (!IS_ERR_OR_NULL(skb))
|
udp: generate gso with UDP_SEGMENT
Support generic segmentation offload for udp datagrams. Callers can
concatenate and send at once the payload of multiple datagrams with
the same destination.
To set segment size, the caller sets socket option UDP_SEGMENT to the
length of each discrete payload. This value must be smaller than or
equal to the relevant MTU.
A follow-up patch adds cmsg UDP_SEGMENT to specify segment size on a
per send call basis.
Total byte length may then exceed MTU. If not an exact multiple of
segment size, the last segment will be shorter.
The implementation adds a gso_size field to the udp socket, ip(v6)
cmsg cookie and inet_cork structure to be able to set the value at
setsockopt or cmsg time and to work with both lockless and corked
paths.
Initial benchmark numbers show UDP GSO about as expensive as TCP GSO.
tcp tso
3197 MB/s 54232 msg/s 54232 calls/s
6,457,754,262 cycles
tcp gso
1765 MB/s 29939 msg/s 29939 calls/s
11,203,021,806 cycles
tcp without tso/gso *
739 MB/s 12548 msg/s 12548 calls/s
11,205,483,630 cycles
udp
876 MB/s 14873 msg/s 624666 calls/s
11,205,777,429 cycles
udp gso
2139 MB/s 36282 msg/s 36282 calls/s
11,204,374,561 cycles
[*] after reverting commit 0a6b2a1dc2a2
("tcp: switch to GSO being always on")
Measured total system cycles ('-a') for one core while pinning both
the network receive path and benchmark process to that core:
perf stat -a -C 12 -e cycles \
./udpgso_bench_tx -C 12 -4 -D "$DST" -l 4
Note the reduction in calls/s with GSO. Bytes per syscall drops
increases from 1470 to 61818.
Signed-off-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-04-26 17:42:17 +00:00
|
|
|
err = udp_send_skb(skb, fl4, &cork);
|
2011-03-01 02:36:48 +00:00
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
2008-03-07 00:22:02 +00:00
|
|
|
lock_sock(sk);
|
|
|
|
if (unlikely(up->pending)) {
|
|
|
|
/* The socket is already corked while preparing it. */
|
|
|
|
/* ... which is an evident application bug. --ANK */
|
|
|
|
release_sock(sk);
|
|
|
|
|
2017-10-19 12:22:17 +00:00
|
|
|
net_dbg_ratelimited("socket already corked\n");
|
2008-03-07 00:22:02 +00:00
|
|
|
err = -EINVAL;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
/*
|
|
|
|
* Now cork the socket to pend data.
|
|
|
|
*/
|
2011-03-12 07:09:18 +00:00
|
|
|
fl4 = &inet->cork.fl.u.ip4;
|
|
|
|
fl4->daddr = daddr;
|
|
|
|
fl4->saddr = saddr;
|
2011-03-12 08:00:33 +00:00
|
|
|
fl4->fl4_dport = dport;
|
|
|
|
fl4->fl4_sport = inet->inet_sport;
|
2008-03-07 00:22:02 +00:00
|
|
|
up->pending = AF_INET;
|
|
|
|
|
|
|
|
do_append_data:
|
|
|
|
up->len += ulen;
|
2014-11-24 18:23:40 +00:00
|
|
|
err = ip_append_data(sk, fl4, getfrag, msg, ulen,
|
2011-05-09 00:24:10 +00:00
|
|
|
sizeof(struct udphdr), &ipc, &rt,
|
|
|
|
corkreq ? msg->msg_flags|MSG_MORE : msg->msg_flags);
|
2008-03-07 00:22:02 +00:00
|
|
|
if (err)
|
|
|
|
udp_flush_pending_frames(sk);
|
|
|
|
else if (!corkreq)
|
|
|
|
err = udp_push_pending_frames(sk);
|
|
|
|
else if (unlikely(skb_queue_empty(&sk->sk_write_queue)))
|
|
|
|
up->pending = 0;
|
|
|
|
release_sock(sk);
|
|
|
|
|
|
|
|
out:
|
|
|
|
ip_rt_put(rt);
|
2018-05-10 17:59:34 +00:00
|
|
|
out_free:
|
2008-03-07 00:22:02 +00:00
|
|
|
if (free)
|
|
|
|
kfree(ipc.opt);
|
|
|
|
if (!err)
|
|
|
|
return len;
|
|
|
|
/*
|
|
|
|
* ENOBUFS = no kernel mem, SOCK_NOSPACE = no sndbuf space. Reporting
|
|
|
|
* ENOBUFS might not be good (it's not tunable per se), but otherwise
|
|
|
|
* we don't have a good statistic (IpOutDiscards but it can be too many
|
|
|
|
* things). We could add another new stat but at least for now that
|
|
|
|
* seems like overkill.
|
|
|
|
*/
|
|
|
|
if (err == -ENOBUFS || test_bit(SOCK_NOSPACE, &sk->sk_socket->flags)) {
|
net: snmp: kill various STATS_USER() helpers
In the old days (before linux-3.0), SNMP counters were duplicated,
one for user context, and one for BH context.
After commit 8f0ea0fe3a03 ("snmp: reduce percpu needs by 50%")
we have a single copy, and what really matters is preemption being
enabled or disabled, since we use this_cpu_inc() or __this_cpu_inc()
respectively.
We therefore kill SNMP_INC_STATS_USER(), SNMP_ADD_STATS_USER(),
NET_INC_STATS_USER(), NET_ADD_STATS_USER(), SCTP_INC_STATS_USER(),
SNMP_INC_STATS64_USER(), SNMP_ADD_STATS64_USER(), TCP_ADD_STATS_USER(),
UDP_INC_STATS_USER(), UDP6_INC_STATS_USER(), and XFRM_INC_STATS_USER()
Following patches will rename __BH helpers to make clear their
usage is not tied to BH being disabled.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-04-27 23:44:27 +00:00
|
|
|
UDP_INC_STATS(sock_net(sk),
|
|
|
|
UDP_MIB_SNDBUFERRORS, is_udplite);
|
2008-03-07 00:22:02 +00:00
|
|
|
}
|
|
|
|
return err;
|
|
|
|
|
|
|
|
do_confirm:
|
2017-02-06 21:14:16 +00:00
|
|
|
if (msg->msg_flags & MSG_PROBE)
|
|
|
|
dst_confirm_neigh(&rt->dst, &fl4->daddr);
|
2008-03-07 00:22:02 +00:00
|
|
|
if (!(msg->msg_flags&MSG_PROBE) || len)
|
|
|
|
goto back_from_confirm;
|
|
|
|
err = 0;
|
|
|
|
goto out;
|
|
|
|
}
|
2009-07-17 00:26:32 +00:00
|
|
|
EXPORT_SYMBOL(udp_sendmsg);
|
2008-03-07 00:22:02 +00:00
|
|
|
|
2023-06-07 18:19:13 +00:00
|
|
|
void udp_splice_eof(struct socket *sock)
|
|
|
|
{
|
|
|
|
struct sock *sk = sock->sk;
|
|
|
|
struct udp_sock *up = udp_sk(sk);
|
|
|
|
|
2023-09-12 09:17:21 +00:00
|
|
|
if (!up->pending || udp_test_bit(CORK, sk))
|
2023-06-07 18:19:13 +00:00
|
|
|
return;
|
|
|
|
|
|
|
|
lock_sock(sk);
|
2023-09-12 09:17:21 +00:00
|
|
|
if (up->pending && !udp_test_bit(CORK, sk))
|
2023-06-07 18:19:13 +00:00
|
|
|
udp_push_pending_frames(sk);
|
|
|
|
release_sock(sk);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(udp_splice_eof);
|
|
|
|
|
2017-07-25 15:57:47 +00:00
|
|
|
#define UDP_SKB_IS_STATELESS 0x80000000
|
|
|
|
|
2019-11-21 05:56:23 +00:00
|
|
|
/* all head states (dst, sk, nf conntrack) except skb extensions are
|
|
|
|
* cleared by udp_rcv().
|
|
|
|
*
|
|
|
|
* We need to preserve secpath, if present, to eventually process
|
|
|
|
* IP_CMSG_PASSSEC at recvmsg() time.
|
|
|
|
*
|
|
|
|
* Other extensions can be cleared.
|
|
|
|
*/
|
|
|
|
static bool udp_try_make_stateless(struct sk_buff *skb)
|
|
|
|
{
|
|
|
|
if (!skb_has_extensions(skb))
|
|
|
|
return true;
|
|
|
|
|
|
|
|
if (!secpath_exists(skb)) {
|
|
|
|
skb_ext_reset(skb);
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2017-06-12 09:23:43 +00:00
|
|
|
static void udp_set_dev_scratch(struct sk_buff *skb)
|
|
|
|
{
|
2017-07-25 15:57:47 +00:00
|
|
|
struct udp_dev_scratch *scratch = udp_skb_scratch(skb);
|
2017-06-12 09:23:43 +00:00
|
|
|
|
|
|
|
BUILD_BUG_ON(sizeof(struct udp_dev_scratch) > sizeof(long));
|
2017-07-25 15:57:47 +00:00
|
|
|
scratch->_tsize_state = skb->truesize;
|
|
|
|
#if BITS_PER_LONG == 64
|
2017-06-12 09:23:43 +00:00
|
|
|
scratch->len = skb->len;
|
|
|
|
scratch->csum_unnecessary = !!skb_csum_unnecessary(skb);
|
|
|
|
scratch->is_linear = !skb_is_nonlinear(skb);
|
2017-07-25 15:57:47 +00:00
|
|
|
#endif
|
2019-11-21 05:56:23 +00:00
|
|
|
if (udp_try_make_stateless(skb))
|
2017-07-25 15:57:47 +00:00
|
|
|
scratch->_tsize_state |= UDP_SKB_IS_STATELESS;
|
2017-06-12 09:23:43 +00:00
|
|
|
}
|
|
|
|
|
2019-10-24 18:43:31 +00:00
|
|
|
static void udp_skb_csum_unnecessary_set(struct sk_buff *skb)
|
|
|
|
{
|
|
|
|
/* We come here after udp_lib_checksum_complete() returned 0.
|
|
|
|
* This means that __skb_checksum_complete() might have
|
|
|
|
* set skb->csum_valid to 1.
|
|
|
|
* On 64bit platforms, we can set csum_unnecessary
|
|
|
|
* to true, but only if the skb is not shared.
|
|
|
|
*/
|
|
|
|
#if BITS_PER_LONG == 64
|
|
|
|
if (!skb_shared(skb))
|
|
|
|
udp_skb_scratch(skb)->csum_unnecessary = true;
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
2017-06-12 09:23:43 +00:00
|
|
|
static int udp_skb_truesize(struct sk_buff *skb)
|
|
|
|
{
|
2017-07-25 15:57:47 +00:00
|
|
|
return udp_skb_scratch(skb)->_tsize_state & ~UDP_SKB_IS_STATELESS;
|
2017-06-12 09:23:43 +00:00
|
|
|
}
|
|
|
|
|
2017-07-25 15:57:47 +00:00
|
|
|
static bool udp_skb_has_head_state(struct sk_buff *skb)
|
2017-06-12 09:23:43 +00:00
|
|
|
{
|
2017-07-25 15:57:47 +00:00
|
|
|
return !(udp_skb_scratch(skb)->_tsize_state & UDP_SKB_IS_STATELESS);
|
2017-06-12 09:23:43 +00:00
|
|
|
}
|
|
|
|
|
2016-11-04 10:28:59 +00:00
|
|
|
/* fully reclaim rmem/fwd memory allocated for skb */
|
2017-05-16 09:20:15 +00:00
|
|
|
static void udp_rmem_release(struct sock *sk, int size, int partial,
|
|
|
|
bool rx_queue_lock_held)
|
udp: implement memory accounting helpers
Avoid using the generic helpers.
Use the receive queue spin lock to protect the memory
accounting operation, both on enqueue and on dequeue.
On dequeue perform partial memory reclaiming, trying to
leave a quantum of forward allocated memory.
On enqueue use a custom helper, to allow some optimizations:
- use a plain spin_lock() variant instead of the slightly
costly spin_lock_irqsave(),
- avoid dst_force check, since the calling code has already
dropped the skb dst
- avoid orphaning the skb, since skb_steal_sock() already did
the work for us
The above needs custom memory reclaiming on shutdown, provided
by the udp_destruct_sock().
v5 -> v6:
- don't orphan the skb on enqueue
v4 -> v5:
- replace the mem_lock with the receive queue spin lock
- ensure that the bh is always allowed to enqueue at least
a skb, even if sk_rcvbuf is exceeded
v3 -> v4:
- reworked memory accunting, simplifying the schema
- provide an helper for both memory scheduling and enqueuing
v1 -> v2:
- use a udp specific destrctor to perform memory reclaiming
- remove a couple of helpers, unneeded after the above cleanup
- do not reclaim memory on dequeue if not under memory
pressure
- reworked the fwd accounting schema to avoid potential
integer overflow
Acked-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-10-21 11:55:46 +00:00
|
|
|
{
|
2016-12-08 19:41:56 +00:00
|
|
|
struct udp_sock *up = udp_sk(sk);
|
2017-05-16 09:20:14 +00:00
|
|
|
struct sk_buff_head *sk_queue;
|
udp: implement memory accounting helpers
Avoid using the generic helpers.
Use the receive queue spin lock to protect the memory
accounting operation, both on enqueue and on dequeue.
On dequeue perform partial memory reclaiming, trying to
leave a quantum of forward allocated memory.
On enqueue use a custom helper, to allow some optimizations:
- use a plain spin_lock() variant instead of the slightly
costly spin_lock_irqsave(),
- avoid dst_force check, since the calling code has already
dropped the skb dst
- avoid orphaning the skb, since skb_steal_sock() already did
the work for us
The above needs custom memory reclaiming on shutdown, provided
by the udp_destruct_sock().
v5 -> v6:
- don't orphan the skb on enqueue
v4 -> v5:
- replace the mem_lock with the receive queue spin lock
- ensure that the bh is always allowed to enqueue at least
a skb, even if sk_rcvbuf is exceeded
v3 -> v4:
- reworked memory accunting, simplifying the schema
- provide an helper for both memory scheduling and enqueuing
v1 -> v2:
- use a udp specific destrctor to perform memory reclaiming
- remove a couple of helpers, unneeded after the above cleanup
- do not reclaim memory on dequeue if not under memory
pressure
- reworked the fwd accounting schema to avoid potential
integer overflow
Acked-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-10-21 11:55:46 +00:00
|
|
|
int amt;
|
|
|
|
|
2016-12-08 19:41:56 +00:00
|
|
|
if (likely(partial)) {
|
|
|
|
up->forward_deficit += size;
|
|
|
|
size = up->forward_deficit;
|
2022-10-20 17:48:52 +00:00
|
|
|
if (size < READ_ONCE(up->forward_threshold) &&
|
2020-01-21 15:50:49 +00:00
|
|
|
!skb_queue_empty(&up->reader_queue))
|
2016-12-08 19:41:56 +00:00
|
|
|
return;
|
|
|
|
} else {
|
|
|
|
size += up->forward_deficit;
|
|
|
|
}
|
|
|
|
up->forward_deficit = 0;
|
|
|
|
|
2017-05-16 09:20:15 +00:00
|
|
|
/* acquire the sk_receive_queue for fwd allocated memory scheduling,
|
|
|
|
* if the called don't held it already
|
|
|
|
*/
|
2017-05-16 09:20:14 +00:00
|
|
|
sk_queue = &sk->sk_receive_queue;
|
2017-05-16 09:20:15 +00:00
|
|
|
if (!rx_queue_lock_held)
|
|
|
|
spin_lock(&sk_queue->lock);
|
|
|
|
|
2017-05-16 09:20:14 +00:00
|
|
|
|
2023-08-31 13:52:09 +00:00
|
|
|
sk_forward_alloc_add(sk, size);
|
2022-06-09 06:34:07 +00:00
|
|
|
amt = (sk->sk_forward_alloc - partial) & ~(PAGE_SIZE - 1);
|
2023-08-31 13:52:09 +00:00
|
|
|
sk_forward_alloc_add(sk, -amt);
|
udp: implement memory accounting helpers
Avoid using the generic helpers.
Use the receive queue spin lock to protect the memory
accounting operation, both on enqueue and on dequeue.
On dequeue perform partial memory reclaiming, trying to
leave a quantum of forward allocated memory.
On enqueue use a custom helper, to allow some optimizations:
- use a plain spin_lock() variant instead of the slightly
costly spin_lock_irqsave(),
- avoid dst_force check, since the calling code has already
dropped the skb dst
- avoid orphaning the skb, since skb_steal_sock() already did
the work for us
The above needs custom memory reclaiming on shutdown, provided
by the udp_destruct_sock().
v5 -> v6:
- don't orphan the skb on enqueue
v4 -> v5:
- replace the mem_lock with the receive queue spin lock
- ensure that the bh is always allowed to enqueue at least
a skb, even if sk_rcvbuf is exceeded
v3 -> v4:
- reworked memory accunting, simplifying the schema
- provide an helper for both memory scheduling and enqueuing
v1 -> v2:
- use a udp specific destrctor to perform memory reclaiming
- remove a couple of helpers, unneeded after the above cleanup
- do not reclaim memory on dequeue if not under memory
pressure
- reworked the fwd accounting schema to avoid potential
integer overflow
Acked-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-10-21 11:55:46 +00:00
|
|
|
|
|
|
|
if (amt)
|
2022-06-09 06:34:07 +00:00
|
|
|
__sk_mem_reduce_allocated(sk, amt >> PAGE_SHIFT);
|
2016-12-08 19:41:57 +00:00
|
|
|
|
|
|
|
atomic_sub(size, &sk->sk_rmem_alloc);
|
2017-05-16 09:20:14 +00:00
|
|
|
|
|
|
|
/* this can save us from acquiring the rx queue lock on next receive */
|
|
|
|
skb_queue_splice_tail_init(sk_queue, &up->reader_queue);
|
|
|
|
|
2017-05-16 09:20:15 +00:00
|
|
|
if (!rx_queue_lock_held)
|
|
|
|
spin_unlock(&sk_queue->lock);
|
udp: implement memory accounting helpers
Avoid using the generic helpers.
Use the receive queue spin lock to protect the memory
accounting operation, both on enqueue and on dequeue.
On dequeue perform partial memory reclaiming, trying to
leave a quantum of forward allocated memory.
On enqueue use a custom helper, to allow some optimizations:
- use a plain spin_lock() variant instead of the slightly
costly spin_lock_irqsave(),
- avoid dst_force check, since the calling code has already
dropped the skb dst
- avoid orphaning the skb, since skb_steal_sock() already did
the work for us
The above needs custom memory reclaiming on shutdown, provided
by the udp_destruct_sock().
v5 -> v6:
- don't orphan the skb on enqueue
v4 -> v5:
- replace the mem_lock with the receive queue spin lock
- ensure that the bh is always allowed to enqueue at least
a skb, even if sk_rcvbuf is exceeded
v3 -> v4:
- reworked memory accunting, simplifying the schema
- provide an helper for both memory scheduling and enqueuing
v1 -> v2:
- use a udp specific destrctor to perform memory reclaiming
- remove a couple of helpers, unneeded after the above cleanup
- do not reclaim memory on dequeue if not under memory
pressure
- reworked the fwd accounting schema to avoid potential
integer overflow
Acked-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-10-21 11:55:46 +00:00
|
|
|
}
|
|
|
|
|
2017-05-16 09:20:14 +00:00
|
|
|
/* Note: called with reader_queue.lock held.
|
2016-12-08 19:41:55 +00:00
|
|
|
* Instead of using skb->truesize here, find a copy of it in skb->dev_scratch
|
|
|
|
* This avoids a cache line miss while receive_queue lock is held.
|
|
|
|
* Look at __udp_enqueue_schedule_skb() to find where this copy is done.
|
|
|
|
*/
|
2016-11-04 10:28:59 +00:00
|
|
|
void udp_skb_destructor(struct sock *sk, struct sk_buff *skb)
|
udp: implement memory accounting helpers
Avoid using the generic helpers.
Use the receive queue spin lock to protect the memory
accounting operation, both on enqueue and on dequeue.
On dequeue perform partial memory reclaiming, trying to
leave a quantum of forward allocated memory.
On enqueue use a custom helper, to allow some optimizations:
- use a plain spin_lock() variant instead of the slightly
costly spin_lock_irqsave(),
- avoid dst_force check, since the calling code has already
dropped the skb dst
- avoid orphaning the skb, since skb_steal_sock() already did
the work for us
The above needs custom memory reclaiming on shutdown, provided
by the udp_destruct_sock().
v5 -> v6:
- don't orphan the skb on enqueue
v4 -> v5:
- replace the mem_lock with the receive queue spin lock
- ensure that the bh is always allowed to enqueue at least
a skb, even if sk_rcvbuf is exceeded
v3 -> v4:
- reworked memory accunting, simplifying the schema
- provide an helper for both memory scheduling and enqueuing
v1 -> v2:
- use a udp specific destrctor to perform memory reclaiming
- remove a couple of helpers, unneeded after the above cleanup
- do not reclaim memory on dequeue if not under memory
pressure
- reworked the fwd accounting schema to avoid potential
integer overflow
Acked-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-10-21 11:55:46 +00:00
|
|
|
{
|
2017-06-12 09:23:43 +00:00
|
|
|
prefetch(&skb->data);
|
|
|
|
udp_rmem_release(sk, udp_skb_truesize(skb), 1, false);
|
udp: implement memory accounting helpers
Avoid using the generic helpers.
Use the receive queue spin lock to protect the memory
accounting operation, both on enqueue and on dequeue.
On dequeue perform partial memory reclaiming, trying to
leave a quantum of forward allocated memory.
On enqueue use a custom helper, to allow some optimizations:
- use a plain spin_lock() variant instead of the slightly
costly spin_lock_irqsave(),
- avoid dst_force check, since the calling code has already
dropped the skb dst
- avoid orphaning the skb, since skb_steal_sock() already did
the work for us
The above needs custom memory reclaiming on shutdown, provided
by the udp_destruct_sock().
v5 -> v6:
- don't orphan the skb on enqueue
v4 -> v5:
- replace the mem_lock with the receive queue spin lock
- ensure that the bh is always allowed to enqueue at least
a skb, even if sk_rcvbuf is exceeded
v3 -> v4:
- reworked memory accunting, simplifying the schema
- provide an helper for both memory scheduling and enqueuing
v1 -> v2:
- use a udp specific destrctor to perform memory reclaiming
- remove a couple of helpers, unneeded after the above cleanup
- do not reclaim memory on dequeue if not under memory
pressure
- reworked the fwd accounting schema to avoid potential
integer overflow
Acked-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-10-21 11:55:46 +00:00
|
|
|
}
|
2016-11-04 10:28:59 +00:00
|
|
|
EXPORT_SYMBOL(udp_skb_destructor);
|
udp: implement memory accounting helpers
Avoid using the generic helpers.
Use the receive queue spin lock to protect the memory
accounting operation, both on enqueue and on dequeue.
On dequeue perform partial memory reclaiming, trying to
leave a quantum of forward allocated memory.
On enqueue use a custom helper, to allow some optimizations:
- use a plain spin_lock() variant instead of the slightly
costly spin_lock_irqsave(),
- avoid dst_force check, since the calling code has already
dropped the skb dst
- avoid orphaning the skb, since skb_steal_sock() already did
the work for us
The above needs custom memory reclaiming on shutdown, provided
by the udp_destruct_sock().
v5 -> v6:
- don't orphan the skb on enqueue
v4 -> v5:
- replace the mem_lock with the receive queue spin lock
- ensure that the bh is always allowed to enqueue at least
a skb, even if sk_rcvbuf is exceeded
v3 -> v4:
- reworked memory accunting, simplifying the schema
- provide an helper for both memory scheduling and enqueuing
v1 -> v2:
- use a udp specific destrctor to perform memory reclaiming
- remove a couple of helpers, unneeded after the above cleanup
- do not reclaim memory on dequeue if not under memory
pressure
- reworked the fwd accounting schema to avoid potential
integer overflow
Acked-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-10-21 11:55:46 +00:00
|
|
|
|
2017-05-16 09:20:15 +00:00
|
|
|
/* as above, but the caller held the rx queue lock, too */
|
2017-05-17 08:50:36 +00:00
|
|
|
static void udp_skb_dtor_locked(struct sock *sk, struct sk_buff *skb)
|
2017-05-16 09:20:15 +00:00
|
|
|
{
|
2017-06-12 09:23:43 +00:00
|
|
|
prefetch(&skb->data);
|
|
|
|
udp_rmem_release(sk, udp_skb_truesize(skb), 1, true);
|
2017-05-16 09:20:15 +00:00
|
|
|
}
|
|
|
|
|
2016-12-08 19:41:54 +00:00
|
|
|
/* Idea of busylocks is to let producers grab an extra spinlock
|
|
|
|
* to relieve pressure on the receive_queue spinlock shared by consumer.
|
|
|
|
* Under flood, this means that only one producer can be in line
|
|
|
|
* trying to acquire the receive_queue spinlock.
|
|
|
|
* These busylock can be allocated on a per cpu manner, instead of a
|
|
|
|
* per socket one (that would consume a cache line per socket)
|
|
|
|
*/
|
|
|
|
static int udp_busylocks_log __read_mostly;
|
|
|
|
static spinlock_t *udp_busylocks __read_mostly;
|
|
|
|
|
|
|
|
static spinlock_t *busylock_acquire(void *ptr)
|
|
|
|
{
|
|
|
|
spinlock_t *busy;
|
|
|
|
|
|
|
|
busy = udp_busylocks + hash_ptr(ptr, udp_busylocks_log);
|
|
|
|
spin_lock(busy);
|
|
|
|
return busy;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void busylock_release(spinlock_t *busy)
|
|
|
|
{
|
|
|
|
if (busy)
|
|
|
|
spin_unlock(busy);
|
|
|
|
}
|
|
|
|
|
2023-03-08 02:11:53 +00:00
|
|
|
static int udp_rmem_schedule(struct sock *sk, int size)
|
|
|
|
{
|
|
|
|
int delta;
|
|
|
|
|
|
|
|
delta = size - sk->sk_forward_alloc;
|
|
|
|
if (delta > 0 && !__sk_mem_schedule(sk, delta, SK_MEM_RECV))
|
|
|
|
return -ENOBUFS;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
udp: implement memory accounting helpers
Avoid using the generic helpers.
Use the receive queue spin lock to protect the memory
accounting operation, both on enqueue and on dequeue.
On dequeue perform partial memory reclaiming, trying to
leave a quantum of forward allocated memory.
On enqueue use a custom helper, to allow some optimizations:
- use a plain spin_lock() variant instead of the slightly
costly spin_lock_irqsave(),
- avoid dst_force check, since the calling code has already
dropped the skb dst
- avoid orphaning the skb, since skb_steal_sock() already did
the work for us
The above needs custom memory reclaiming on shutdown, provided
by the udp_destruct_sock().
v5 -> v6:
- don't orphan the skb on enqueue
v4 -> v5:
- replace the mem_lock with the receive queue spin lock
- ensure that the bh is always allowed to enqueue at least
a skb, even if sk_rcvbuf is exceeded
v3 -> v4:
- reworked memory accunting, simplifying the schema
- provide an helper for both memory scheduling and enqueuing
v1 -> v2:
- use a udp specific destrctor to perform memory reclaiming
- remove a couple of helpers, unneeded after the above cleanup
- do not reclaim memory on dequeue if not under memory
pressure
- reworked the fwd accounting schema to avoid potential
integer overflow
Acked-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-10-21 11:55:46 +00:00
|
|
|
int __udp_enqueue_schedule_skb(struct sock *sk, struct sk_buff *skb)
|
|
|
|
{
|
|
|
|
struct sk_buff_head *list = &sk->sk_receive_queue;
|
2023-03-08 02:11:53 +00:00
|
|
|
int rmem, err = -ENOMEM;
|
2016-12-08 19:41:54 +00:00
|
|
|
spinlock_t *busy = NULL;
|
udp: under rx pressure, try to condense skbs
Under UDP flood, many softirq producers try to add packets to
UDP receive queue, and one user thread is burning one cpu trying
to dequeue packets as fast as possible.
Two parts of the per packet cost are :
- copying payload from kernel space to user space,
- freeing memory pieces associated with skb.
If socket is under pressure, softirq handler(s) can try to pull in
skb->head the payload of the packet if it fits.
Meaning the softirq handler(s) can free/reuse the page fragment
immediately, instead of letting udp_recvmsg() do this hundreds of usec
later, possibly from another node.
Additional gains :
- We reduce skb->truesize and thus can store more packets per SO_RCVBUF
- We avoid cache line misses at copyout() time and consume_skb() time,
and avoid one put_page() with potential alien freeing on NUMA hosts.
This comes at the cost of a copy, bounded to available tail room, which
is usually small. (We might have to fix GRO_MAX_HEAD which looks bigger
than necessary)
This patch gave me about 5 % increase in throughput in my tests.
skb_condense() helper could probably used in other contexts.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Cc: Paolo Abeni <pabeni@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-12-07 17:19:33 +00:00
|
|
|
int size;
|
udp: implement memory accounting helpers
Avoid using the generic helpers.
Use the receive queue spin lock to protect the memory
accounting operation, both on enqueue and on dequeue.
On dequeue perform partial memory reclaiming, trying to
leave a quantum of forward allocated memory.
On enqueue use a custom helper, to allow some optimizations:
- use a plain spin_lock() variant instead of the slightly
costly spin_lock_irqsave(),
- avoid dst_force check, since the calling code has already
dropped the skb dst
- avoid orphaning the skb, since skb_steal_sock() already did
the work for us
The above needs custom memory reclaiming on shutdown, provided
by the udp_destruct_sock().
v5 -> v6:
- don't orphan the skb on enqueue
v4 -> v5:
- replace the mem_lock with the receive queue spin lock
- ensure that the bh is always allowed to enqueue at least
a skb, even if sk_rcvbuf is exceeded
v3 -> v4:
- reworked memory accunting, simplifying the schema
- provide an helper for both memory scheduling and enqueuing
v1 -> v2:
- use a udp specific destrctor to perform memory reclaiming
- remove a couple of helpers, unneeded after the above cleanup
- do not reclaim memory on dequeue if not under memory
pressure
- reworked the fwd accounting schema to avoid potential
integer overflow
Acked-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-10-21 11:55:46 +00:00
|
|
|
|
|
|
|
/* try to avoid the costly atomic add/sub pair when the receive
|
|
|
|
* queue is full; always allow at least a packet
|
|
|
|
*/
|
|
|
|
rmem = atomic_read(&sk->sk_rmem_alloc);
|
2016-12-02 16:35:49 +00:00
|
|
|
if (rmem > sk->sk_rcvbuf)
|
udp: implement memory accounting helpers
Avoid using the generic helpers.
Use the receive queue spin lock to protect the memory
accounting operation, both on enqueue and on dequeue.
On dequeue perform partial memory reclaiming, trying to
leave a quantum of forward allocated memory.
On enqueue use a custom helper, to allow some optimizations:
- use a plain spin_lock() variant instead of the slightly
costly spin_lock_irqsave(),
- avoid dst_force check, since the calling code has already
dropped the skb dst
- avoid orphaning the skb, since skb_steal_sock() already did
the work for us
The above needs custom memory reclaiming on shutdown, provided
by the udp_destruct_sock().
v5 -> v6:
- don't orphan the skb on enqueue
v4 -> v5:
- replace the mem_lock with the receive queue spin lock
- ensure that the bh is always allowed to enqueue at least
a skb, even if sk_rcvbuf is exceeded
v3 -> v4:
- reworked memory accunting, simplifying the schema
- provide an helper for both memory scheduling and enqueuing
v1 -> v2:
- use a udp specific destrctor to perform memory reclaiming
- remove a couple of helpers, unneeded after the above cleanup
- do not reclaim memory on dequeue if not under memory
pressure
- reworked the fwd accounting schema to avoid potential
integer overflow
Acked-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-10-21 11:55:46 +00:00
|
|
|
goto drop;
|
|
|
|
|
udp: under rx pressure, try to condense skbs
Under UDP flood, many softirq producers try to add packets to
UDP receive queue, and one user thread is burning one cpu trying
to dequeue packets as fast as possible.
Two parts of the per packet cost are :
- copying payload from kernel space to user space,
- freeing memory pieces associated with skb.
If socket is under pressure, softirq handler(s) can try to pull in
skb->head the payload of the packet if it fits.
Meaning the softirq handler(s) can free/reuse the page fragment
immediately, instead of letting udp_recvmsg() do this hundreds of usec
later, possibly from another node.
Additional gains :
- We reduce skb->truesize and thus can store more packets per SO_RCVBUF
- We avoid cache line misses at copyout() time and consume_skb() time,
and avoid one put_page() with potential alien freeing on NUMA hosts.
This comes at the cost of a copy, bounded to available tail room, which
is usually small. (We might have to fix GRO_MAX_HEAD which looks bigger
than necessary)
This patch gave me about 5 % increase in throughput in my tests.
skb_condense() helper could probably used in other contexts.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Cc: Paolo Abeni <pabeni@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-12-07 17:19:33 +00:00
|
|
|
/* Under mem pressure, it might be helpful to help udp_recvmsg()
|
|
|
|
* having linear skbs :
|
|
|
|
* - Reduce memory overhead and thus increase receive queue capacity
|
|
|
|
* - Less cache line misses at copyout() time
|
|
|
|
* - Less work at consume_skb() (less alien page frag freeing)
|
|
|
|
*/
|
2016-12-08 19:41:54 +00:00
|
|
|
if (rmem > (sk->sk_rcvbuf >> 1)) {
|
udp: under rx pressure, try to condense skbs
Under UDP flood, many softirq producers try to add packets to
UDP receive queue, and one user thread is burning one cpu trying
to dequeue packets as fast as possible.
Two parts of the per packet cost are :
- copying payload from kernel space to user space,
- freeing memory pieces associated with skb.
If socket is under pressure, softirq handler(s) can try to pull in
skb->head the payload of the packet if it fits.
Meaning the softirq handler(s) can free/reuse the page fragment
immediately, instead of letting udp_recvmsg() do this hundreds of usec
later, possibly from another node.
Additional gains :
- We reduce skb->truesize and thus can store more packets per SO_RCVBUF
- We avoid cache line misses at copyout() time and consume_skb() time,
and avoid one put_page() with potential alien freeing on NUMA hosts.
This comes at the cost of a copy, bounded to available tail room, which
is usually small. (We might have to fix GRO_MAX_HEAD which looks bigger
than necessary)
This patch gave me about 5 % increase in throughput in my tests.
skb_condense() helper could probably used in other contexts.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Cc: Paolo Abeni <pabeni@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-12-07 17:19:33 +00:00
|
|
|
skb_condense(skb);
|
2016-12-08 19:41:54 +00:00
|
|
|
|
|
|
|
busy = busylock_acquire(sk);
|
|
|
|
}
|
udp: under rx pressure, try to condense skbs
Under UDP flood, many softirq producers try to add packets to
UDP receive queue, and one user thread is burning one cpu trying
to dequeue packets as fast as possible.
Two parts of the per packet cost are :
- copying payload from kernel space to user space,
- freeing memory pieces associated with skb.
If socket is under pressure, softirq handler(s) can try to pull in
skb->head the payload of the packet if it fits.
Meaning the softirq handler(s) can free/reuse the page fragment
immediately, instead of letting udp_recvmsg() do this hundreds of usec
later, possibly from another node.
Additional gains :
- We reduce skb->truesize and thus can store more packets per SO_RCVBUF
- We avoid cache line misses at copyout() time and consume_skb() time,
and avoid one put_page() with potential alien freeing on NUMA hosts.
This comes at the cost of a copy, bounded to available tail room, which
is usually small. (We might have to fix GRO_MAX_HEAD which looks bigger
than necessary)
This patch gave me about 5 % increase in throughput in my tests.
skb_condense() helper could probably used in other contexts.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Cc: Paolo Abeni <pabeni@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-12-07 17:19:33 +00:00
|
|
|
size = skb->truesize;
|
2017-06-12 09:23:43 +00:00
|
|
|
udp_set_dev_scratch(skb);
|
udp: under rx pressure, try to condense skbs
Under UDP flood, many softirq producers try to add packets to
UDP receive queue, and one user thread is burning one cpu trying
to dequeue packets as fast as possible.
Two parts of the per packet cost are :
- copying payload from kernel space to user space,
- freeing memory pieces associated with skb.
If socket is under pressure, softirq handler(s) can try to pull in
skb->head the payload of the packet if it fits.
Meaning the softirq handler(s) can free/reuse the page fragment
immediately, instead of letting udp_recvmsg() do this hundreds of usec
later, possibly from another node.
Additional gains :
- We reduce skb->truesize and thus can store more packets per SO_RCVBUF
- We avoid cache line misses at copyout() time and consume_skb() time,
and avoid one put_page() with potential alien freeing on NUMA hosts.
This comes at the cost of a copy, bounded to available tail room, which
is usually small. (We might have to fix GRO_MAX_HEAD which looks bigger
than necessary)
This patch gave me about 5 % increase in throughput in my tests.
skb_condense() helper could probably used in other contexts.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Cc: Paolo Abeni <pabeni@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-12-07 17:19:33 +00:00
|
|
|
|
udp: implement memory accounting helpers
Avoid using the generic helpers.
Use the receive queue spin lock to protect the memory
accounting operation, both on enqueue and on dequeue.
On dequeue perform partial memory reclaiming, trying to
leave a quantum of forward allocated memory.
On enqueue use a custom helper, to allow some optimizations:
- use a plain spin_lock() variant instead of the slightly
costly spin_lock_irqsave(),
- avoid dst_force check, since the calling code has already
dropped the skb dst
- avoid orphaning the skb, since skb_steal_sock() already did
the work for us
The above needs custom memory reclaiming on shutdown, provided
by the udp_destruct_sock().
v5 -> v6:
- don't orphan the skb on enqueue
v4 -> v5:
- replace the mem_lock with the receive queue spin lock
- ensure that the bh is always allowed to enqueue at least
a skb, even if sk_rcvbuf is exceeded
v3 -> v4:
- reworked memory accunting, simplifying the schema
- provide an helper for both memory scheduling and enqueuing
v1 -> v2:
- use a udp specific destrctor to perform memory reclaiming
- remove a couple of helpers, unneeded after the above cleanup
- do not reclaim memory on dequeue if not under memory
pressure
- reworked the fwd accounting schema to avoid potential
integer overflow
Acked-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-10-21 11:55:46 +00:00
|
|
|
/* we drop only if the receive buf is full and the receive
|
|
|
|
* queue contains some other skb
|
|
|
|
*/
|
|
|
|
rmem = atomic_add_return(size, &sk->sk_rmem_alloc);
|
2019-12-19 14:08:03 +00:00
|
|
|
if (rmem > (size + (unsigned int)sk->sk_rcvbuf))
|
udp: implement memory accounting helpers
Avoid using the generic helpers.
Use the receive queue spin lock to protect the memory
accounting operation, both on enqueue and on dequeue.
On dequeue perform partial memory reclaiming, trying to
leave a quantum of forward allocated memory.
On enqueue use a custom helper, to allow some optimizations:
- use a plain spin_lock() variant instead of the slightly
costly spin_lock_irqsave(),
- avoid dst_force check, since the calling code has already
dropped the skb dst
- avoid orphaning the skb, since skb_steal_sock() already did
the work for us
The above needs custom memory reclaiming on shutdown, provided
by the udp_destruct_sock().
v5 -> v6:
- don't orphan the skb on enqueue
v4 -> v5:
- replace the mem_lock with the receive queue spin lock
- ensure that the bh is always allowed to enqueue at least
a skb, even if sk_rcvbuf is exceeded
v3 -> v4:
- reworked memory accunting, simplifying the schema
- provide an helper for both memory scheduling and enqueuing
v1 -> v2:
- use a udp specific destrctor to perform memory reclaiming
- remove a couple of helpers, unneeded after the above cleanup
- do not reclaim memory on dequeue if not under memory
pressure
- reworked the fwd accounting schema to avoid potential
integer overflow
Acked-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-10-21 11:55:46 +00:00
|
|
|
goto uncharge_drop;
|
|
|
|
|
|
|
|
spin_lock(&list->lock);
|
2023-03-08 02:11:53 +00:00
|
|
|
err = udp_rmem_schedule(sk, size);
|
|
|
|
if (err) {
|
|
|
|
spin_unlock(&list->lock);
|
|
|
|
goto uncharge_drop;
|
udp: implement memory accounting helpers
Avoid using the generic helpers.
Use the receive queue spin lock to protect the memory
accounting operation, both on enqueue and on dequeue.
On dequeue perform partial memory reclaiming, trying to
leave a quantum of forward allocated memory.
On enqueue use a custom helper, to allow some optimizations:
- use a plain spin_lock() variant instead of the slightly
costly spin_lock_irqsave(),
- avoid dst_force check, since the calling code has already
dropped the skb dst
- avoid orphaning the skb, since skb_steal_sock() already did
the work for us
The above needs custom memory reclaiming on shutdown, provided
by the udp_destruct_sock().
v5 -> v6:
- don't orphan the skb on enqueue
v4 -> v5:
- replace the mem_lock with the receive queue spin lock
- ensure that the bh is always allowed to enqueue at least
a skb, even if sk_rcvbuf is exceeded
v3 -> v4:
- reworked memory accunting, simplifying the schema
- provide an helper for both memory scheduling and enqueuing
v1 -> v2:
- use a udp specific destrctor to perform memory reclaiming
- remove a couple of helpers, unneeded after the above cleanup
- do not reclaim memory on dequeue if not under memory
pressure
- reworked the fwd accounting schema to avoid potential
integer overflow
Acked-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-10-21 11:55:46 +00:00
|
|
|
}
|
|
|
|
|
2023-08-31 13:52:09 +00:00
|
|
|
sk_forward_alloc_add(sk, -size);
|
udp: implement memory accounting helpers
Avoid using the generic helpers.
Use the receive queue spin lock to protect the memory
accounting operation, both on enqueue and on dequeue.
On dequeue perform partial memory reclaiming, trying to
leave a quantum of forward allocated memory.
On enqueue use a custom helper, to allow some optimizations:
- use a plain spin_lock() variant instead of the slightly
costly spin_lock_irqsave(),
- avoid dst_force check, since the calling code has already
dropped the skb dst
- avoid orphaning the skb, since skb_steal_sock() already did
the work for us
The above needs custom memory reclaiming on shutdown, provided
by the udp_destruct_sock().
v5 -> v6:
- don't orphan the skb on enqueue
v4 -> v5:
- replace the mem_lock with the receive queue spin lock
- ensure that the bh is always allowed to enqueue at least
a skb, even if sk_rcvbuf is exceeded
v3 -> v4:
- reworked memory accunting, simplifying the schema
- provide an helper for both memory scheduling and enqueuing
v1 -> v2:
- use a udp specific destrctor to perform memory reclaiming
- remove a couple of helpers, unneeded after the above cleanup
- do not reclaim memory on dequeue if not under memory
pressure
- reworked the fwd accounting schema to avoid potential
integer overflow
Acked-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-10-21 11:55:46 +00:00
|
|
|
|
2016-11-04 10:28:59 +00:00
|
|
|
/* no need to setup a destructor, we will explicitly release the
|
|
|
|
* forward allocated memory on dequeue
|
|
|
|
*/
|
udp: implement memory accounting helpers
Avoid using the generic helpers.
Use the receive queue spin lock to protect the memory
accounting operation, both on enqueue and on dequeue.
On dequeue perform partial memory reclaiming, trying to
leave a quantum of forward allocated memory.
On enqueue use a custom helper, to allow some optimizations:
- use a plain spin_lock() variant instead of the slightly
costly spin_lock_irqsave(),
- avoid dst_force check, since the calling code has already
dropped the skb dst
- avoid orphaning the skb, since skb_steal_sock() already did
the work for us
The above needs custom memory reclaiming on shutdown, provided
by the udp_destruct_sock().
v5 -> v6:
- don't orphan the skb on enqueue
v4 -> v5:
- replace the mem_lock with the receive queue spin lock
- ensure that the bh is always allowed to enqueue at least
a skb, even if sk_rcvbuf is exceeded
v3 -> v4:
- reworked memory accunting, simplifying the schema
- provide an helper for both memory scheduling and enqueuing
v1 -> v2:
- use a udp specific destrctor to perform memory reclaiming
- remove a couple of helpers, unneeded after the above cleanup
- do not reclaim memory on dequeue if not under memory
pressure
- reworked the fwd accounting schema to avoid potential
integer overflow
Acked-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-10-21 11:55:46 +00:00
|
|
|
sock_skb_set_dropcount(sk, skb);
|
|
|
|
|
|
|
|
__skb_queue_tail(list, skb);
|
|
|
|
spin_unlock(&list->lock);
|
|
|
|
|
|
|
|
if (!sock_flag(sk, SOCK_DEAD))
|
2023-07-18 14:38:09 +00:00
|
|
|
INDIRECT_CALL_1(sk->sk_data_ready, sock_def_readable, sk);
|
udp: implement memory accounting helpers
Avoid using the generic helpers.
Use the receive queue spin lock to protect the memory
accounting operation, both on enqueue and on dequeue.
On dequeue perform partial memory reclaiming, trying to
leave a quantum of forward allocated memory.
On enqueue use a custom helper, to allow some optimizations:
- use a plain spin_lock() variant instead of the slightly
costly spin_lock_irqsave(),
- avoid dst_force check, since the calling code has already
dropped the skb dst
- avoid orphaning the skb, since skb_steal_sock() already did
the work for us
The above needs custom memory reclaiming on shutdown, provided
by the udp_destruct_sock().
v5 -> v6:
- don't orphan the skb on enqueue
v4 -> v5:
- replace the mem_lock with the receive queue spin lock
- ensure that the bh is always allowed to enqueue at least
a skb, even if sk_rcvbuf is exceeded
v3 -> v4:
- reworked memory accunting, simplifying the schema
- provide an helper for both memory scheduling and enqueuing
v1 -> v2:
- use a udp specific destrctor to perform memory reclaiming
- remove a couple of helpers, unneeded after the above cleanup
- do not reclaim memory on dequeue if not under memory
pressure
- reworked the fwd accounting schema to avoid potential
integer overflow
Acked-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-10-21 11:55:46 +00:00
|
|
|
|
2016-12-08 19:41:54 +00:00
|
|
|
busylock_release(busy);
|
udp: implement memory accounting helpers
Avoid using the generic helpers.
Use the receive queue spin lock to protect the memory
accounting operation, both on enqueue and on dequeue.
On dequeue perform partial memory reclaiming, trying to
leave a quantum of forward allocated memory.
On enqueue use a custom helper, to allow some optimizations:
- use a plain spin_lock() variant instead of the slightly
costly spin_lock_irqsave(),
- avoid dst_force check, since the calling code has already
dropped the skb dst
- avoid orphaning the skb, since skb_steal_sock() already did
the work for us
The above needs custom memory reclaiming on shutdown, provided
by the udp_destruct_sock().
v5 -> v6:
- don't orphan the skb on enqueue
v4 -> v5:
- replace the mem_lock with the receive queue spin lock
- ensure that the bh is always allowed to enqueue at least
a skb, even if sk_rcvbuf is exceeded
v3 -> v4:
- reworked memory accunting, simplifying the schema
- provide an helper for both memory scheduling and enqueuing
v1 -> v2:
- use a udp specific destrctor to perform memory reclaiming
- remove a couple of helpers, unneeded after the above cleanup
- do not reclaim memory on dequeue if not under memory
pressure
- reworked the fwd accounting schema to avoid potential
integer overflow
Acked-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-10-21 11:55:46 +00:00
|
|
|
return 0;
|
|
|
|
|
|
|
|
uncharge_drop:
|
|
|
|
atomic_sub(skb->truesize, &sk->sk_rmem_alloc);
|
|
|
|
|
|
|
|
drop:
|
|
|
|
atomic_inc(&sk->sk_drops);
|
2016-12-08 19:41:54 +00:00
|
|
|
busylock_release(busy);
|
udp: implement memory accounting helpers
Avoid using the generic helpers.
Use the receive queue spin lock to protect the memory
accounting operation, both on enqueue and on dequeue.
On dequeue perform partial memory reclaiming, trying to
leave a quantum of forward allocated memory.
On enqueue use a custom helper, to allow some optimizations:
- use a plain spin_lock() variant instead of the slightly
costly spin_lock_irqsave(),
- avoid dst_force check, since the calling code has already
dropped the skb dst
- avoid orphaning the skb, since skb_steal_sock() already did
the work for us
The above needs custom memory reclaiming on shutdown, provided
by the udp_destruct_sock().
v5 -> v6:
- don't orphan the skb on enqueue
v4 -> v5:
- replace the mem_lock with the receive queue spin lock
- ensure that the bh is always allowed to enqueue at least
a skb, even if sk_rcvbuf is exceeded
v3 -> v4:
- reworked memory accunting, simplifying the schema
- provide an helper for both memory scheduling and enqueuing
v1 -> v2:
- use a udp specific destrctor to perform memory reclaiming
- remove a couple of helpers, unneeded after the above cleanup
- do not reclaim memory on dequeue if not under memory
pressure
- reworked the fwd accounting schema to avoid potential
integer overflow
Acked-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-10-21 11:55:46 +00:00
|
|
|
return err;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(__udp_enqueue_schedule_skb);
|
|
|
|
|
tcp/udp: Call inet6_destroy_sock() in IPv6 sk->sk_destruct().
Originally, inet6_sk(sk)->XXX were changed under lock_sock(), so we were
able to clean them up by calling inet6_destroy_sock() during the IPv6 ->
IPv4 conversion by IPV6_ADDRFORM. However, commit 03485f2adcde ("udpv6:
Add lockless sendmsg() support") added a lockless memory allocation path,
which could cause a memory leak:
setsockopt(IPV6_ADDRFORM) sendmsg()
+-----------------------+ +-------+
- do_ipv6_setsockopt(sk, ...) - udpv6_sendmsg(sk, ...)
- sockopt_lock_sock(sk) ^._ called via udpv6_prot
- lock_sock(sk) before WRITE_ONCE()
- WRITE_ONCE(sk->sk_prot, &tcp_prot)
- inet6_destroy_sock() - if (!corkreq)
- sockopt_release_sock(sk) - ip6_make_skb(sk, ...)
- release_sock(sk) ^._ lockless fast path for
the non-corking case
- __ip6_append_data(sk, ...)
- ipv6_local_rxpmtu(sk, ...)
- xchg(&np->rxpmtu, skb)
^._ rxpmtu is never freed.
- goto out_no_dst;
- lock_sock(sk)
For now, rxpmtu is only the case, but not to miss the future change
and a similar bug fixed in commit e27326009a3d ("net: ping6: Fix
memleak in ipv6_renew_options()."), let's set a new function to IPv6
sk->sk_destruct() and call inet6_cleanup_sock() there. Since the
conversion does not change sk->sk_destruct(), we can guarantee that
we can clean up IPv6 resources finally.
We can now remove all inet6_destroy_sock() calls from IPv6 protocol
specific ->destroy() functions, but such changes are invasive to
backport. So they can be posted as a follow-up later for net-next.
Fixes: 03485f2adcde ("udpv6: Add lockless sendmsg() support")
Signed-off-by: Kuniyuki Iwashima <kuniyu@amazon.com>
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
2022-10-06 18:53:47 +00:00
|
|
|
void udp_destruct_common(struct sock *sk)
|
udp: implement memory accounting helpers
Avoid using the generic helpers.
Use the receive queue spin lock to protect the memory
accounting operation, both on enqueue and on dequeue.
On dequeue perform partial memory reclaiming, trying to
leave a quantum of forward allocated memory.
On enqueue use a custom helper, to allow some optimizations:
- use a plain spin_lock() variant instead of the slightly
costly spin_lock_irqsave(),
- avoid dst_force check, since the calling code has already
dropped the skb dst
- avoid orphaning the skb, since skb_steal_sock() already did
the work for us
The above needs custom memory reclaiming on shutdown, provided
by the udp_destruct_sock().
v5 -> v6:
- don't orphan the skb on enqueue
v4 -> v5:
- replace the mem_lock with the receive queue spin lock
- ensure that the bh is always allowed to enqueue at least
a skb, even if sk_rcvbuf is exceeded
v3 -> v4:
- reworked memory accunting, simplifying the schema
- provide an helper for both memory scheduling and enqueuing
v1 -> v2:
- use a udp specific destrctor to perform memory reclaiming
- remove a couple of helpers, unneeded after the above cleanup
- do not reclaim memory on dequeue if not under memory
pressure
- reworked the fwd accounting schema to avoid potential
integer overflow
Acked-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-10-21 11:55:46 +00:00
|
|
|
{
|
|
|
|
/* reclaim completely the forward allocated memory */
|
2017-05-16 09:20:14 +00:00
|
|
|
struct udp_sock *up = udp_sk(sk);
|
2016-11-04 10:28:59 +00:00
|
|
|
unsigned int total = 0;
|
|
|
|
struct sk_buff *skb;
|
|
|
|
|
2017-05-16 09:20:14 +00:00
|
|
|
skb_queue_splice_tail_init(&sk->sk_receive_queue, &up->reader_queue);
|
|
|
|
while ((skb = __skb_dequeue(&up->reader_queue)) != NULL) {
|
2016-11-04 10:28:59 +00:00
|
|
|
total += skb->truesize;
|
|
|
|
kfree_skb(skb);
|
|
|
|
}
|
2017-05-16 09:20:15 +00:00
|
|
|
udp_rmem_release(sk, total, 0, true);
|
tcp/udp: Call inet6_destroy_sock() in IPv6 sk->sk_destruct().
Originally, inet6_sk(sk)->XXX were changed under lock_sock(), so we were
able to clean them up by calling inet6_destroy_sock() during the IPv6 ->
IPv4 conversion by IPV6_ADDRFORM. However, commit 03485f2adcde ("udpv6:
Add lockless sendmsg() support") added a lockless memory allocation path,
which could cause a memory leak:
setsockopt(IPV6_ADDRFORM) sendmsg()
+-----------------------+ +-------+
- do_ipv6_setsockopt(sk, ...) - udpv6_sendmsg(sk, ...)
- sockopt_lock_sock(sk) ^._ called via udpv6_prot
- lock_sock(sk) before WRITE_ONCE()
- WRITE_ONCE(sk->sk_prot, &tcp_prot)
- inet6_destroy_sock() - if (!corkreq)
- sockopt_release_sock(sk) - ip6_make_skb(sk, ...)
- release_sock(sk) ^._ lockless fast path for
the non-corking case
- __ip6_append_data(sk, ...)
- ipv6_local_rxpmtu(sk, ...)
- xchg(&np->rxpmtu, skb)
^._ rxpmtu is never freed.
- goto out_no_dst;
- lock_sock(sk)
For now, rxpmtu is only the case, but not to miss the future change
and a similar bug fixed in commit e27326009a3d ("net: ping6: Fix
memleak in ipv6_renew_options()."), let's set a new function to IPv6
sk->sk_destruct() and call inet6_cleanup_sock() there. Since the
conversion does not change sk->sk_destruct(), we can guarantee that
we can clean up IPv6 resources finally.
We can now remove all inet6_destroy_sock() calls from IPv6 protocol
specific ->destroy() functions, but such changes are invasive to
backport. So they can be posted as a follow-up later for net-next.
Fixes: 03485f2adcde ("udpv6: Add lockless sendmsg() support")
Signed-off-by: Kuniyuki Iwashima <kuniyu@amazon.com>
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
2022-10-06 18:53:47 +00:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(udp_destruct_common);
|
2016-11-04 10:28:59 +00:00
|
|
|
|
tcp/udp: Call inet6_destroy_sock() in IPv6 sk->sk_destruct().
Originally, inet6_sk(sk)->XXX were changed under lock_sock(), so we were
able to clean them up by calling inet6_destroy_sock() during the IPv6 ->
IPv4 conversion by IPV6_ADDRFORM. However, commit 03485f2adcde ("udpv6:
Add lockless sendmsg() support") added a lockless memory allocation path,
which could cause a memory leak:
setsockopt(IPV6_ADDRFORM) sendmsg()
+-----------------------+ +-------+
- do_ipv6_setsockopt(sk, ...) - udpv6_sendmsg(sk, ...)
- sockopt_lock_sock(sk) ^._ called via udpv6_prot
- lock_sock(sk) before WRITE_ONCE()
- WRITE_ONCE(sk->sk_prot, &tcp_prot)
- inet6_destroy_sock() - if (!corkreq)
- sockopt_release_sock(sk) - ip6_make_skb(sk, ...)
- release_sock(sk) ^._ lockless fast path for
the non-corking case
- __ip6_append_data(sk, ...)
- ipv6_local_rxpmtu(sk, ...)
- xchg(&np->rxpmtu, skb)
^._ rxpmtu is never freed.
- goto out_no_dst;
- lock_sock(sk)
For now, rxpmtu is only the case, but not to miss the future change
and a similar bug fixed in commit e27326009a3d ("net: ping6: Fix
memleak in ipv6_renew_options()."), let's set a new function to IPv6
sk->sk_destruct() and call inet6_cleanup_sock() there. Since the
conversion does not change sk->sk_destruct(), we can guarantee that
we can clean up IPv6 resources finally.
We can now remove all inet6_destroy_sock() calls from IPv6 protocol
specific ->destroy() functions, but such changes are invasive to
backport. So they can be posted as a follow-up later for net-next.
Fixes: 03485f2adcde ("udpv6: Add lockless sendmsg() support")
Signed-off-by: Kuniyuki Iwashima <kuniyu@amazon.com>
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
2022-10-06 18:53:47 +00:00
|
|
|
static void udp_destruct_sock(struct sock *sk)
|
|
|
|
{
|
|
|
|
udp_destruct_common(sk);
|
udp: implement memory accounting helpers
Avoid using the generic helpers.
Use the receive queue spin lock to protect the memory
accounting operation, both on enqueue and on dequeue.
On dequeue perform partial memory reclaiming, trying to
leave a quantum of forward allocated memory.
On enqueue use a custom helper, to allow some optimizations:
- use a plain spin_lock() variant instead of the slightly
costly spin_lock_irqsave(),
- avoid dst_force check, since the calling code has already
dropped the skb dst
- avoid orphaning the skb, since skb_steal_sock() already did
the work for us
The above needs custom memory reclaiming on shutdown, provided
by the udp_destruct_sock().
v5 -> v6:
- don't orphan the skb on enqueue
v4 -> v5:
- replace the mem_lock with the receive queue spin lock
- ensure that the bh is always allowed to enqueue at least
a skb, even if sk_rcvbuf is exceeded
v3 -> v4:
- reworked memory accunting, simplifying the schema
- provide an helper for both memory scheduling and enqueuing
v1 -> v2:
- use a udp specific destrctor to perform memory reclaiming
- remove a couple of helpers, unneeded after the above cleanup
- do not reclaim memory on dequeue if not under memory
pressure
- reworked the fwd accounting schema to avoid potential
integer overflow
Acked-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-10-21 11:55:46 +00:00
|
|
|
inet_sock_destruct(sk);
|
|
|
|
}
|
|
|
|
|
|
|
|
int udp_init_sock(struct sock *sk)
|
|
|
|
{
|
2022-10-20 17:48:52 +00:00
|
|
|
udp_lib_init_sock(sk);
|
udp: implement memory accounting helpers
Avoid using the generic helpers.
Use the receive queue spin lock to protect the memory
accounting operation, both on enqueue and on dequeue.
On dequeue perform partial memory reclaiming, trying to
leave a quantum of forward allocated memory.
On enqueue use a custom helper, to allow some optimizations:
- use a plain spin_lock() variant instead of the slightly
costly spin_lock_irqsave(),
- avoid dst_force check, since the calling code has already
dropped the skb dst
- avoid orphaning the skb, since skb_steal_sock() already did
the work for us
The above needs custom memory reclaiming on shutdown, provided
by the udp_destruct_sock().
v5 -> v6:
- don't orphan the skb on enqueue
v4 -> v5:
- replace the mem_lock with the receive queue spin lock
- ensure that the bh is always allowed to enqueue at least
a skb, even if sk_rcvbuf is exceeded
v3 -> v4:
- reworked memory accunting, simplifying the schema
- provide an helper for both memory scheduling and enqueuing
v1 -> v2:
- use a udp specific destrctor to perform memory reclaiming
- remove a couple of helpers, unneeded after the above cleanup
- do not reclaim memory on dequeue if not under memory
pressure
- reworked the fwd accounting schema to avoid potential
integer overflow
Acked-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-10-21 11:55:46 +00:00
|
|
|
sk->sk_destruct = udp_destruct_sock;
|
2022-10-21 10:16:39 +00:00
|
|
|
set_bit(SOCK_SUPPORT_ZC, &sk->sk_socket->flags);
|
udp: implement memory accounting helpers
Avoid using the generic helpers.
Use the receive queue spin lock to protect the memory
accounting operation, both on enqueue and on dequeue.
On dequeue perform partial memory reclaiming, trying to
leave a quantum of forward allocated memory.
On enqueue use a custom helper, to allow some optimizations:
- use a plain spin_lock() variant instead of the slightly
costly spin_lock_irqsave(),
- avoid dst_force check, since the calling code has already
dropped the skb dst
- avoid orphaning the skb, since skb_steal_sock() already did
the work for us
The above needs custom memory reclaiming on shutdown, provided
by the udp_destruct_sock().
v5 -> v6:
- don't orphan the skb on enqueue
v4 -> v5:
- replace the mem_lock with the receive queue spin lock
- ensure that the bh is always allowed to enqueue at least
a skb, even if sk_rcvbuf is exceeded
v3 -> v4:
- reworked memory accunting, simplifying the schema
- provide an helper for both memory scheduling and enqueuing
v1 -> v2:
- use a udp specific destrctor to perform memory reclaiming
- remove a couple of helpers, unneeded after the above cleanup
- do not reclaim memory on dequeue if not under memory
pressure
- reworked the fwd accounting schema to avoid potential
integer overflow
Acked-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-10-21 11:55:46 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
void skb_consume_udp(struct sock *sk, struct sk_buff *skb, int len)
|
|
|
|
{
|
|
|
|
if (unlikely(READ_ONCE(sk->sk_peek_off) >= 0)) {
|
|
|
|
bool slow = lock_sock_fast(sk);
|
|
|
|
|
|
|
|
sk_peek_offset_bwd(sk, len);
|
|
|
|
unlock_sock_fast(sk, slow);
|
|
|
|
}
|
2017-06-12 09:23:42 +00:00
|
|
|
|
2017-09-06 12:44:36 +00:00
|
|
|
if (!skb_unref(skb))
|
|
|
|
return;
|
|
|
|
|
2017-07-25 15:57:47 +00:00
|
|
|
/* In the more common cases we cleared the head states previously,
|
|
|
|
* see __udp_queue_rcv_skb().
|
2017-07-18 09:57:55 +00:00
|
|
|
*/
|
2017-07-25 15:57:47 +00:00
|
|
|
if (unlikely(udp_skb_has_head_state(skb)))
|
2017-07-18 09:57:55 +00:00
|
|
|
skb_release_head_state(skb);
|
2017-09-06 12:44:36 +00:00
|
|
|
__consume_stateless_skb(skb);
|
udp: implement memory accounting helpers
Avoid using the generic helpers.
Use the receive queue spin lock to protect the memory
accounting operation, both on enqueue and on dequeue.
On dequeue perform partial memory reclaiming, trying to
leave a quantum of forward allocated memory.
On enqueue use a custom helper, to allow some optimizations:
- use a plain spin_lock() variant instead of the slightly
costly spin_lock_irqsave(),
- avoid dst_force check, since the calling code has already
dropped the skb dst
- avoid orphaning the skb, since skb_steal_sock() already did
the work for us
The above needs custom memory reclaiming on shutdown, provided
by the udp_destruct_sock().
v5 -> v6:
- don't orphan the skb on enqueue
v4 -> v5:
- replace the mem_lock with the receive queue spin lock
- ensure that the bh is always allowed to enqueue at least
a skb, even if sk_rcvbuf is exceeded
v3 -> v4:
- reworked memory accunting, simplifying the schema
- provide an helper for both memory scheduling and enqueuing
v1 -> v2:
- use a udp specific destrctor to perform memory reclaiming
- remove a couple of helpers, unneeded after the above cleanup
- do not reclaim memory on dequeue if not under memory
pressure
- reworked the fwd accounting schema to avoid potential
integer overflow
Acked-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
Acked-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-10-21 11:55:46 +00:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(skb_consume_udp);
|
|
|
|
|
2017-05-16 09:20:14 +00:00
|
|
|
static struct sk_buff *__first_packet_length(struct sock *sk,
|
|
|
|
struct sk_buff_head *rcvq,
|
|
|
|
int *total)
|
|
|
|
{
|
|
|
|
struct sk_buff *skb;
|
|
|
|
|
2017-06-23 12:19:51 +00:00
|
|
|
while ((skb = skb_peek(rcvq)) != NULL) {
|
|
|
|
if (udp_lib_checksum_complete(skb)) {
|
|
|
|
__UDP_INC_STATS(sock_net(sk), UDP_MIB_CSUMERRORS,
|
|
|
|
IS_UDPLITE(sk));
|
|
|
|
__UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS,
|
|
|
|
IS_UDPLITE(sk));
|
|
|
|
atomic_inc(&sk->sk_drops);
|
|
|
|
__skb_unlink(skb, rcvq);
|
|
|
|
*total += skb->truesize;
|
|
|
|
kfree_skb(skb);
|
|
|
|
} else {
|
2019-10-24 18:43:31 +00:00
|
|
|
udp_skb_csum_unnecessary_set(skb);
|
2017-06-23 12:19:51 +00:00
|
|
|
break;
|
|
|
|
}
|
2017-05-16 09:20:14 +00:00
|
|
|
}
|
|
|
|
return skb;
|
|
|
|
}
|
|
|
|
|
2009-10-09 04:43:40 +00:00
|
|
|
/**
|
|
|
|
* first_packet_length - return length of first packet in receive queue
|
|
|
|
* @sk: socket
|
|
|
|
*
|
|
|
|
* Drops all bad checksum frames, until a valid one is found.
|
2016-08-23 20:59:33 +00:00
|
|
|
* Returns the length of found skb, or -1 if none is found.
|
2009-10-09 04:43:40 +00:00
|
|
|
*/
|
2016-08-23 20:59:33 +00:00
|
|
|
static int first_packet_length(struct sock *sk)
|
2009-10-09 04:43:40 +00:00
|
|
|
{
|
2017-05-16 09:20:14 +00:00
|
|
|
struct sk_buff_head *rcvq = &udp_sk(sk)->reader_queue;
|
|
|
|
struct sk_buff_head *sk_queue = &sk->sk_receive_queue;
|
2009-10-09 04:43:40 +00:00
|
|
|
struct sk_buff *skb;
|
2016-11-04 10:28:59 +00:00
|
|
|
int total = 0;
|
2016-08-23 20:59:33 +00:00
|
|
|
int res;
|
2009-10-09 04:43:40 +00:00
|
|
|
|
|
|
|
spin_lock_bh(&rcvq->lock);
|
2017-05-16 09:20:14 +00:00
|
|
|
skb = __first_packet_length(sk, rcvq, &total);
|
2019-10-24 05:44:49 +00:00
|
|
|
if (!skb && !skb_queue_empty_lockless(sk_queue)) {
|
2017-05-16 09:20:14 +00:00
|
|
|
spin_lock(&sk_queue->lock);
|
|
|
|
skb_queue_splice_tail_init(sk_queue, rcvq);
|
|
|
|
spin_unlock(&sk_queue->lock);
|
|
|
|
|
|
|
|
skb = __first_packet_length(sk, rcvq, &total);
|
2009-10-09 04:43:40 +00:00
|
|
|
}
|
2016-08-23 20:59:33 +00:00
|
|
|
res = skb ? skb->len : -1;
|
2016-11-04 10:28:59 +00:00
|
|
|
if (total)
|
2017-05-16 09:20:15 +00:00
|
|
|
udp_rmem_release(sk, total, 1, false);
|
2009-10-09 04:43:40 +00:00
|
|
|
spin_unlock_bh(&rcvq->lock);
|
|
|
|
return res;
|
|
|
|
}
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
* IOCTL requests applicable to the UDP protocol
|
|
|
|
*/
|
2007-02-09 14:24:47 +00:00
|
|
|
|
net: ioctl: Use kernel memory on protocol ioctl callbacks
Most of the ioctls to net protocols operates directly on userspace
argument (arg). Usually doing get_user()/put_user() directly in the
ioctl callback. This is not flexible, because it is hard to reuse these
functions without passing userspace buffers.
Change the "struct proto" ioctls to avoid touching userspace memory and
operate on kernel buffers, i.e., all protocol's ioctl callbacks is
adapted to operate on a kernel memory other than on userspace (so, no
more {put,get}_user() and friends being called in the ioctl callback).
This changes the "struct proto" ioctl format in the following way:
int (*ioctl)(struct sock *sk, int cmd,
- unsigned long arg);
+ int *karg);
(Important to say that this patch does not touch the "struct proto_ops"
protocols)
So, the "karg" argument, which is passed to the ioctl callback, is a
pointer allocated to kernel space memory (inside a function wrapper).
This buffer (karg) may contain input argument (copied from userspace in
a prep function) and it might return a value/buffer, which is copied
back to userspace if necessary. There is not one-size-fits-all format
(that is I am using 'may' above), but basically, there are three type of
ioctls:
1) Do not read from userspace, returns a result to userspace
2) Read an input parameter from userspace, and does not return anything
to userspace
3) Read an input from userspace, and return a buffer to userspace.
The default case (1) (where no input parameter is given, and an "int" is
returned to userspace) encompasses more than 90% of the cases, but there
are two other exceptions. Here is a list of exceptions:
* Protocol RAW:
* cmd = SIOCGETVIFCNT:
* input and output = struct sioc_vif_req
* cmd = SIOCGETSGCNT
* input and output = struct sioc_sg_req
* Explanation: for the SIOCGETVIFCNT case, userspace passes the input
argument, which is struct sioc_vif_req. Then the callback populates
the struct, which is copied back to userspace.
* Protocol RAW6:
* cmd = SIOCGETMIFCNT_IN6
* input and output = struct sioc_mif_req6
* cmd = SIOCGETSGCNT_IN6
* input and output = struct sioc_sg_req6
* Protocol PHONET:
* cmd == SIOCPNADDRESOURCE | SIOCPNDELRESOURCE
* input int (4 bytes)
* Nothing is copied back to userspace.
For the exception cases, functions sock_sk_ioctl_inout() will
copy the userspace input, and copy it back to kernel space.
The wrapper that prepare the buffer and put the buffer back to user is
sk_ioctl(), so, instead of calling sk->sk_prot->ioctl(), the callee now
calls sk_ioctl(), which will handle all cases.
Signed-off-by: Breno Leitao <leitao@debian.org>
Reviewed-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: David Ahern <dsahern@kernel.org>
Reviewed-by: Kuniyuki Iwashima <kuniyu@amazon.com>
Link: https://lore.kernel.org/r/20230609152800.830401-1-leitao@debian.org
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
2023-06-09 15:27:42 +00:00
|
|
|
int udp_ioctl(struct sock *sk, int cmd, int *karg)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2007-03-09 04:41:55 +00:00
|
|
|
switch (cmd) {
|
|
|
|
case SIOCOUTQ:
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
net: ioctl: Use kernel memory on protocol ioctl callbacks
Most of the ioctls to net protocols operates directly on userspace
argument (arg). Usually doing get_user()/put_user() directly in the
ioctl callback. This is not flexible, because it is hard to reuse these
functions without passing userspace buffers.
Change the "struct proto" ioctls to avoid touching userspace memory and
operate on kernel buffers, i.e., all protocol's ioctl callbacks is
adapted to operate on a kernel memory other than on userspace (so, no
more {put,get}_user() and friends being called in the ioctl callback).
This changes the "struct proto" ioctl format in the following way:
int (*ioctl)(struct sock *sk, int cmd,
- unsigned long arg);
+ int *karg);
(Important to say that this patch does not touch the "struct proto_ops"
protocols)
So, the "karg" argument, which is passed to the ioctl callback, is a
pointer allocated to kernel space memory (inside a function wrapper).
This buffer (karg) may contain input argument (copied from userspace in
a prep function) and it might return a value/buffer, which is copied
back to userspace if necessary. There is not one-size-fits-all format
(that is I am using 'may' above), but basically, there are three type of
ioctls:
1) Do not read from userspace, returns a result to userspace
2) Read an input parameter from userspace, and does not return anything
to userspace
3) Read an input from userspace, and return a buffer to userspace.
The default case (1) (where no input parameter is given, and an "int" is
returned to userspace) encompasses more than 90% of the cases, but there
are two other exceptions. Here is a list of exceptions:
* Protocol RAW:
* cmd = SIOCGETVIFCNT:
* input and output = struct sioc_vif_req
* cmd = SIOCGETSGCNT
* input and output = struct sioc_sg_req
* Explanation: for the SIOCGETVIFCNT case, userspace passes the input
argument, which is struct sioc_vif_req. Then the callback populates
the struct, which is copied back to userspace.
* Protocol RAW6:
* cmd = SIOCGETMIFCNT_IN6
* input and output = struct sioc_mif_req6
* cmd = SIOCGETSGCNT_IN6
* input and output = struct sioc_sg_req6
* Protocol PHONET:
* cmd == SIOCPNADDRESOURCE | SIOCPNDELRESOURCE
* input int (4 bytes)
* Nothing is copied back to userspace.
For the exception cases, functions sock_sk_ioctl_inout() will
copy the userspace input, and copy it back to kernel space.
The wrapper that prepare the buffer and put the buffer back to user is
sk_ioctl(), so, instead of calling sk->sk_prot->ioctl(), the callee now
calls sk_ioctl(), which will handle all cases.
Signed-off-by: Breno Leitao <leitao@debian.org>
Reviewed-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: David Ahern <dsahern@kernel.org>
Reviewed-by: Kuniyuki Iwashima <kuniyu@amazon.com>
Link: https://lore.kernel.org/r/20230609152800.830401-1-leitao@debian.org
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
2023-06-09 15:27:42 +00:00
|
|
|
*karg = sk_wmem_alloc_get(sk);
|
|
|
|
return 0;
|
2007-03-09 04:41:55 +00:00
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2007-03-09 04:41:55 +00:00
|
|
|
case SIOCINQ:
|
|
|
|
{
|
net: ioctl: Use kernel memory on protocol ioctl callbacks
Most of the ioctls to net protocols operates directly on userspace
argument (arg). Usually doing get_user()/put_user() directly in the
ioctl callback. This is not flexible, because it is hard to reuse these
functions without passing userspace buffers.
Change the "struct proto" ioctls to avoid touching userspace memory and
operate on kernel buffers, i.e., all protocol's ioctl callbacks is
adapted to operate on a kernel memory other than on userspace (so, no
more {put,get}_user() and friends being called in the ioctl callback).
This changes the "struct proto" ioctl format in the following way:
int (*ioctl)(struct sock *sk, int cmd,
- unsigned long arg);
+ int *karg);
(Important to say that this patch does not touch the "struct proto_ops"
protocols)
So, the "karg" argument, which is passed to the ioctl callback, is a
pointer allocated to kernel space memory (inside a function wrapper).
This buffer (karg) may contain input argument (copied from userspace in
a prep function) and it might return a value/buffer, which is copied
back to userspace if necessary. There is not one-size-fits-all format
(that is I am using 'may' above), but basically, there are three type of
ioctls:
1) Do not read from userspace, returns a result to userspace
2) Read an input parameter from userspace, and does not return anything
to userspace
3) Read an input from userspace, and return a buffer to userspace.
The default case (1) (where no input parameter is given, and an "int" is
returned to userspace) encompasses more than 90% of the cases, but there
are two other exceptions. Here is a list of exceptions:
* Protocol RAW:
* cmd = SIOCGETVIFCNT:
* input and output = struct sioc_vif_req
* cmd = SIOCGETSGCNT
* input and output = struct sioc_sg_req
* Explanation: for the SIOCGETVIFCNT case, userspace passes the input
argument, which is struct sioc_vif_req. Then the callback populates
the struct, which is copied back to userspace.
* Protocol RAW6:
* cmd = SIOCGETMIFCNT_IN6
* input and output = struct sioc_mif_req6
* cmd = SIOCGETSGCNT_IN6
* input and output = struct sioc_sg_req6
* Protocol PHONET:
* cmd == SIOCPNADDRESOURCE | SIOCPNDELRESOURCE
* input int (4 bytes)
* Nothing is copied back to userspace.
For the exception cases, functions sock_sk_ioctl_inout() will
copy the userspace input, and copy it back to kernel space.
The wrapper that prepare the buffer and put the buffer back to user is
sk_ioctl(), so, instead of calling sk->sk_prot->ioctl(), the callee now
calls sk_ioctl(), which will handle all cases.
Signed-off-by: Breno Leitao <leitao@debian.org>
Reviewed-by: Willem de Bruijn <willemb@google.com>
Reviewed-by: David Ahern <dsahern@kernel.org>
Reviewed-by: Kuniyuki Iwashima <kuniyu@amazon.com>
Link: https://lore.kernel.org/r/20230609152800.830401-1-leitao@debian.org
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
2023-06-09 15:27:42 +00:00
|
|
|
*karg = max_t(int, 0, first_packet_length(sk));
|
|
|
|
return 0;
|
2007-03-09 04:41:55 +00:00
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2007-03-09 04:41:55 +00:00
|
|
|
default:
|
|
|
|
return -ENOIOCTLCMD;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2007-03-09 04:41:55 +00:00
|
|
|
|
|
|
|
return 0;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2009-07-17 00:26:32 +00:00
|
|
|
EXPORT_SYMBOL(udp_ioctl);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2017-05-16 09:20:14 +00:00
|
|
|
struct sk_buff *__skb_recv_udp(struct sock *sk, unsigned int flags,
|
net: remove noblock parameter from recvmsg() entities
The internal recvmsg() functions have two parameters 'flags' and 'noblock'
that were merged inside skb_recv_datagram(). As a follow up patch to commit
f4b41f062c42 ("net: remove noblock parameter from skb_recv_datagram()")
this patch removes the separate 'noblock' parameter for recvmsg().
Analogue to the referenced patch for skb_recv_datagram() the 'flags' and
'noblock' parameters are unnecessarily split up with e.g.
err = sk->sk_prot->recvmsg(sk, msg, size, flags & MSG_DONTWAIT,
flags & ~MSG_DONTWAIT, &addr_len);
or in
err = INDIRECT_CALL_2(sk->sk_prot->recvmsg, tcp_recvmsg, udp_recvmsg,
sk, msg, size, flags & MSG_DONTWAIT,
flags & ~MSG_DONTWAIT, &addr_len);
instead of simply using only flags all the time and check for MSG_DONTWAIT
where needed (to preserve for the formerly separated no(n)block condition).
Signed-off-by: Oliver Hartkopp <socketcan@hartkopp.net>
Link: https://lore.kernel.org/r/20220411124955.154876-1-socketcan@hartkopp.net
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
2022-04-11 12:49:55 +00:00
|
|
|
int *off, int *err)
|
2017-05-16 09:20:14 +00:00
|
|
|
{
|
|
|
|
struct sk_buff_head *sk_queue = &sk->sk_receive_queue;
|
|
|
|
struct sk_buff_head *queue;
|
|
|
|
struct sk_buff *last;
|
|
|
|
long timeo;
|
|
|
|
int error;
|
|
|
|
|
|
|
|
queue = &udp_sk(sk)->reader_queue;
|
|
|
|
timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT);
|
|
|
|
do {
|
|
|
|
struct sk_buff *skb;
|
|
|
|
|
|
|
|
error = sock_error(sk);
|
|
|
|
if (error)
|
|
|
|
break;
|
|
|
|
|
|
|
|
error = -EAGAIN;
|
|
|
|
do {
|
|
|
|
spin_lock_bh(&queue->lock);
|
2020-02-28 13:45:22 +00:00
|
|
|
skb = __skb_try_recv_from_queue(sk, queue, flags, off,
|
|
|
|
err, &last);
|
2017-05-16 09:20:14 +00:00
|
|
|
if (skb) {
|
2020-02-28 13:45:22 +00:00
|
|
|
if (!(flags & MSG_PEEK))
|
|
|
|
udp_skb_destructor(sk, skb);
|
2017-05-16 09:20:14 +00:00
|
|
|
spin_unlock_bh(&queue->lock);
|
|
|
|
return skb;
|
|
|
|
}
|
|
|
|
|
2019-10-24 05:44:49 +00:00
|
|
|
if (skb_queue_empty_lockless(sk_queue)) {
|
2017-05-16 09:20:14 +00:00
|
|
|
spin_unlock_bh(&queue->lock);
|
|
|
|
goto busy_check;
|
|
|
|
}
|
|
|
|
|
2017-05-16 09:20:15 +00:00
|
|
|
/* refill the reader queue and walk it again
|
|
|
|
* keep both queues locked to avoid re-acquiring
|
|
|
|
* the sk_receive_queue lock if fwd memory scheduling
|
|
|
|
* is needed.
|
|
|
|
*/
|
2017-05-16 09:20:14 +00:00
|
|
|
spin_lock(&sk_queue->lock);
|
|
|
|
skb_queue_splice_tail_init(sk_queue, queue);
|
|
|
|
|
2020-02-28 13:45:22 +00:00
|
|
|
skb = __skb_try_recv_from_queue(sk, queue, flags, off,
|
|
|
|
err, &last);
|
|
|
|
if (skb && !(flags & MSG_PEEK))
|
|
|
|
udp_skb_dtor_locked(sk, skb);
|
2017-05-16 09:20:15 +00:00
|
|
|
spin_unlock(&sk_queue->lock);
|
2017-05-16 09:20:14 +00:00
|
|
|
spin_unlock_bh(&queue->lock);
|
2017-05-17 18:39:05 +00:00
|
|
|
if (skb)
|
2017-05-16 09:20:14 +00:00
|
|
|
return skb;
|
|
|
|
|
|
|
|
busy_check:
|
|
|
|
if (!sk_can_busy_loop(sk))
|
|
|
|
break;
|
|
|
|
|
|
|
|
sk_busy_loop(sk, flags & MSG_DONTWAIT);
|
2019-10-24 05:44:49 +00:00
|
|
|
} while (!skb_queue_empty_lockless(sk_queue));
|
2017-05-16 09:20:14 +00:00
|
|
|
|
|
|
|
/* sk_queue is empty, reader_queue may contain peeked packets */
|
|
|
|
} while (timeo &&
|
2019-11-25 13:48:57 +00:00
|
|
|
!__skb_wait_for_more_packets(sk, &sk->sk_receive_queue,
|
|
|
|
&error, &timeo,
|
2017-05-16 09:20:14 +00:00
|
|
|
(struct sk_buff *)sk_queue));
|
|
|
|
|
|
|
|
*err = error;
|
|
|
|
return NULL;
|
|
|
|
}
|
2018-10-04 11:37:32 +00:00
|
|
|
EXPORT_SYMBOL(__skb_recv_udp);
|
2017-05-16 09:20:14 +00:00
|
|
|
|
2022-06-15 16:20:12 +00:00
|
|
|
int udp_read_skb(struct sock *sk, skb_read_actor_t recv_actor)
|
2021-03-31 02:32:32 +00:00
|
|
|
{
|
2022-09-23 04:59:13 +00:00
|
|
|
struct sk_buff *skb;
|
2023-05-23 02:56:05 +00:00
|
|
|
int err;
|
2021-11-15 04:40:06 +00:00
|
|
|
|
2022-09-23 04:59:13 +00:00
|
|
|
try_again:
|
|
|
|
skb = skb_recv_udp(sk, MSG_DONTWAIT, &err);
|
|
|
|
if (!skb)
|
|
|
|
return err;
|
2021-11-15 04:40:06 +00:00
|
|
|
|
2022-09-23 04:59:13 +00:00
|
|
|
if (udp_lib_checksum_complete(skb)) {
|
|
|
|
int is_udplite = IS_UDPLITE(sk);
|
|
|
|
struct net *net = sock_net(sk);
|
2021-03-31 02:32:32 +00:00
|
|
|
|
2022-09-23 04:59:13 +00:00
|
|
|
__UDP_INC_STATS(net, UDP_MIB_CSUMERRORS, is_udplite);
|
|
|
|
__UDP_INC_STATS(net, UDP_MIB_INERRORS, is_udplite);
|
|
|
|
atomic_inc(&sk->sk_drops);
|
2021-06-15 02:13:37 +00:00
|
|
|
kfree_skb(skb);
|
2022-09-23 04:59:13 +00:00
|
|
|
goto try_again;
|
2021-03-31 02:32:32 +00:00
|
|
|
}
|
|
|
|
|
2022-09-23 04:59:13 +00:00
|
|
|
WARN_ON_ONCE(!skb_set_owner_sk_safe(skb, sk));
|
2023-05-23 02:56:05 +00:00
|
|
|
return recv_actor(sk, skb);
|
2021-03-31 02:32:32 +00:00
|
|
|
}
|
2022-06-15 16:20:12 +00:00
|
|
|
EXPORT_SYMBOL(udp_read_skb);
|
2021-03-31 02:32:32 +00:00
|
|
|
|
2008-03-07 00:22:02 +00:00
|
|
|
/*
|
|
|
|
* This should be easy, if there is something there we
|
|
|
|
* return it, otherwise we block.
|
|
|
|
*/
|
|
|
|
|
net: remove noblock parameter from recvmsg() entities
The internal recvmsg() functions have two parameters 'flags' and 'noblock'
that were merged inside skb_recv_datagram(). As a follow up patch to commit
f4b41f062c42 ("net: remove noblock parameter from skb_recv_datagram()")
this patch removes the separate 'noblock' parameter for recvmsg().
Analogue to the referenced patch for skb_recv_datagram() the 'flags' and
'noblock' parameters are unnecessarily split up with e.g.
err = sk->sk_prot->recvmsg(sk, msg, size, flags & MSG_DONTWAIT,
flags & ~MSG_DONTWAIT, &addr_len);
or in
err = INDIRECT_CALL_2(sk->sk_prot->recvmsg, tcp_recvmsg, udp_recvmsg,
sk, msg, size, flags & MSG_DONTWAIT,
flags & ~MSG_DONTWAIT, &addr_len);
instead of simply using only flags all the time and check for MSG_DONTWAIT
where needed (to preserve for the formerly separated no(n)block condition).
Signed-off-by: Oliver Hartkopp <socketcan@hartkopp.net>
Link: https://lore.kernel.org/r/20220411124955.154876-1-socketcan@hartkopp.net
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
2022-04-11 12:49:55 +00:00
|
|
|
int udp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int flags,
|
|
|
|
int *addr_len)
|
2008-03-07 00:22:02 +00:00
|
|
|
{
|
|
|
|
struct inet_sock *inet = inet_sk(sk);
|
2014-01-17 21:53:15 +00:00
|
|
|
DECLARE_SOCKADDR(struct sockaddr_in *, sin, msg->msg_name);
|
2008-03-07 00:22:02 +00:00
|
|
|
struct sk_buff *skb;
|
2011-12-01 19:12:55 +00:00
|
|
|
unsigned int ulen, copied;
|
2019-04-08 08:15:59 +00:00
|
|
|
int off, err, peeking = flags & MSG_PEEK;
|
2008-03-07 00:22:02 +00:00
|
|
|
int is_udplite = IS_UDPLITE(sk);
|
udp: properly support MSG_PEEK with truncated buffers
Backport of this upstream commit into stable kernels :
89c22d8c3b27 ("net: Fix skb csum races when peeking")
exposed a bug in udp stack vs MSG_PEEK support, when user provides
a buffer smaller than skb payload.
In this case,
skb_copy_and_csum_datagram_iovec(skb, sizeof(struct udphdr),
msg->msg_iov);
returns -EFAULT.
This bug does not happen in upstream kernels since Al Viro did a great
job to replace this into :
skb_copy_and_csum_datagram_msg(skb, sizeof(struct udphdr), msg);
This variant is safe vs short buffers.
For the time being, instead reverting Herbert Xu patch and add back
skb->ip_summed invalid changes, simply store the result of
udp_lib_checksum_complete() so that we avoid computing the checksum a
second time, and avoid the problematic
skb_copy_and_csum_datagram_iovec() call.
This patch can be applied on recent kernels as it avoids a double
checksumming, then backported to stable kernels as a bug fix.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Acked-by: Herbert Xu <herbert@gondor.apana.org.au>
Signed-off-by: David S. Miller <davem@davemloft.net>
2015-12-30 13:51:12 +00:00
|
|
|
bool checksum_valid = false;
|
2008-03-07 00:22:02 +00:00
|
|
|
|
|
|
|
if (flags & MSG_ERRQUEUE)
|
2013-11-22 23:46:12 +00:00
|
|
|
return ip_recv_error(sk, msg, len, addr_len);
|
2008-03-07 00:22:02 +00:00
|
|
|
|
|
|
|
try_again:
|
datagram: When peeking datagrams with offset < 0 don't skip empty skbs
Due to commit e6afc8ace6dd5cef5e812f26c72579da8806f5ac ("udp: remove
headers from UDP packets before queueing"), when udp packets are being
peeked the requested extra offset is always 0 as there is no need to skip
the udp header. However, when the offset is 0 and the next skb is
of length 0, it is only returned once. The behaviour can be seen with
the following python script:
from socket import *;
f=socket(AF_INET6, SOCK_DGRAM | SOCK_NONBLOCK, 0);
g=socket(AF_INET6, SOCK_DGRAM | SOCK_NONBLOCK, 0);
f.bind(('::', 0));
addr=('::1', f.getsockname()[1]);
g.sendto(b'', addr)
g.sendto(b'b', addr)
print(f.recvfrom(10, MSG_PEEK));
print(f.recvfrom(10, MSG_PEEK));
Where the expected output should be the empty string twice.
Instead, make sk_peek_offset return negative values, and pass those values
to __skb_try_recv_datagram/__skb_try_recv_from_queue. If the passed offset
to __skb_try_recv_from_queue is negative, the checked skb is never skipped.
__skb_try_recv_from_queue will then ensure the offset is reset back to 0
if a peek is requested without an offset, unless no packets are found.
Also simplify the if condition in __skb_try_recv_from_queue. If _off is
greater then 0, and off is greater then or equal to skb->len, then
(_off || skb->len) must always be true assuming skb->len >= 0 is always
true.
Also remove a redundant check around a call to sk_peek_offset in af_unix.c,
as it double checked if MSG_PEEK was set in the flags.
V2:
- Moved the negative fixup into __skb_try_recv_from_queue, and remove now
redundant checks
- Fix peeking in udp{,v6}_recvmsg to report the right value when the
offset is 0
V3:
- Marked new branch in __skb_try_recv_from_queue as unlikely.
Signed-off-by: Matthew Dawson <matthew@mjdsystems.ca>
Acked-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2017-08-18 19:04:54 +00:00
|
|
|
off = sk_peek_offset(sk, flags);
|
net: remove noblock parameter from recvmsg() entities
The internal recvmsg() functions have two parameters 'flags' and 'noblock'
that were merged inside skb_recv_datagram(). As a follow up patch to commit
f4b41f062c42 ("net: remove noblock parameter from skb_recv_datagram()")
this patch removes the separate 'noblock' parameter for recvmsg().
Analogue to the referenced patch for skb_recv_datagram() the 'flags' and
'noblock' parameters are unnecessarily split up with e.g.
err = sk->sk_prot->recvmsg(sk, msg, size, flags & MSG_DONTWAIT,
flags & ~MSG_DONTWAIT, &addr_len);
or in
err = INDIRECT_CALL_2(sk->sk_prot->recvmsg, tcp_recvmsg, udp_recvmsg,
sk, msg, size, flags & MSG_DONTWAIT,
flags & ~MSG_DONTWAIT, &addr_len);
instead of simply using only flags all the time and check for MSG_DONTWAIT
where needed (to preserve for the formerly separated no(n)block condition).
Signed-off-by: Oliver Hartkopp <socketcan@hartkopp.net>
Link: https://lore.kernel.org/r/20220411124955.154876-1-socketcan@hartkopp.net
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
2022-04-11 12:49:55 +00:00
|
|
|
skb = __skb_recv_udp(sk, flags, &off, &err);
|
2008-03-07 00:22:02 +00:00
|
|
|
if (!skb)
|
2016-04-05 16:41:16 +00:00
|
|
|
return err;
|
2008-03-07 00:22:02 +00:00
|
|
|
|
2017-06-12 09:23:43 +00:00
|
|
|
ulen = udp_skb_len(skb);
|
2011-12-01 19:12:55 +00:00
|
|
|
copied = len;
|
2016-04-05 16:41:16 +00:00
|
|
|
if (copied > ulen - off)
|
|
|
|
copied = ulen - off;
|
2011-12-01 19:12:55 +00:00
|
|
|
else if (copied < ulen)
|
2008-03-07 00:22:02 +00:00
|
|
|
msg->msg_flags |= MSG_TRUNC;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If checksum is needed at all, try to do it while copying the
|
|
|
|
* data. If the data is truncated, or if we only want a partial
|
|
|
|
* coverage checksum (UDP-Lite), do it before the copy.
|
|
|
|
*/
|
|
|
|
|
2016-11-19 01:18:03 +00:00
|
|
|
if (copied < ulen || peeking ||
|
|
|
|
(is_udplite && UDP_SKB_CB(skb)->partial_cov)) {
|
2017-06-12 09:23:43 +00:00
|
|
|
checksum_valid = udp_skb_csum_unnecessary(skb) ||
|
|
|
|
!__udp_lib_checksum_complete(skb);
|
udp: properly support MSG_PEEK with truncated buffers
Backport of this upstream commit into stable kernels :
89c22d8c3b27 ("net: Fix skb csum races when peeking")
exposed a bug in udp stack vs MSG_PEEK support, when user provides
a buffer smaller than skb payload.
In this case,
skb_copy_and_csum_datagram_iovec(skb, sizeof(struct udphdr),
msg->msg_iov);
returns -EFAULT.
This bug does not happen in upstream kernels since Al Viro did a great
job to replace this into :
skb_copy_and_csum_datagram_msg(skb, sizeof(struct udphdr), msg);
This variant is safe vs short buffers.
For the time being, instead reverting Herbert Xu patch and add back
skb->ip_summed invalid changes, simply store the result of
udp_lib_checksum_complete() so that we avoid computing the checksum a
second time, and avoid the problematic
skb_copy_and_csum_datagram_iovec() call.
This patch can be applied on recent kernels as it avoids a double
checksumming, then backported to stable kernels as a bug fix.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Acked-by: Herbert Xu <herbert@gondor.apana.org.au>
Signed-off-by: David S. Miller <davem@davemloft.net>
2015-12-30 13:51:12 +00:00
|
|
|
if (!checksum_valid)
|
2008-03-07 00:22:02 +00:00
|
|
|
goto csum_copy_err;
|
|
|
|
}
|
|
|
|
|
2017-06-12 09:23:43 +00:00
|
|
|
if (checksum_valid || udp_skb_csum_unnecessary(skb)) {
|
|
|
|
if (udp_skb_is_linear(skb))
|
|
|
|
err = copy_linear_skb(skb, copied, off, &msg->msg_iter);
|
|
|
|
else
|
|
|
|
err = skb_copy_datagram_msg(skb, off, msg, copied);
|
|
|
|
} else {
|
2016-04-05 16:41:16 +00:00
|
|
|
err = skb_copy_and_csum_datagram_msg(skb, off, msg);
|
2008-03-07 00:22:02 +00:00
|
|
|
|
|
|
|
if (err == -EINVAL)
|
|
|
|
goto csum_copy_err;
|
|
|
|
}
|
|
|
|
|
2012-06-27 00:23:44 +00:00
|
|
|
if (unlikely(err)) {
|
2019-04-08 08:15:59 +00:00
|
|
|
if (!peeking) {
|
2012-09-05 23:34:44 +00:00
|
|
|
atomic_inc(&sk->sk_drops);
|
net: snmp: kill various STATS_USER() helpers
In the old days (before linux-3.0), SNMP counters were duplicated,
one for user context, and one for BH context.
After commit 8f0ea0fe3a03 ("snmp: reduce percpu needs by 50%")
we have a single copy, and what really matters is preemption being
enabled or disabled, since we use this_cpu_inc() or __this_cpu_inc()
respectively.
We therefore kill SNMP_INC_STATS_USER(), SNMP_ADD_STATS_USER(),
NET_INC_STATS_USER(), NET_ADD_STATS_USER(), SCTP_INC_STATS_USER(),
SNMP_INC_STATS64_USER(), SNMP_ADD_STATS64_USER(), TCP_ADD_STATS_USER(),
UDP_INC_STATS_USER(), UDP6_INC_STATS_USER(), and XFRM_INC_STATS_USER()
Following patches will rename __BH helpers to make clear their
usage is not tied to BH being disabled.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-04-27 23:44:27 +00:00
|
|
|
UDP_INC_STATS(sock_net(sk),
|
|
|
|
UDP_MIB_INERRORS, is_udplite);
|
2012-09-05 23:34:44 +00:00
|
|
|
}
|
2016-10-21 11:55:47 +00:00
|
|
|
kfree_skb(skb);
|
2016-04-05 16:41:16 +00:00
|
|
|
return err;
|
2012-06-27 00:23:44 +00:00
|
|
|
}
|
2008-03-07 00:22:02 +00:00
|
|
|
|
2019-04-08 08:15:59 +00:00
|
|
|
if (!peeking)
|
net: snmp: kill various STATS_USER() helpers
In the old days (before linux-3.0), SNMP counters were duplicated,
one for user context, and one for BH context.
After commit 8f0ea0fe3a03 ("snmp: reduce percpu needs by 50%")
we have a single copy, and what really matters is preemption being
enabled or disabled, since we use this_cpu_inc() or __this_cpu_inc()
respectively.
We therefore kill SNMP_INC_STATS_USER(), SNMP_ADD_STATS_USER(),
NET_INC_STATS_USER(), NET_ADD_STATS_USER(), SCTP_INC_STATS_USER(),
SNMP_INC_STATS64_USER(), SNMP_ADD_STATS64_USER(), TCP_ADD_STATS_USER(),
UDP_INC_STATS_USER(), UDP6_INC_STATS_USER(), and XFRM_INC_STATS_USER()
Following patches will rename __BH helpers to make clear their
usage is not tied to BH being disabled.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-04-27 23:44:27 +00:00
|
|
|
UDP_INC_STATS(sock_net(sk),
|
|
|
|
UDP_MIB_INDATAGRAMS, is_udplite);
|
2008-03-07 00:22:02 +00:00
|
|
|
|
2022-04-27 20:02:37 +00:00
|
|
|
sock_recv_cmsgs(msg, sk, skb);
|
2008-03-07 00:22:02 +00:00
|
|
|
|
|
|
|
/* Copy the address. */
|
2009-07-17 00:26:32 +00:00
|
|
|
if (sin) {
|
2008-03-07 00:22:02 +00:00
|
|
|
sin->sin_family = AF_INET;
|
|
|
|
sin->sin_port = udp_hdr(skb)->source;
|
|
|
|
sin->sin_addr.s_addr = ip_hdr(skb)->saddr;
|
|
|
|
memset(sin->sin_zero, 0, sizeof(sin->sin_zero));
|
2013-11-18 03:20:45 +00:00
|
|
|
*addr_len = sizeof(*sin);
|
bpf: fix unconnected udp hooks
Intention of cgroup bind/connect/sendmsg BPF hooks is to act transparently
to applications as also stated in original motivation in 7828f20e3779 ("Merge
branch 'bpf-cgroup-bind-connect'"). When recently integrating the latter
two hooks into Cilium to enable host based load-balancing with Kubernetes,
I ran into the issue that pods couldn't start up as DNS got broken. Kubernetes
typically sets up DNS as a service and is thus subject to load-balancing.
Upon further debugging, it turns out that the cgroupv2 sendmsg BPF hooks API
is currently insufficient and thus not usable as-is for standard applications
shipped with most distros. To break down the issue we ran into with a simple
example:
# cat /etc/resolv.conf
nameserver 147.75.207.207
nameserver 147.75.207.208
For the purpose of a simple test, we set up above IPs as service IPs and
transparently redirect traffic to a different DNS backend server for that
node:
# cilium service list
ID Frontend Backend
1 147.75.207.207:53 1 => 8.8.8.8:53
2 147.75.207.208:53 1 => 8.8.8.8:53
The attached BPF program is basically selecting one of the backends if the
service IP/port matches on the cgroup hook. DNS breaks here, because the
hooks are not transparent enough to applications which have built-in msg_name
address checks:
# nslookup 1.1.1.1
;; reply from unexpected source: 8.8.8.8#53, expected 147.75.207.207#53
;; reply from unexpected source: 8.8.8.8#53, expected 147.75.207.208#53
;; reply from unexpected source: 8.8.8.8#53, expected 147.75.207.207#53
[...]
;; connection timed out; no servers could be reached
# dig 1.1.1.1
;; reply from unexpected source: 8.8.8.8#53, expected 147.75.207.207#53
;; reply from unexpected source: 8.8.8.8#53, expected 147.75.207.208#53
;; reply from unexpected source: 8.8.8.8#53, expected 147.75.207.207#53
[...]
; <<>> DiG 9.11.3-1ubuntu1.7-Ubuntu <<>> 1.1.1.1
;; global options: +cmd
;; connection timed out; no servers could be reached
For comparison, if none of the service IPs is used, and we tell nslookup
to use 8.8.8.8 directly it works just fine, of course:
# nslookup 1.1.1.1 8.8.8.8
1.1.1.1.in-addr.arpa name = one.one.one.one.
In order to fix this and thus act more transparent to the application,
this needs reverse translation on recvmsg() side. A minimal fix for this
API is to add similar recvmsg() hooks behind the BPF cgroups static key
such that the program can track state and replace the current sockaddr_in{,6}
with the original service IP. From BPF side, this basically tracks the
service tuple plus socket cookie in an LRU map where the reverse NAT can
then be retrieved via map value as one example. Side-note: the BPF cgroups
static key should be converted to a per-hook static key in future.
Same example after this fix:
# cilium service list
ID Frontend Backend
1 147.75.207.207:53 1 => 8.8.8.8:53
2 147.75.207.208:53 1 => 8.8.8.8:53
Lookups work fine now:
# nslookup 1.1.1.1
1.1.1.1.in-addr.arpa name = one.one.one.one.
Authoritative answers can be found from:
# dig 1.1.1.1
; <<>> DiG 9.11.3-1ubuntu1.7-Ubuntu <<>> 1.1.1.1
;; global options: +cmd
;; Got answer:
;; ->>HEADER<<- opcode: QUERY, status: NXDOMAIN, id: 51550
;; flags: qr rd ra ad; QUERY: 1, ANSWER: 0, AUTHORITY: 1, ADDITIONAL: 1
;; OPT PSEUDOSECTION:
; EDNS: version: 0, flags:; udp: 512
;; QUESTION SECTION:
;1.1.1.1. IN A
;; AUTHORITY SECTION:
. 23426 IN SOA a.root-servers.net. nstld.verisign-grs.com. 2019052001 1800 900 604800 86400
;; Query time: 17 msec
;; SERVER: 147.75.207.207#53(147.75.207.207)
;; WHEN: Tue May 21 12:59:38 UTC 2019
;; MSG SIZE rcvd: 111
And from an actual packet level it shows that we're using the back end
server when talking via 147.75.207.20{7,8} front end:
# tcpdump -i any udp
[...]
12:59:52.698732 IP foo.42011 > google-public-dns-a.google.com.domain: 18803+ PTR? 1.1.1.1.in-addr.arpa. (38)
12:59:52.698735 IP foo.42011 > google-public-dns-a.google.com.domain: 18803+ PTR? 1.1.1.1.in-addr.arpa. (38)
12:59:52.701208 IP google-public-dns-a.google.com.domain > foo.42011: 18803 1/0/0 PTR one.one.one.one. (67)
12:59:52.701208 IP google-public-dns-a.google.com.domain > foo.42011: 18803 1/0/0 PTR one.one.one.one. (67)
[...]
In order to be flexible and to have same semantics as in sendmsg BPF
programs, we only allow return codes in [1,1] range. In the sendmsg case
the program is called if msg->msg_name is present which can be the case
in both, connected and unconnected UDP.
The former only relies on the sockaddr_in{,6} passed via connect(2) if
passed msg->msg_name was NULL. Therefore, on recvmsg side, we act in similar
way to call into the BPF program whenever a non-NULL msg->msg_name was
passed independent of sk->sk_state being TCP_ESTABLISHED or not. Note
that for TCP case, the msg->msg_name is ignored in the regular recvmsg
path and therefore not relevant.
For the case of ip{,v6}_recv_error() paths, picked up via MSG_ERRQUEUE,
the hook is not called. This is intentional as it aligns with the same
semantics as in case of TCP cgroup BPF hooks right now. This might be
better addressed in future through a different bpf_attach_type such
that this case can be distinguished from the regular recvmsg paths,
for example.
Fixes: 1cedee13d25a ("bpf: Hooks for sys_sendmsg")
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Acked-by: Andrey Ignatov <rdna@fb.com>
Acked-by: Martin KaFai Lau <kafai@fb.com>
Acked-by: Martynas Pumputis <m@lambda.lt>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
2019-06-06 23:48:57 +00:00
|
|
|
|
2021-01-15 16:35:01 +00:00
|
|
|
BPF_CGROUP_RUN_PROG_UDP4_RECVMSG_LOCK(sk,
|
2023-10-11 18:51:04 +00:00
|
|
|
(struct sockaddr *)sin,
|
|
|
|
addr_len);
|
2008-03-07 00:22:02 +00:00
|
|
|
}
|
2018-11-07 11:38:30 +00:00
|
|
|
|
2023-09-12 09:17:24 +00:00
|
|
|
if (udp_test_bit(GRO_ENABLED, sk))
|
2018-11-07 11:38:30 +00:00
|
|
|
udp_cmsg_recv(msg, sk, skb);
|
|
|
|
|
2023-08-16 08:15:33 +00:00
|
|
|
if (inet_cmsg_flags(inet))
|
2016-11-04 10:28:58 +00:00
|
|
|
ip_cmsg_recv_offset(msg, sk, skb, sizeof(struct udphdr), off);
|
2008-03-07 00:22:02 +00:00
|
|
|
|
2011-12-01 19:12:55 +00:00
|
|
|
err = copied;
|
2008-03-07 00:22:02 +00:00
|
|
|
if (flags & MSG_TRUNC)
|
|
|
|
err = ulen;
|
|
|
|
|
2016-10-21 11:55:47 +00:00
|
|
|
skb_consume_udp(sk, skb, peeking ? -err : err);
|
2008-03-07 00:22:02 +00:00
|
|
|
return err;
|
|
|
|
|
|
|
|
csum_copy_err:
|
2017-05-16 09:20:14 +00:00
|
|
|
if (!__sk_queue_drop_skb(sk, &udp_sk(sk)->reader_queue, skb, flags,
|
|
|
|
udp_skb_destructor)) {
|
net: snmp: kill various STATS_USER() helpers
In the old days (before linux-3.0), SNMP counters were duplicated,
one for user context, and one for BH context.
After commit 8f0ea0fe3a03 ("snmp: reduce percpu needs by 50%")
we have a single copy, and what really matters is preemption being
enabled or disabled, since we use this_cpu_inc() or __this_cpu_inc()
respectively.
We therefore kill SNMP_INC_STATS_USER(), SNMP_ADD_STATS_USER(),
NET_INC_STATS_USER(), NET_ADD_STATS_USER(), SCTP_INC_STATS_USER(),
SNMP_INC_STATS64_USER(), SNMP_ADD_STATS64_USER(), TCP_ADD_STATS_USER(),
UDP_INC_STATS_USER(), UDP6_INC_STATS_USER(), and XFRM_INC_STATS_USER()
Following patches will rename __BH helpers to make clear their
usage is not tied to BH being disabled.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-04-27 23:44:27 +00:00
|
|
|
UDP_INC_STATS(sock_net(sk), UDP_MIB_CSUMERRORS, is_udplite);
|
|
|
|
UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite);
|
2013-04-29 08:39:56 +00:00
|
|
|
}
|
2016-10-21 11:55:47 +00:00
|
|
|
kfree_skb(skb);
|
2008-03-07 00:22:02 +00:00
|
|
|
|
2015-05-30 16:16:53 +00:00
|
|
|
/* starting over for a new packet, but check if we need to yield */
|
|
|
|
cond_resched();
|
2011-06-21 10:43:40 +00:00
|
|
|
msg->msg_flags &= ~MSG_TRUNC;
|
2008-03-07 00:22:02 +00:00
|
|
|
goto try_again;
|
|
|
|
}
|
|
|
|
|
2018-03-30 22:08:05 +00:00
|
|
|
int udp_pre_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)
|
|
|
|
{
|
|
|
|
/* This check is replicated from __ip4_datagram_connect() and
|
|
|
|
* intended to prevent BPF program called below from accessing bytes
|
|
|
|
* that are out of the bound specified by user in addr_len.
|
|
|
|
*/
|
|
|
|
if (addr_len < sizeof(struct sockaddr_in))
|
|
|
|
return -EINVAL;
|
|
|
|
|
2023-10-11 18:51:04 +00:00
|
|
|
return BPF_CGROUP_RUN_PROG_INET4_CONNECT_LOCK(sk, uaddr, &addr_len);
|
2018-03-30 22:08:05 +00:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(udp_pre_connect);
|
|
|
|
|
2016-10-20 16:39:40 +00:00
|
|
|
int __udp_disconnect(struct sock *sk, int flags)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
struct inet_sock *inet = inet_sk(sk);
|
|
|
|
/*
|
|
|
|
* 1003.1g - break association.
|
|
|
|
*/
|
2007-02-09 14:24:47 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
sk->sk_state = TCP_CLOSE;
|
2009-10-15 06:30:45 +00:00
|
|
|
inet->inet_daddr = 0;
|
|
|
|
inet->inet_dport = 0;
|
2011-08-14 19:45:55 +00:00
|
|
|
sock_rps_reset_rxhash(sk);
|
2005-04-16 22:20:36 +00:00
|
|
|
sk->sk_bound_dev_if = 0;
|
2020-02-19 19:16:32 +00:00
|
|
|
if (!(sk->sk_userlocks & SOCK_BINDADDR_LOCK)) {
|
2005-04-16 22:20:36 +00:00
|
|
|
inet_reset_saddr(sk);
|
2020-02-19 19:16:32 +00:00
|
|
|
if (sk->sk_prot->rehash &&
|
|
|
|
(sk->sk_userlocks & SOCK_BINDPORT_LOCK))
|
|
|
|
sk->sk_prot->rehash(sk);
|
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
if (!(sk->sk_userlocks & SOCK_BINDPORT_LOCK)) {
|
|
|
|
sk->sk_prot->unhash(sk);
|
2009-10-15 06:30:45 +00:00
|
|
|
inet->inet_sport = 0;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
sk_dst_reset(sk);
|
|
|
|
return 0;
|
|
|
|
}
|
2016-10-20 16:39:40 +00:00
|
|
|
EXPORT_SYMBOL(__udp_disconnect);
|
|
|
|
|
|
|
|
int udp_disconnect(struct sock *sk, int flags)
|
|
|
|
{
|
|
|
|
lock_sock(sk);
|
|
|
|
__udp_disconnect(sk, flags);
|
|
|
|
release_sock(sk);
|
|
|
|
return 0;
|
|
|
|
}
|
2009-07-17 00:26:32 +00:00
|
|
|
EXPORT_SYMBOL(udp_disconnect);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2008-10-29 08:41:45 +00:00
|
|
|
void udp_lib_unhash(struct sock *sk)
|
|
|
|
{
|
2008-11-25 21:55:15 +00:00
|
|
|
if (sk_hashed(sk)) {
|
2022-11-14 21:57:54 +00:00
|
|
|
struct udp_table *udptable = udp_get_table_prot(sk);
|
2009-11-08 10:17:58 +00:00
|
|
|
struct udp_hslot *hslot, *hslot2;
|
|
|
|
|
|
|
|
hslot = udp_hashslot(udptable, sock_net(sk),
|
|
|
|
udp_sk(sk)->udp_port_hash);
|
|
|
|
hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash);
|
2008-10-29 08:41:45 +00:00
|
|
|
|
2008-11-25 21:55:15 +00:00
|
|
|
spin_lock_bh(&hslot->lock);
|
2016-01-04 22:41:46 +00:00
|
|
|
if (rcu_access_pointer(sk->sk_reuseport_cb))
|
|
|
|
reuseport_detach_sock(sk);
|
2016-04-01 15:52:13 +00:00
|
|
|
if (sk_del_node_init_rcu(sk)) {
|
2009-11-08 10:17:05 +00:00
|
|
|
hslot->count--;
|
2009-10-15 06:30:45 +00:00
|
|
|
inet_sk(sk)->inet_num = 0;
|
2008-11-25 21:55:15 +00:00
|
|
|
sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
|
2009-11-08 10:17:58 +00:00
|
|
|
|
|
|
|
spin_lock(&hslot2->lock);
|
2016-04-01 15:52:13 +00:00
|
|
|
hlist_del_init_rcu(&udp_sk(sk)->udp_portaddr_node);
|
2009-11-08 10:17:58 +00:00
|
|
|
hslot2->count--;
|
|
|
|
spin_unlock(&hslot2->lock);
|
2008-11-25 21:55:15 +00:00
|
|
|
}
|
|
|
|
spin_unlock_bh(&hslot->lock);
|
2008-10-29 08:41:45 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(udp_lib_unhash);
|
|
|
|
|
udp: add rehash on connect()
commit 30fff923 introduced in linux-2.6.33 (udp: bind() optimisation)
added a secondary hash on UDP, hashed on (local addr, local port).
Problem is that following sequence :
fd = socket(...)
connect(fd, &remote, ...)
not only selects remote end point (address and port), but also sets
local address, while UDP stack stored in secondary hash table the socket
while its local address was INADDR_ANY (or ipv6 equivalent)
Sequence is :
- autobind() : choose a random local port, insert socket in hash tables
[while local address is INADDR_ANY]
- connect() : set remote address and port, change local address to IP
given by a route lookup.
When an incoming UDP frame comes, if more than 10 sockets are found in
primary hash table, we switch to secondary table, and fail to find
socket because its local address changed.
One solution to this problem is to rehash datagram socket if needed.
We add a new rehash(struct socket *) method in "struct proto", and
implement this method for UDP v4 & v6, using a common helper.
This rehashing only takes care of secondary hash table, since primary
hash (based on local port only) is not changed.
Reported-by: Krzysztof Piotr Oledzki <ole@ans.pl>
Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Tested-by: Krzysztof Piotr Oledzki <ole@ans.pl>
Signed-off-by: David S. Miller <davem@davemloft.net>
2010-09-08 05:08:44 +00:00
|
|
|
/*
|
|
|
|
* inet_rcv_saddr was changed, we must rehash secondary hash
|
|
|
|
*/
|
|
|
|
void udp_lib_rehash(struct sock *sk, u16 newhash)
|
|
|
|
{
|
|
|
|
if (sk_hashed(sk)) {
|
2022-11-14 21:57:54 +00:00
|
|
|
struct udp_table *udptable = udp_get_table_prot(sk);
|
udp: add rehash on connect()
commit 30fff923 introduced in linux-2.6.33 (udp: bind() optimisation)
added a secondary hash on UDP, hashed on (local addr, local port).
Problem is that following sequence :
fd = socket(...)
connect(fd, &remote, ...)
not only selects remote end point (address and port), but also sets
local address, while UDP stack stored in secondary hash table the socket
while its local address was INADDR_ANY (or ipv6 equivalent)
Sequence is :
- autobind() : choose a random local port, insert socket in hash tables
[while local address is INADDR_ANY]
- connect() : set remote address and port, change local address to IP
given by a route lookup.
When an incoming UDP frame comes, if more than 10 sockets are found in
primary hash table, we switch to secondary table, and fail to find
socket because its local address changed.
One solution to this problem is to rehash datagram socket if needed.
We add a new rehash(struct socket *) method in "struct proto", and
implement this method for UDP v4 & v6, using a common helper.
This rehashing only takes care of secondary hash table, since primary
hash (based on local port only) is not changed.
Reported-by: Krzysztof Piotr Oledzki <ole@ans.pl>
Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Tested-by: Krzysztof Piotr Oledzki <ole@ans.pl>
Signed-off-by: David S. Miller <davem@davemloft.net>
2010-09-08 05:08:44 +00:00
|
|
|
struct udp_hslot *hslot, *hslot2, *nhslot2;
|
|
|
|
|
|
|
|
hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash);
|
|
|
|
nhslot2 = udp_hashslot2(udptable, newhash);
|
|
|
|
udp_sk(sk)->udp_portaddr_hash = newhash;
|
2016-01-04 22:41:46 +00:00
|
|
|
|
|
|
|
if (hslot2 != nhslot2 ||
|
|
|
|
rcu_access_pointer(sk->sk_reuseport_cb)) {
|
udp: add rehash on connect()
commit 30fff923 introduced in linux-2.6.33 (udp: bind() optimisation)
added a secondary hash on UDP, hashed on (local addr, local port).
Problem is that following sequence :
fd = socket(...)
connect(fd, &remote, ...)
not only selects remote end point (address and port), but also sets
local address, while UDP stack stored in secondary hash table the socket
while its local address was INADDR_ANY (or ipv6 equivalent)
Sequence is :
- autobind() : choose a random local port, insert socket in hash tables
[while local address is INADDR_ANY]
- connect() : set remote address and port, change local address to IP
given by a route lookup.
When an incoming UDP frame comes, if more than 10 sockets are found in
primary hash table, we switch to secondary table, and fail to find
socket because its local address changed.
One solution to this problem is to rehash datagram socket if needed.
We add a new rehash(struct socket *) method in "struct proto", and
implement this method for UDP v4 & v6, using a common helper.
This rehashing only takes care of secondary hash table, since primary
hash (based on local port only) is not changed.
Reported-by: Krzysztof Piotr Oledzki <ole@ans.pl>
Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Tested-by: Krzysztof Piotr Oledzki <ole@ans.pl>
Signed-off-by: David S. Miller <davem@davemloft.net>
2010-09-08 05:08:44 +00:00
|
|
|
hslot = udp_hashslot(udptable, sock_net(sk),
|
|
|
|
udp_sk(sk)->udp_port_hash);
|
|
|
|
/* we must lock primary chain too */
|
|
|
|
spin_lock_bh(&hslot->lock);
|
2016-01-04 22:41:46 +00:00
|
|
|
if (rcu_access_pointer(sk->sk_reuseport_cb))
|
|
|
|
reuseport_detach_sock(sk);
|
|
|
|
|
|
|
|
if (hslot2 != nhslot2) {
|
|
|
|
spin_lock(&hslot2->lock);
|
2016-04-01 15:52:13 +00:00
|
|
|
hlist_del_init_rcu(&udp_sk(sk)->udp_portaddr_node);
|
2016-01-04 22:41:46 +00:00
|
|
|
hslot2->count--;
|
|
|
|
spin_unlock(&hslot2->lock);
|
|
|
|
|
|
|
|
spin_lock(&nhslot2->lock);
|
2016-04-01 15:52:13 +00:00
|
|
|
hlist_add_head_rcu(&udp_sk(sk)->udp_portaddr_node,
|
2016-01-04 22:41:46 +00:00
|
|
|
&nhslot2->head);
|
|
|
|
nhslot2->count++;
|
|
|
|
spin_unlock(&nhslot2->lock);
|
|
|
|
}
|
udp: add rehash on connect()
commit 30fff923 introduced in linux-2.6.33 (udp: bind() optimisation)
added a secondary hash on UDP, hashed on (local addr, local port).
Problem is that following sequence :
fd = socket(...)
connect(fd, &remote, ...)
not only selects remote end point (address and port), but also sets
local address, while UDP stack stored in secondary hash table the socket
while its local address was INADDR_ANY (or ipv6 equivalent)
Sequence is :
- autobind() : choose a random local port, insert socket in hash tables
[while local address is INADDR_ANY]
- connect() : set remote address and port, change local address to IP
given by a route lookup.
When an incoming UDP frame comes, if more than 10 sockets are found in
primary hash table, we switch to secondary table, and fail to find
socket because its local address changed.
One solution to this problem is to rehash datagram socket if needed.
We add a new rehash(struct socket *) method in "struct proto", and
implement this method for UDP v4 & v6, using a common helper.
This rehashing only takes care of secondary hash table, since primary
hash (based on local port only) is not changed.
Reported-by: Krzysztof Piotr Oledzki <ole@ans.pl>
Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Tested-by: Krzysztof Piotr Oledzki <ole@ans.pl>
Signed-off-by: David S. Miller <davem@davemloft.net>
2010-09-08 05:08:44 +00:00
|
|
|
|
|
|
|
spin_unlock_bh(&hslot->lock);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(udp_lib_rehash);
|
|
|
|
|
2019-01-16 16:17:44 +00:00
|
|
|
void udp_v4_rehash(struct sock *sk)
|
udp: add rehash on connect()
commit 30fff923 introduced in linux-2.6.33 (udp: bind() optimisation)
added a secondary hash on UDP, hashed on (local addr, local port).
Problem is that following sequence :
fd = socket(...)
connect(fd, &remote, ...)
not only selects remote end point (address and port), but also sets
local address, while UDP stack stored in secondary hash table the socket
while its local address was INADDR_ANY (or ipv6 equivalent)
Sequence is :
- autobind() : choose a random local port, insert socket in hash tables
[while local address is INADDR_ANY]
- connect() : set remote address and port, change local address to IP
given by a route lookup.
When an incoming UDP frame comes, if more than 10 sockets are found in
primary hash table, we switch to secondary table, and fail to find
socket because its local address changed.
One solution to this problem is to rehash datagram socket if needed.
We add a new rehash(struct socket *) method in "struct proto", and
implement this method for UDP v4 & v6, using a common helper.
This rehashing only takes care of secondary hash table, since primary
hash (based on local port only) is not changed.
Reported-by: Krzysztof Piotr Oledzki <ole@ans.pl>
Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Tested-by: Krzysztof Piotr Oledzki <ole@ans.pl>
Signed-off-by: David S. Miller <davem@davemloft.net>
2010-09-08 05:08:44 +00:00
|
|
|
{
|
2017-12-01 20:52:30 +00:00
|
|
|
u16 new_hash = ipv4_portaddr_hash(sock_net(sk),
|
udp: add rehash on connect()
commit 30fff923 introduced in linux-2.6.33 (udp: bind() optimisation)
added a secondary hash on UDP, hashed on (local addr, local port).
Problem is that following sequence :
fd = socket(...)
connect(fd, &remote, ...)
not only selects remote end point (address and port), but also sets
local address, while UDP stack stored in secondary hash table the socket
while its local address was INADDR_ANY (or ipv6 equivalent)
Sequence is :
- autobind() : choose a random local port, insert socket in hash tables
[while local address is INADDR_ANY]
- connect() : set remote address and port, change local address to IP
given by a route lookup.
When an incoming UDP frame comes, if more than 10 sockets are found in
primary hash table, we switch to secondary table, and fail to find
socket because its local address changed.
One solution to this problem is to rehash datagram socket if needed.
We add a new rehash(struct socket *) method in "struct proto", and
implement this method for UDP v4 & v6, using a common helper.
This rehashing only takes care of secondary hash table, since primary
hash (based on local port only) is not changed.
Reported-by: Krzysztof Piotr Oledzki <ole@ans.pl>
Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Tested-by: Krzysztof Piotr Oledzki <ole@ans.pl>
Signed-off-by: David S. Miller <davem@davemloft.net>
2010-09-08 05:08:44 +00:00
|
|
|
inet_sk(sk)->inet_rcv_saddr,
|
|
|
|
inet_sk(sk)->inet_num);
|
|
|
|
udp_lib_rehash(sk, new_hash);
|
|
|
|
}
|
|
|
|
|
2017-05-17 12:52:16 +00:00
|
|
|
static int __udp_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
|
2008-09-15 18:48:46 +00:00
|
|
|
{
|
rfs: Receive Flow Steering
This patch implements receive flow steering (RFS). RFS steers
received packets for layer 3 and 4 processing to the CPU where
the application for the corresponding flow is running. RFS is an
extension of Receive Packet Steering (RPS).
The basic idea of RFS is that when an application calls recvmsg
(or sendmsg) the application's running CPU is stored in a hash
table that is indexed by the connection's rxhash which is stored in
the socket structure. The rxhash is passed in skb's received on
the connection from netif_receive_skb. For each received packet,
the associated rxhash is used to look up the CPU in the hash table,
if a valid CPU is set then the packet is steered to that CPU using
the RPS mechanisms.
The convolution of the simple approach is that it would potentially
allow OOO packets. If threads are thrashing around CPUs or multiple
threads are trying to read from the same sockets, a quickly changing
CPU value in the hash table could cause rampant OOO packets--
we consider this a non-starter.
To avoid OOO packets, this solution implements two types of hash
tables: rps_sock_flow_table and rps_dev_flow_table.
rps_sock_table is a global hash table. Each entry is just a CPU
number and it is populated in recvmsg and sendmsg as described above.
This table contains the "desired" CPUs for flows.
rps_dev_flow_table is specific to each device queue. Each entry
contains a CPU and a tail queue counter. The CPU is the "current"
CPU for a matching flow. The tail queue counter holds the value
of a tail queue counter for the associated CPU's backlog queue at
the time of last enqueue for a flow matching the entry.
Each backlog queue has a queue head counter which is incremented
on dequeue, and so a queue tail counter is computed as queue head
count + queue length. When a packet is enqueued on a backlog queue,
the current value of the queue tail counter is saved in the hash
entry of the rps_dev_flow_table.
And now the trick: when selecting the CPU for RPS (get_rps_cpu)
the rps_sock_flow table and the rps_dev_flow table for the RX queue
are consulted. When the desired CPU for the flow (found in the
rps_sock_flow table) does not match the current CPU (found in the
rps_dev_flow table), the current CPU is changed to the desired CPU
if one of the following is true:
- The current CPU is unset (equal to RPS_NO_CPU)
- Current CPU is offline
- The current CPU's queue head counter >= queue tail counter in the
rps_dev_flow table. This checks if the queue tail has advanced
beyond the last packet that was enqueued using this table entry.
This guarantees that all packets queued using this entry have been
dequeued, thus preserving in order delivery.
Making each queue have its own rps_dev_flow table has two advantages:
1) the tail queue counters will be written on each receive, so
keeping the table local to interrupting CPU s good for locality. 2)
this allows lockless access to the table-- the CPU number and queue
tail counter need to be accessed together under mutual exclusion
from netif_receive_skb, we assume that this is only called from
device napi_poll which is non-reentrant.
This patch implements RFS for TCP and connected UDP sockets.
It should be usable for other flow oriented protocols.
There are two configuration parameters for RFS. The
"rps_flow_entries" kernel init parameter sets the number of
entries in the rps_sock_flow_table, the per rxqueue sysfs entry
"rps_flow_cnt" contains the number of entries in the rps_dev_flow
table for the rxqueue. Both are rounded to power of two.
The obvious benefit of RFS (over just RPS) is that it achieves
CPU locality between the receive processing for a flow and the
applications processing; this can result in increased performance
(higher pps, lower latency).
The benefits of RFS are dependent on cache hierarchy, application
load, and other factors. On simple benchmarks, we don't necessarily
see improvement and sometimes see degradation. However, for more
complex benchmarks and for applications where cache pressure is
much higher this technique seems to perform very well.
Below are some benchmark results which show the potential benfit of
this patch. The netperf test has 500 instances of netperf TCP_RR
test with 1 byte req. and resp. The RPC test is an request/response
test similar in structure to netperf RR test ith 100 threads on
each host, but does more work in userspace that netperf.
e1000e on 8 core Intel
No RFS or RPS 104K tps at 30% CPU
No RFS (best RPS config): 290K tps at 63% CPU
RFS 303K tps at 61% CPU
RPC test tps CPU% 50/90/99% usec latency Latency StdDev
No RFS/RPS 103K 48% 757/900/3185 4472.35
RPS only: 174K 73% 415/993/2468 491.66
RFS 223K 73% 379/651/1382 315.61
Signed-off-by: Tom Herbert <therbert@google.com>
Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2010-04-16 23:01:27 +00:00
|
|
|
int rc;
|
2009-10-15 03:40:11 +00:00
|
|
|
|
2013-10-07 16:01:38 +00:00
|
|
|
if (inet_sk(sk)->inet_daddr) {
|
2011-08-14 19:45:55 +00:00
|
|
|
sock_rps_save_rxhash(sk, skb);
|
2013-10-07 16:01:38 +00:00
|
|
|
sk_mark_napi_id(sk, skb);
|
net: introduce SO_INCOMING_CPU
Alternative to RPS/RFS is to use hardware support for multiple
queues.
Then split a set of million of sockets into worker threads, each
one using epoll() to manage events on its own socket pool.
Ideally, we want one thread per RX/TX queue/cpu, but we have no way to
know after accept() or connect() on which queue/cpu a socket is managed.
We normally use one cpu per RX queue (IRQ smp_affinity being properly
set), so remembering on socket structure which cpu delivered last packet
is enough to solve the problem.
After accept(), connect(), or even file descriptor passing around
processes, applications can use :
int cpu;
socklen_t len = sizeof(cpu);
getsockopt(fd, SOL_SOCKET, SO_INCOMING_CPU, &cpu, &len);
And use this information to put the socket into the right silo
for optimal performance, as all networking stack should run
on the appropriate cpu, without need to send IPI (RPS/RFS).
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-11-11 13:54:28 +00:00
|
|
|
sk_incoming_cpu_update(sk);
|
2016-11-16 17:10:42 +00:00
|
|
|
} else {
|
|
|
|
sk_mark_napi_id_once(sk, skb);
|
2013-10-07 16:01:38 +00:00
|
|
|
}
|
rfs: Receive Flow Steering
This patch implements receive flow steering (RFS). RFS steers
received packets for layer 3 and 4 processing to the CPU where
the application for the corresponding flow is running. RFS is an
extension of Receive Packet Steering (RPS).
The basic idea of RFS is that when an application calls recvmsg
(or sendmsg) the application's running CPU is stored in a hash
table that is indexed by the connection's rxhash which is stored in
the socket structure. The rxhash is passed in skb's received on
the connection from netif_receive_skb. For each received packet,
the associated rxhash is used to look up the CPU in the hash table,
if a valid CPU is set then the packet is steered to that CPU using
the RPS mechanisms.
The convolution of the simple approach is that it would potentially
allow OOO packets. If threads are thrashing around CPUs or multiple
threads are trying to read from the same sockets, a quickly changing
CPU value in the hash table could cause rampant OOO packets--
we consider this a non-starter.
To avoid OOO packets, this solution implements two types of hash
tables: rps_sock_flow_table and rps_dev_flow_table.
rps_sock_table is a global hash table. Each entry is just a CPU
number and it is populated in recvmsg and sendmsg as described above.
This table contains the "desired" CPUs for flows.
rps_dev_flow_table is specific to each device queue. Each entry
contains a CPU and a tail queue counter. The CPU is the "current"
CPU for a matching flow. The tail queue counter holds the value
of a tail queue counter for the associated CPU's backlog queue at
the time of last enqueue for a flow matching the entry.
Each backlog queue has a queue head counter which is incremented
on dequeue, and so a queue tail counter is computed as queue head
count + queue length. When a packet is enqueued on a backlog queue,
the current value of the queue tail counter is saved in the hash
entry of the rps_dev_flow_table.
And now the trick: when selecting the CPU for RPS (get_rps_cpu)
the rps_sock_flow table and the rps_dev_flow table for the RX queue
are consulted. When the desired CPU for the flow (found in the
rps_sock_flow table) does not match the current CPU (found in the
rps_dev_flow table), the current CPU is changed to the desired CPU
if one of the following is true:
- The current CPU is unset (equal to RPS_NO_CPU)
- Current CPU is offline
- The current CPU's queue head counter >= queue tail counter in the
rps_dev_flow table. This checks if the queue tail has advanced
beyond the last packet that was enqueued using this table entry.
This guarantees that all packets queued using this entry have been
dequeued, thus preserving in order delivery.
Making each queue have its own rps_dev_flow table has two advantages:
1) the tail queue counters will be written on each receive, so
keeping the table local to interrupting CPU s good for locality. 2)
this allows lockless access to the table-- the CPU number and queue
tail counter need to be accessed together under mutual exclusion
from netif_receive_skb, we assume that this is only called from
device napi_poll which is non-reentrant.
This patch implements RFS for TCP and connected UDP sockets.
It should be usable for other flow oriented protocols.
There are two configuration parameters for RFS. The
"rps_flow_entries" kernel init parameter sets the number of
entries in the rps_sock_flow_table, the per rxqueue sysfs entry
"rps_flow_cnt" contains the number of entries in the rps_dev_flow
table for the rxqueue. Both are rounded to power of two.
The obvious benefit of RFS (over just RPS) is that it achieves
CPU locality between the receive processing for a flow and the
applications processing; this can result in increased performance
(higher pps, lower latency).
The benefits of RFS are dependent on cache hierarchy, application
load, and other factors. On simple benchmarks, we don't necessarily
see improvement and sometimes see degradation. However, for more
complex benchmarks and for applications where cache pressure is
much higher this technique seems to perform very well.
Below are some benchmark results which show the potential benfit of
this patch. The netperf test has 500 instances of netperf TCP_RR
test with 1 byte req. and resp. The RPC test is an request/response
test similar in structure to netperf RR test ith 100 threads on
each host, but does more work in userspace that netperf.
e1000e on 8 core Intel
No RFS or RPS 104K tps at 30% CPU
No RFS (best RPS config): 290K tps at 63% CPU
RFS 303K tps at 61% CPU
RPC test tps CPU% 50/90/99% usec latency Latency StdDev
No RFS/RPS 103K 48% 757/900/3185 4472.35
RPS only: 174K 73% 415/993/2468 491.66
RFS 223K 73% 379/651/1382 315.61
Signed-off-by: Tom Herbert <therbert@google.com>
Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2010-04-16 23:01:27 +00:00
|
|
|
|
2016-10-21 11:55:47 +00:00
|
|
|
rc = __udp_enqueue_schedule_skb(sk, skb);
|
2009-10-15 03:40:11 +00:00
|
|
|
if (rc < 0) {
|
|
|
|
int is_udplite = IS_UDPLITE(sk);
|
2022-02-05 07:47:39 +00:00
|
|
|
int drop_reason;
|
2008-09-15 18:48:46 +00:00
|
|
|
|
|
|
|
/* Note that an ENOMEM error is charged twice */
|
2022-02-05 07:47:39 +00:00
|
|
|
if (rc == -ENOMEM) {
|
2016-04-29 21:16:50 +00:00
|
|
|
UDP_INC_STATS(sock_net(sk), UDP_MIB_RCVBUFERRORS,
|
2016-04-27 23:44:30 +00:00
|
|
|
is_udplite);
|
2022-02-05 07:47:39 +00:00
|
|
|
drop_reason = SKB_DROP_REASON_SOCKET_RCVBUFF;
|
|
|
|
} else {
|
2020-11-06 01:49:14 +00:00
|
|
|
UDP_INC_STATS(sock_net(sk), UDP_MIB_MEMERRORS,
|
|
|
|
is_udplite);
|
2022-02-05 07:47:39 +00:00
|
|
|
drop_reason = SKB_DROP_REASON_PROTO_MEM;
|
|
|
|
}
|
2016-04-29 21:16:50 +00:00
|
|
|
UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite);
|
2022-02-05 07:47:39 +00:00
|
|
|
kfree_skb_reason(skb, drop_reason);
|
2011-06-17 11:58:39 +00:00
|
|
|
trace_udp_fail_queue_rcv_skb(rc, sk);
|
2009-10-15 03:40:11 +00:00
|
|
|
return -1;
|
2008-09-15 18:48:46 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2008-03-07 00:22:02 +00:00
|
|
|
/* returns:
|
|
|
|
* -1: error
|
|
|
|
* 0: success
|
|
|
|
* >0: "udp encap" protocol resubmission
|
|
|
|
*
|
|
|
|
* Note that in the success and error cases, the skb is assumed to
|
|
|
|
* have either been requeued or freed.
|
|
|
|
*/
|
2018-11-07 11:38:33 +00:00
|
|
|
static int udp_queue_rcv_one_skb(struct sock *sk, struct sk_buff *skb)
|
2008-03-07 00:22:02 +00:00
|
|
|
{
|
2022-02-05 07:47:38 +00:00
|
|
|
int drop_reason = SKB_DROP_REASON_NOT_SPECIFIED;
|
2008-03-07 00:22:02 +00:00
|
|
|
struct udp_sock *up = udp_sk(sk);
|
|
|
|
int is_udplite = IS_UDPLITE(sk);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Charge it to the socket, dropping if the queue is full.
|
|
|
|
*/
|
2022-02-05 07:47:38 +00:00
|
|
|
if (!xfrm4_policy_check(sk, XFRM_POLICY_IN, skb)) {
|
|
|
|
drop_reason = SKB_DROP_REASON_XFRM_POLICY;
|
2008-03-07 00:22:02 +00:00
|
|
|
goto drop;
|
2022-02-05 07:47:38 +00:00
|
|
|
}
|
2019-09-29 18:54:03 +00:00
|
|
|
nf_reset_ct(skb);
|
2008-03-07 00:22:02 +00:00
|
|
|
|
2023-09-12 09:17:28 +00:00
|
|
|
if (static_branch_unlikely(&udp_encap_needed_key) &&
|
|
|
|
READ_ONCE(up->encap_type)) {
|
2011-11-01 12:56:59 +00:00
|
|
|
int (*encap_rcv)(struct sock *sk, struct sk_buff *skb);
|
|
|
|
|
2008-03-07 00:22:02 +00:00
|
|
|
/*
|
|
|
|
* This is an encapsulation socket so pass the skb to
|
|
|
|
* the socket's udp_encap_rcv() hook. Otherwise, just
|
|
|
|
* fall through and pass this up the UDP socket.
|
|
|
|
* up->encap_rcv() returns the following value:
|
|
|
|
* =0 if skb was successfully passed to the encap
|
|
|
|
* handler or was discarded by it.
|
|
|
|
* >0 if skb should be passed on to UDP.
|
|
|
|
* <0 if skb should be resubmitted as proto -N
|
|
|
|
*/
|
|
|
|
|
|
|
|
/* if we're overly short, let UDP handle it */
|
locking/atomics: COCCINELLE/treewide: Convert trivial ACCESS_ONCE() patterns to READ_ONCE()/WRITE_ONCE()
Please do not apply this to mainline directly, instead please re-run the
coccinelle script shown below and apply its output.
For several reasons, it is desirable to use {READ,WRITE}_ONCE() in
preference to ACCESS_ONCE(), and new code is expected to use one of the
former. So far, there's been no reason to change most existing uses of
ACCESS_ONCE(), as these aren't harmful, and changing them results in
churn.
However, for some features, the read/write distinction is critical to
correct operation. To distinguish these cases, separate read/write
accessors must be used. This patch migrates (most) remaining
ACCESS_ONCE() instances to {READ,WRITE}_ONCE(), using the following
coccinelle script:
----
// Convert trivial ACCESS_ONCE() uses to equivalent READ_ONCE() and
// WRITE_ONCE()
// $ make coccicheck COCCI=/home/mark/once.cocci SPFLAGS="--include-headers" MODE=patch
virtual patch
@ depends on patch @
expression E1, E2;
@@
- ACCESS_ONCE(E1) = E2
+ WRITE_ONCE(E1, E2)
@ depends on patch @
expression E;
@@
- ACCESS_ONCE(E)
+ READ_ONCE(E)
----
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: davem@davemloft.net
Cc: linux-arch@vger.kernel.org
Cc: mpe@ellerman.id.au
Cc: shuah@kernel.org
Cc: snitzer@redhat.com
Cc: thor.thayer@linux.intel.com
Cc: tj@kernel.org
Cc: viro@zeniv.linux.org.uk
Cc: will.deacon@arm.com
Link: http://lkml.kernel.org/r/1508792849-3115-19-git-send-email-paulmck@linux.vnet.ibm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2017-10-23 21:07:29 +00:00
|
|
|
encap_rcv = READ_ONCE(up->encap_rcv);
|
2016-05-19 13:58:33 +00:00
|
|
|
if (encap_rcv) {
|
2008-03-07 00:22:02 +00:00
|
|
|
int ret;
|
|
|
|
|
2014-05-07 23:52:39 +00:00
|
|
|
/* Verify checksum before giving to encap */
|
|
|
|
if (udp_lib_checksum_complete(skb))
|
|
|
|
goto csum_error;
|
|
|
|
|
2011-11-01 12:56:59 +00:00
|
|
|
ret = encap_rcv(sk, skb);
|
2008-03-07 00:22:02 +00:00
|
|
|
if (ret <= 0) {
|
2016-04-27 23:44:30 +00:00
|
|
|
__UDP_INC_STATS(sock_net(sk),
|
|
|
|
UDP_MIB_INDATAGRAMS,
|
|
|
|
is_udplite);
|
2008-03-07 00:22:02 +00:00
|
|
|
return -ret;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/* FALLTHROUGH -- it's a UDP Packet */
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* UDP-Lite specific tests, ignored on UDP sockets
|
|
|
|
*/
|
2023-09-12 09:17:30 +00:00
|
|
|
if (udp_test_bit(UDPLITE_RECV_CC, sk) && UDP_SKB_CB(skb)->partial_cov) {
|
|
|
|
u16 pcrlen = READ_ONCE(up->pcrlen);
|
2008-03-07 00:22:02 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* MIB statistics other than incrementing the error count are
|
|
|
|
* disabled for the following two types of errors: these depend
|
|
|
|
* on the application settings, not on the functioning of the
|
|
|
|
* protocol stack as such.
|
|
|
|
*
|
|
|
|
* RFC 3828 here recommends (sec 3.3): "There should also be a
|
|
|
|
* way ... to ... at least let the receiving application block
|
|
|
|
* delivery of packets with coverage values less than a value
|
|
|
|
* provided by the application."
|
|
|
|
*/
|
2023-09-12 09:17:30 +00:00
|
|
|
if (pcrlen == 0) { /* full coverage was set */
|
2014-11-11 18:59:17 +00:00
|
|
|
net_dbg_ratelimited("UDPLite: partial coverage %d while full coverage %d requested\n",
|
|
|
|
UDP_SKB_CB(skb)->cscov, skb->len);
|
2008-03-07 00:22:02 +00:00
|
|
|
goto drop;
|
|
|
|
}
|
|
|
|
/* The next case involves violating the min. coverage requested
|
|
|
|
* by the receiver. This is subtle: if receiver wants x and x is
|
|
|
|
* greater than the buffersize/MTU then receiver will complain
|
|
|
|
* that it wants x while sender emits packets of smaller size y.
|
|
|
|
* Therefore the above ...()->partial_cov statement is essential.
|
|
|
|
*/
|
2023-09-12 09:17:30 +00:00
|
|
|
if (UDP_SKB_CB(skb)->cscov < pcrlen) {
|
2014-11-11 18:59:17 +00:00
|
|
|
net_dbg_ratelimited("UDPLite: coverage %d too small, need min %d\n",
|
2023-09-12 09:17:30 +00:00
|
|
|
UDP_SKB_CB(skb)->cscov, pcrlen);
|
2008-03-07 00:22:02 +00:00
|
|
|
goto drop;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-06-21 08:24:40 +00:00
|
|
|
prefetch(&sk->sk_rmem_alloc);
|
2016-06-02 21:52:43 +00:00
|
|
|
if (rcu_access_pointer(sk->sk_filter) &&
|
|
|
|
udp_lib_checksum_complete(skb))
|
2016-04-05 16:41:15 +00:00
|
|
|
goto csum_error;
|
2016-06-02 21:52:43 +00:00
|
|
|
|
2022-02-05 07:47:38 +00:00
|
|
|
if (sk_filter_trim_cap(sk, skb, sizeof(struct udphdr))) {
|
|
|
|
drop_reason = SKB_DROP_REASON_SOCKET_FILTER;
|
2016-07-08 15:52:33 +00:00
|
|
|
goto drop;
|
2022-02-05 07:47:38 +00:00
|
|
|
}
|
2008-03-07 00:22:02 +00:00
|
|
|
|
2016-04-05 16:41:15 +00:00
|
|
|
udp_csum_pull_header(skb);
|
2008-03-07 00:22:02 +00:00
|
|
|
|
2013-10-07 16:01:40 +00:00
|
|
|
ipv4_pktinfo_prepare(sk, skb);
|
2016-10-21 11:55:47 +00:00
|
|
|
return __udp_queue_rcv_skb(sk, skb);
|
2008-03-07 00:22:02 +00:00
|
|
|
|
2013-04-29 08:39:56 +00:00
|
|
|
csum_error:
|
2022-02-05 07:47:38 +00:00
|
|
|
drop_reason = SKB_DROP_REASON_UDP_CSUM;
|
2016-04-27 23:44:30 +00:00
|
|
|
__UDP_INC_STATS(sock_net(sk), UDP_MIB_CSUMERRORS, is_udplite);
|
2008-03-07 00:22:02 +00:00
|
|
|
drop:
|
2016-04-27 23:44:30 +00:00
|
|
|
__UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite);
|
2009-10-15 00:12:40 +00:00
|
|
|
atomic_inc(&sk->sk_drops);
|
2022-02-05 07:47:38 +00:00
|
|
|
kfree_skb_reason(skb, drop_reason);
|
2008-03-07 00:22:02 +00:00
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
2018-11-07 11:38:33 +00:00
|
|
|
static int udp_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
|
|
|
|
{
|
|
|
|
struct sk_buff *next, *segs;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
if (likely(!udp_unexpected_gso(sk, skb)))
|
|
|
|
return udp_queue_rcv_one_skb(sk, skb);
|
|
|
|
|
2020-03-26 07:33:14 +00:00
|
|
|
BUILD_BUG_ON(sizeof(struct udp_skb_cb) > SKB_GSO_CB_OFFSET);
|
2018-11-07 11:38:33 +00:00
|
|
|
__skb_push(skb, -skb_mac_offset(skb));
|
|
|
|
segs = udp_rcv_segment(sk, skb, true);
|
2020-01-13 23:42:27 +00:00
|
|
|
skb_list_walk_safe(segs, skb, next) {
|
2018-11-07 11:38:33 +00:00
|
|
|
__skb_pull(skb, skb_transport_offset(skb));
|
2021-03-30 10:28:49 +00:00
|
|
|
|
|
|
|
udp_post_segment_fix_csum(skb);
|
2018-11-07 11:38:33 +00:00
|
|
|
ret = udp_queue_rcv_one_skb(sk, skb);
|
|
|
|
if (ret > 0)
|
2020-12-07 07:55:40 +00:00
|
|
|
ip_protocol_deliver_rcu(dev_net(skb->dev), skb, ret);
|
2018-11-07 11:38:33 +00:00
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2013-12-11 22:46:51 +00:00
|
|
|
/* For TCP sockets, sk_rx_dst is protected by socket lock
|
2013-12-15 18:53:46 +00:00
|
|
|
* For UDP, we use xchg() to guard against concurrent changes.
|
2013-12-11 22:46:51 +00:00
|
|
|
*/
|
2017-08-25 12:31:01 +00:00
|
|
|
bool udp_sk_rx_dst_set(struct sock *sk, struct dst_entry *dst)
|
2013-10-07 16:01:39 +00:00
|
|
|
{
|
2013-12-11 22:46:51 +00:00
|
|
|
struct dst_entry *old;
|
|
|
|
|
2017-06-17 17:42:25 +00:00
|
|
|
if (dst_hold_safe(dst)) {
|
inet: fully convert sk->sk_rx_dst to RCU rules
syzbot reported various issues around early demux,
one being included in this changelog [1]
sk->sk_rx_dst is using RCU protection without clearly
documenting it.
And following sequences in tcp_v4_do_rcv()/tcp_v6_do_rcv()
are not following standard RCU rules.
[a] dst_release(dst);
[b] sk->sk_rx_dst = NULL;
They look wrong because a delete operation of RCU protected
pointer is supposed to clear the pointer before
the call_rcu()/synchronize_rcu() guarding actual memory freeing.
In some cases indeed, dst could be freed before [b] is done.
We could cheat by clearing sk_rx_dst before calling
dst_release(), but this seems the right time to stick
to standard RCU annotations and debugging facilities.
[1]
BUG: KASAN: use-after-free in dst_check include/net/dst.h:470 [inline]
BUG: KASAN: use-after-free in tcp_v4_early_demux+0x95b/0x960 net/ipv4/tcp_ipv4.c:1792
Read of size 2 at addr ffff88807f1cb73a by task syz-executor.5/9204
CPU: 0 PID: 9204 Comm: syz-executor.5 Not tainted 5.16.0-rc5-syzkaller #0
Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011
Call Trace:
<TASK>
__dump_stack lib/dump_stack.c:88 [inline]
dump_stack_lvl+0xcd/0x134 lib/dump_stack.c:106
print_address_description.constprop.0.cold+0x8d/0x320 mm/kasan/report.c:247
__kasan_report mm/kasan/report.c:433 [inline]
kasan_report.cold+0x83/0xdf mm/kasan/report.c:450
dst_check include/net/dst.h:470 [inline]
tcp_v4_early_demux+0x95b/0x960 net/ipv4/tcp_ipv4.c:1792
ip_rcv_finish_core.constprop.0+0x15de/0x1e80 net/ipv4/ip_input.c:340
ip_list_rcv_finish.constprop.0+0x1b2/0x6e0 net/ipv4/ip_input.c:583
ip_sublist_rcv net/ipv4/ip_input.c:609 [inline]
ip_list_rcv+0x34e/0x490 net/ipv4/ip_input.c:644
__netif_receive_skb_list_ptype net/core/dev.c:5508 [inline]
__netif_receive_skb_list_core+0x549/0x8e0 net/core/dev.c:5556
__netif_receive_skb_list net/core/dev.c:5608 [inline]
netif_receive_skb_list_internal+0x75e/0xd80 net/core/dev.c:5699
gro_normal_list net/core/dev.c:5853 [inline]
gro_normal_list net/core/dev.c:5849 [inline]
napi_complete_done+0x1f1/0x880 net/core/dev.c:6590
virtqueue_napi_complete drivers/net/virtio_net.c:339 [inline]
virtnet_poll+0xca2/0x11b0 drivers/net/virtio_net.c:1557
__napi_poll+0xaf/0x440 net/core/dev.c:7023
napi_poll net/core/dev.c:7090 [inline]
net_rx_action+0x801/0xb40 net/core/dev.c:7177
__do_softirq+0x29b/0x9c2 kernel/softirq.c:558
invoke_softirq kernel/softirq.c:432 [inline]
__irq_exit_rcu+0x123/0x180 kernel/softirq.c:637
irq_exit_rcu+0x5/0x20 kernel/softirq.c:649
common_interrupt+0x52/0xc0 arch/x86/kernel/irq.c:240
asm_common_interrupt+0x1e/0x40 arch/x86/include/asm/idtentry.h:629
RIP: 0033:0x7f5e972bfd57
Code: 39 d1 73 14 0f 1f 80 00 00 00 00 48 8b 50 f8 48 83 e8 08 48 39 ca 77 f3 48 39 c3 73 3e 48 89 13 48 8b 50 f8 48 89 38 49 8b 0e <48> 8b 3e 48 83 c3 08 48 83 c6 08 eb bc 48 39 d1 72 9e 48 39 d0 73
RSP: 002b:00007fff8a413210 EFLAGS: 00000283
RAX: 00007f5e97108990 RBX: 00007f5e97108338 RCX: ffffffff81d3aa45
RDX: ffffffff81d3aa45 RSI: 00007f5e97108340 RDI: ffffffff81d3aa45
RBP: 00007f5e97107eb8 R08: 00007f5e97108d88 R09: 0000000093c2e8d9
R10: 0000000000000000 R11: 0000000000000000 R12: 00007f5e97107eb0
R13: 00007f5e97108338 R14: 00007f5e97107ea8 R15: 0000000000000019
</TASK>
Allocated by task 13:
kasan_save_stack+0x1e/0x50 mm/kasan/common.c:38
kasan_set_track mm/kasan/common.c:46 [inline]
set_alloc_info mm/kasan/common.c:434 [inline]
__kasan_slab_alloc+0x90/0xc0 mm/kasan/common.c:467
kasan_slab_alloc include/linux/kasan.h:259 [inline]
slab_post_alloc_hook mm/slab.h:519 [inline]
slab_alloc_node mm/slub.c:3234 [inline]
slab_alloc mm/slub.c:3242 [inline]
kmem_cache_alloc+0x202/0x3a0 mm/slub.c:3247
dst_alloc+0x146/0x1f0 net/core/dst.c:92
rt_dst_alloc+0x73/0x430 net/ipv4/route.c:1613
ip_route_input_slow+0x1817/0x3a20 net/ipv4/route.c:2340
ip_route_input_rcu net/ipv4/route.c:2470 [inline]
ip_route_input_noref+0x116/0x2a0 net/ipv4/route.c:2415
ip_rcv_finish_core.constprop.0+0x288/0x1e80 net/ipv4/ip_input.c:354
ip_list_rcv_finish.constprop.0+0x1b2/0x6e0 net/ipv4/ip_input.c:583
ip_sublist_rcv net/ipv4/ip_input.c:609 [inline]
ip_list_rcv+0x34e/0x490 net/ipv4/ip_input.c:644
__netif_receive_skb_list_ptype net/core/dev.c:5508 [inline]
__netif_receive_skb_list_core+0x549/0x8e0 net/core/dev.c:5556
__netif_receive_skb_list net/core/dev.c:5608 [inline]
netif_receive_skb_list_internal+0x75e/0xd80 net/core/dev.c:5699
gro_normal_list net/core/dev.c:5853 [inline]
gro_normal_list net/core/dev.c:5849 [inline]
napi_complete_done+0x1f1/0x880 net/core/dev.c:6590
virtqueue_napi_complete drivers/net/virtio_net.c:339 [inline]
virtnet_poll+0xca2/0x11b0 drivers/net/virtio_net.c:1557
__napi_poll+0xaf/0x440 net/core/dev.c:7023
napi_poll net/core/dev.c:7090 [inline]
net_rx_action+0x801/0xb40 net/core/dev.c:7177
__do_softirq+0x29b/0x9c2 kernel/softirq.c:558
Freed by task 13:
kasan_save_stack+0x1e/0x50 mm/kasan/common.c:38
kasan_set_track+0x21/0x30 mm/kasan/common.c:46
kasan_set_free_info+0x20/0x30 mm/kasan/generic.c:370
____kasan_slab_free mm/kasan/common.c:366 [inline]
____kasan_slab_free mm/kasan/common.c:328 [inline]
__kasan_slab_free+0xff/0x130 mm/kasan/common.c:374
kasan_slab_free include/linux/kasan.h:235 [inline]
slab_free_hook mm/slub.c:1723 [inline]
slab_free_freelist_hook+0x8b/0x1c0 mm/slub.c:1749
slab_free mm/slub.c:3513 [inline]
kmem_cache_free+0xbd/0x5d0 mm/slub.c:3530
dst_destroy+0x2d6/0x3f0 net/core/dst.c:127
rcu_do_batch kernel/rcu/tree.c:2506 [inline]
rcu_core+0x7ab/0x1470 kernel/rcu/tree.c:2741
__do_softirq+0x29b/0x9c2 kernel/softirq.c:558
Last potentially related work creation:
kasan_save_stack+0x1e/0x50 mm/kasan/common.c:38
__kasan_record_aux_stack+0xf5/0x120 mm/kasan/generic.c:348
__call_rcu kernel/rcu/tree.c:2985 [inline]
call_rcu+0xb1/0x740 kernel/rcu/tree.c:3065
dst_release net/core/dst.c:177 [inline]
dst_release+0x79/0xe0 net/core/dst.c:167
tcp_v4_do_rcv+0x612/0x8d0 net/ipv4/tcp_ipv4.c:1712
sk_backlog_rcv include/net/sock.h:1030 [inline]
__release_sock+0x134/0x3b0 net/core/sock.c:2768
release_sock+0x54/0x1b0 net/core/sock.c:3300
tcp_sendmsg+0x36/0x40 net/ipv4/tcp.c:1441
inet_sendmsg+0x99/0xe0 net/ipv4/af_inet.c:819
sock_sendmsg_nosec net/socket.c:704 [inline]
sock_sendmsg+0xcf/0x120 net/socket.c:724
sock_write_iter+0x289/0x3c0 net/socket.c:1057
call_write_iter include/linux/fs.h:2162 [inline]
new_sync_write+0x429/0x660 fs/read_write.c:503
vfs_write+0x7cd/0xae0 fs/read_write.c:590
ksys_write+0x1ee/0x250 fs/read_write.c:643
do_syscall_x64 arch/x86/entry/common.c:50 [inline]
do_syscall_64+0x35/0xb0 arch/x86/entry/common.c:80
entry_SYSCALL_64_after_hwframe+0x44/0xae
The buggy address belongs to the object at ffff88807f1cb700
which belongs to the cache ip_dst_cache of size 176
The buggy address is located 58 bytes inside of
176-byte region [ffff88807f1cb700, ffff88807f1cb7b0)
The buggy address belongs to the page:
page:ffffea0001fc72c0 refcount:1 mapcount:0 mapping:0000000000000000 index:0x0 pfn:0x7f1cb
flags: 0xfff00000000200(slab|node=0|zone=1|lastcpupid=0x7ff)
raw: 00fff00000000200 dead000000000100 dead000000000122 ffff8881413bb780
raw: 0000000000000000 0000000000100010 00000001ffffffff 0000000000000000
page dumped because: kasan: bad access detected
page_owner tracks the page as allocated
page last allocated via order 0, migratetype Unmovable, gfp_mask 0x112a20(GFP_ATOMIC|__GFP_NOWARN|__GFP_NORETRY|__GFP_HARDWALL), pid 5, ts 108466983062, free_ts 108048976062
prep_new_page mm/page_alloc.c:2418 [inline]
get_page_from_freelist+0xa72/0x2f50 mm/page_alloc.c:4149
__alloc_pages+0x1b2/0x500 mm/page_alloc.c:5369
alloc_pages+0x1a7/0x300 mm/mempolicy.c:2191
alloc_slab_page mm/slub.c:1793 [inline]
allocate_slab mm/slub.c:1930 [inline]
new_slab+0x32d/0x4a0 mm/slub.c:1993
___slab_alloc+0x918/0xfe0 mm/slub.c:3022
__slab_alloc.constprop.0+0x4d/0xa0 mm/slub.c:3109
slab_alloc_node mm/slub.c:3200 [inline]
slab_alloc mm/slub.c:3242 [inline]
kmem_cache_alloc+0x35c/0x3a0 mm/slub.c:3247
dst_alloc+0x146/0x1f0 net/core/dst.c:92
rt_dst_alloc+0x73/0x430 net/ipv4/route.c:1613
__mkroute_output net/ipv4/route.c:2564 [inline]
ip_route_output_key_hash_rcu+0x921/0x2d00 net/ipv4/route.c:2791
ip_route_output_key_hash+0x18b/0x300 net/ipv4/route.c:2619
__ip_route_output_key include/net/route.h:126 [inline]
ip_route_output_flow+0x23/0x150 net/ipv4/route.c:2850
ip_route_output_key include/net/route.h:142 [inline]
geneve_get_v4_rt+0x3a6/0x830 drivers/net/geneve.c:809
geneve_xmit_skb drivers/net/geneve.c:899 [inline]
geneve_xmit+0xc4a/0x3540 drivers/net/geneve.c:1082
__netdev_start_xmit include/linux/netdevice.h:4994 [inline]
netdev_start_xmit include/linux/netdevice.h:5008 [inline]
xmit_one net/core/dev.c:3590 [inline]
dev_hard_start_xmit+0x1eb/0x920 net/core/dev.c:3606
__dev_queue_xmit+0x299a/0x3650 net/core/dev.c:4229
page last free stack trace:
reset_page_owner include/linux/page_owner.h:24 [inline]
free_pages_prepare mm/page_alloc.c:1338 [inline]
free_pcp_prepare+0x374/0x870 mm/page_alloc.c:1389
free_unref_page_prepare mm/page_alloc.c:3309 [inline]
free_unref_page+0x19/0x690 mm/page_alloc.c:3388
qlink_free mm/kasan/quarantine.c:146 [inline]
qlist_free_all+0x5a/0xc0 mm/kasan/quarantine.c:165
kasan_quarantine_reduce+0x180/0x200 mm/kasan/quarantine.c:272
__kasan_slab_alloc+0xa2/0xc0 mm/kasan/common.c:444
kasan_slab_alloc include/linux/kasan.h:259 [inline]
slab_post_alloc_hook mm/slab.h:519 [inline]
slab_alloc_node mm/slub.c:3234 [inline]
kmem_cache_alloc_node+0x255/0x3f0 mm/slub.c:3270
__alloc_skb+0x215/0x340 net/core/skbuff.c:414
alloc_skb include/linux/skbuff.h:1126 [inline]
alloc_skb_with_frags+0x93/0x620 net/core/skbuff.c:6078
sock_alloc_send_pskb+0x783/0x910 net/core/sock.c:2575
mld_newpack+0x1df/0x770 net/ipv6/mcast.c:1754
add_grhead+0x265/0x330 net/ipv6/mcast.c:1857
add_grec+0x1053/0x14e0 net/ipv6/mcast.c:1995
mld_send_initial_cr.part.0+0xf6/0x230 net/ipv6/mcast.c:2242
mld_send_initial_cr net/ipv6/mcast.c:1232 [inline]
mld_dad_work+0x1d3/0x690 net/ipv6/mcast.c:2268
process_one_work+0x9b2/0x1690 kernel/workqueue.c:2298
worker_thread+0x658/0x11f0 kernel/workqueue.c:2445
Memory state around the buggy address:
ffff88807f1cb600: fa fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb
ffff88807f1cb680: fb fb fb fb fb fb fc fc fc fc fc fc fc fc fc fc
>ffff88807f1cb700: fa fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb
^
ffff88807f1cb780: fb fb fb fb fb fb fc fc fc fc fc fc fc fc fc fc
ffff88807f1cb800: fa fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb
Fixes: 41063e9dd119 ("ipv4: Early TCP socket demux.")
Signed-off-by: Eric Dumazet <edumazet@google.com>
Link: https://lore.kernel.org/r/20211220143330.680945-1-eric.dumazet@gmail.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
2021-12-20 14:33:30 +00:00
|
|
|
old = xchg((__force struct dst_entry **)&sk->sk_rx_dst, dst);
|
2017-06-17 17:42:25 +00:00
|
|
|
dst_release(old);
|
2017-08-25 12:31:01 +00:00
|
|
|
return old != dst;
|
2017-06-17 17:42:25 +00:00
|
|
|
}
|
2017-08-25 12:31:01 +00:00
|
|
|
return false;
|
2013-10-07 16:01:39 +00:00
|
|
|
}
|
2017-07-27 12:45:09 +00:00
|
|
|
EXPORT_SYMBOL(udp_sk_rx_dst_set);
|
2013-10-07 16:01:39 +00:00
|
|
|
|
2008-03-07 00:22:02 +00:00
|
|
|
/*
|
|
|
|
* Multicasts and broadcasts go to each listener.
|
|
|
|
*
|
2009-11-08 10:18:44 +00:00
|
|
|
* Note: called only from the BH handler context.
|
2008-03-07 00:22:02 +00:00
|
|
|
*/
|
2008-06-17 00:12:11 +00:00
|
|
|
static int __udp4_lib_mcast_deliver(struct net *net, struct sk_buff *skb,
|
2008-03-07 00:22:02 +00:00
|
|
|
struct udphdr *uh,
|
|
|
|
__be32 saddr, __be32 daddr,
|
2014-11-06 18:37:54 +00:00
|
|
|
struct udp_table *udptable,
|
|
|
|
int proto)
|
2008-03-07 00:22:02 +00:00
|
|
|
{
|
2016-04-01 15:52:13 +00:00
|
|
|
struct sock *sk, *first = NULL;
|
2014-07-16 03:28:31 +00:00
|
|
|
unsigned short hnum = ntohs(uh->dest);
|
|
|
|
struct udp_hslot *hslot = udp_hashslot(udptable, net, hnum);
|
2014-07-16 03:28:32 +00:00
|
|
|
unsigned int hash2 = 0, hash2_any = 0, use_hash2 = (hslot->count > 10);
|
2016-04-01 15:52:13 +00:00
|
|
|
unsigned int offset = offsetof(typeof(*sk), sk_node);
|
|
|
|
int dif = skb->dev->ifindex;
|
2017-08-07 15:44:16 +00:00
|
|
|
int sdif = inet_sdif(skb);
|
2016-04-01 15:52:13 +00:00
|
|
|
struct hlist_node *node;
|
|
|
|
struct sk_buff *nskb;
|
2014-07-16 03:28:32 +00:00
|
|
|
|
|
|
|
if (use_hash2) {
|
2017-12-01 20:52:30 +00:00
|
|
|
hash2_any = ipv4_portaddr_hash(net, htonl(INADDR_ANY), hnum) &
|
2016-11-14 22:40:30 +00:00
|
|
|
udptable->mask;
|
2017-12-01 20:52:30 +00:00
|
|
|
hash2 = ipv4_portaddr_hash(net, daddr, hnum) & udptable->mask;
|
2014-07-16 03:28:32 +00:00
|
|
|
start_lookup:
|
2016-11-14 22:40:30 +00:00
|
|
|
hslot = &udptable->hash2[hash2];
|
2014-07-16 03:28:32 +00:00
|
|
|
offset = offsetof(typeof(*sk), __sk_common.skc_portaddr_node);
|
|
|
|
}
|
2008-03-07 00:22:02 +00:00
|
|
|
|
2016-04-01 15:52:13 +00:00
|
|
|
sk_for_each_entry_offset_rcu(sk, node, &hslot->head, offset) {
|
|
|
|
if (!__udp_is_mcast_sock(net, sk, uh->dest, daddr,
|
2017-08-07 15:44:16 +00:00
|
|
|
uh->source, saddr, dif, sdif, hnum))
|
2016-04-01 15:52:13 +00:00
|
|
|
continue;
|
|
|
|
|
|
|
|
if (!first) {
|
|
|
|
first = sk;
|
|
|
|
continue;
|
2009-11-08 10:18:44 +00:00
|
|
|
}
|
2016-04-01 15:52:13 +00:00
|
|
|
nskb = skb_clone(skb, GFP_ATOMIC);
|
2009-11-08 10:18:44 +00:00
|
|
|
|
2016-04-01 15:52:13 +00:00
|
|
|
if (unlikely(!nskb)) {
|
|
|
|
atomic_inc(&sk->sk_drops);
|
2016-04-27 23:44:30 +00:00
|
|
|
__UDP_INC_STATS(net, UDP_MIB_RCVBUFERRORS,
|
|
|
|
IS_UDPLITE(sk));
|
|
|
|
__UDP_INC_STATS(net, UDP_MIB_INERRORS,
|
|
|
|
IS_UDPLITE(sk));
|
2016-04-01 15:52:13 +00:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
if (udp_queue_rcv_skb(sk, nskb) > 0)
|
|
|
|
consume_skb(nskb);
|
|
|
|
}
|
2009-11-08 10:18:44 +00:00
|
|
|
|
2014-07-16 03:28:32 +00:00
|
|
|
/* Also lookup *:port if we are using hash2 and haven't done so yet. */
|
|
|
|
if (use_hash2 && hash2 != hash2_any) {
|
|
|
|
hash2 = hash2_any;
|
|
|
|
goto start_lookup;
|
|
|
|
}
|
|
|
|
|
2016-04-01 15:52:13 +00:00
|
|
|
if (first) {
|
|
|
|
if (udp_queue_rcv_skb(first, skb) > 0)
|
|
|
|
consume_skb(skb);
|
2009-11-08 10:18:44 +00:00
|
|
|
} else {
|
2016-04-01 15:52:13 +00:00
|
|
|
kfree_skb(skb);
|
2016-04-27 23:44:30 +00:00
|
|
|
__UDP_INC_STATS(net, UDP_MIB_IGNOREDMULTI,
|
|
|
|
proto == IPPROTO_UDPLITE);
|
2009-11-08 10:18:44 +00:00
|
|
|
}
|
2008-03-07 00:22:02 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Initialize UDP checksum. If exited with zero value (success),
|
|
|
|
* CHECKSUM_UNNECESSARY means, that no more checks are required.
|
2019-07-18 02:19:23 +00:00
|
|
|
* Otherwise, csum completion requires checksumming packet body,
|
2008-03-07 00:22:02 +00:00
|
|
|
* including udp header and folding it to skb->csum.
|
|
|
|
*/
|
|
|
|
static inline int udp4_csum_init(struct sk_buff *skb, struct udphdr *uh,
|
|
|
|
int proto)
|
|
|
|
{
|
|
|
|
int err;
|
|
|
|
|
|
|
|
UDP_SKB_CB(skb)->partial_cov = 0;
|
|
|
|
UDP_SKB_CB(skb)->cscov = skb->len;
|
|
|
|
|
|
|
|
if (proto == IPPROTO_UDPLITE) {
|
|
|
|
err = udplite_checksum_init(skb, uh);
|
|
|
|
if (err)
|
|
|
|
return err;
|
2018-02-15 17:18:43 +00:00
|
|
|
|
|
|
|
if (UDP_SKB_CB(skb)->partial_cov) {
|
|
|
|
skb->csum = inet_compute_pseudo(skb, proto);
|
|
|
|
return 0;
|
|
|
|
}
|
2008-03-07 00:22:02 +00:00
|
|
|
}
|
|
|
|
|
2016-06-12 10:02:46 +00:00
|
|
|
/* Note, we are only interested in != 0 or == 0, thus the
|
|
|
|
* force to int.
|
|
|
|
*/
|
net: udp: fix handling of CHECKSUM_COMPLETE packets
Current handling of CHECKSUM_COMPLETE packets by the UDP stack is
incorrect for any packet that has an incorrect checksum value.
udp4/6_csum_init() will both make a call to
__skb_checksum_validate_complete() to initialize/validate the csum
field when receiving a CHECKSUM_COMPLETE packet. When this packet
fails validation, skb->csum will be overwritten with the pseudoheader
checksum so the packet can be fully validated by software, but the
skb->ip_summed value will be left as CHECKSUM_COMPLETE so that way
the stack can later warn the user about their hardware spewing bad
checksums. Unfortunately, leaving the SKB in this state can cause
problems later on in the checksum calculation.
Since the the packet is still marked as CHECKSUM_COMPLETE,
udp_csum_pull_header() will SUBTRACT the checksum of the UDP header
from skb->csum instead of adding it, leaving us with a garbage value
in that field. Once we try to copy the packet to userspace in the
udp4/6_recvmsg(), we'll make a call to skb_copy_and_csum_datagram_msg()
to checksum the packet data and add it in the garbage skb->csum value
to perform our final validation check.
Since the value we're validating is not the proper checksum, it's possible
that the folded value could come out to 0, causing us not to drop the
packet. Instead, we believe that the packet was checksummed incorrectly
by hardware since skb->ip_summed is still CHECKSUM_COMPLETE, and we attempt
to warn the user with netdev_rx_csum_fault(skb->dev);
Unfortunately, since this is the UDP path, skb->dev has been overwritten
by skb->dev_scratch and is no longer a valid pointer, so we end up
reading invalid memory.
This patch addresses this problem in two ways:
1) Do not use the dev pointer when calling netdev_rx_csum_fault()
from skb_copy_and_csum_datagram_msg(). Since this gets called
from the UDP path where skb->dev has been overwritten, we have
no way of knowing if the pointer is still valid. Also for the
sake of consistency with the other uses of
netdev_rx_csum_fault(), don't attempt to call it if the
packet was checksummed by software.
2) Add better CHECKSUM_COMPLETE handling to udp4/6_csum_init().
If we receive a packet that's CHECKSUM_COMPLETE that fails
verification (i.e. skb->csum_valid == 0), check who performed
the calculation. It's possible that the checksum was done in
software by the network stack earlier (such as Netfilter's
CONNTRACK module), and if that says the checksum is bad,
we can drop the packet immediately instead of waiting until
we try and copy it to userspace. Otherwise, we need to
mark the SKB as CHECKSUM_NONE, since the skb->csum field
no longer contains the full packet checksum after the
call to __skb_checksum_validate_complete().
Fixes: e6afc8ace6dd ("udp: remove headers from UDP packets before queueing")
Fixes: c84d949057ca ("udp: copy skb->truesize in the first cache line")
Cc: Sam Kumar <samanthakumar@google.com>
Cc: Eric Dumazet <edumazet@google.com>
Signed-off-by: Sean Tranchetti <stranche@codeaurora.org>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-10-23 22:04:31 +00:00
|
|
|
err = (__force int)skb_checksum_init_zero_check(skb, proto, uh->check,
|
|
|
|
inet_compute_pseudo);
|
|
|
|
if (err)
|
|
|
|
return err;
|
|
|
|
|
|
|
|
if (skb->ip_summed == CHECKSUM_COMPLETE && !skb->csum_valid) {
|
|
|
|
/* If SW calculated the value, we know it's bad */
|
|
|
|
if (skb->csum_complete_sw)
|
|
|
|
return 1;
|
|
|
|
|
|
|
|
/* HW says the value is bad. Let's validate that.
|
|
|
|
* skb->csum is no longer the full packet checksum,
|
|
|
|
* so don't treat it as such.
|
|
|
|
*/
|
|
|
|
skb_checksum_complete_unset(skb);
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
2008-03-07 00:22:02 +00:00
|
|
|
}
|
|
|
|
|
2018-09-13 14:27:20 +00:00
|
|
|
/* wrapper for udp_queue_rcv_skb tacking care of csum conversion and
|
|
|
|
* return code conversion for ip layer consumption
|
|
|
|
*/
|
|
|
|
static int udp_unicast_rcv_skb(struct sock *sk, struct sk_buff *skb,
|
|
|
|
struct udphdr *uh)
|
|
|
|
{
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
if (inet_get_convert_csum(sk) && uh->check && !IS_UDPLITE(sk))
|
2019-07-04 09:03:26 +00:00
|
|
|
skb_checksum_try_convert(skb, IPPROTO_UDP, inet_compute_pseudo);
|
2018-09-13 14:27:20 +00:00
|
|
|
|
|
|
|
ret = udp_queue_rcv_skb(sk, skb);
|
|
|
|
|
|
|
|
/* a return value > 0 means to resubmit the input, but
|
|
|
|
* it wants the return to be -protocol, or 0
|
|
|
|
*/
|
|
|
|
if (ret > 0)
|
|
|
|
return -ret;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2008-03-07 00:22:02 +00:00
|
|
|
/*
|
|
|
|
* All we need to do is get the socket, and then do a checksum.
|
|
|
|
*/
|
|
|
|
|
2008-10-29 08:41:45 +00:00
|
|
|
int __udp4_lib_rcv(struct sk_buff *skb, struct udp_table *udptable,
|
2008-03-07 00:22:02 +00:00
|
|
|
int proto)
|
|
|
|
{
|
|
|
|
struct sock *sk;
|
2009-02-05 23:05:45 +00:00
|
|
|
struct udphdr *uh;
|
2008-03-07 00:22:02 +00:00
|
|
|
unsigned short ulen;
|
2009-06-02 05:19:30 +00:00
|
|
|
struct rtable *rt = skb_rtable(skb);
|
2009-02-06 09:59:12 +00:00
|
|
|
__be32 saddr, daddr;
|
2008-07-06 04:18:48 +00:00
|
|
|
struct net *net = dev_net(skb->dev);
|
2020-03-29 22:53:39 +00:00
|
|
|
bool refcounted;
|
2022-01-09 06:36:28 +00:00
|
|
|
int drop_reason;
|
|
|
|
|
|
|
|
drop_reason = SKB_DROP_REASON_NOT_SPECIFIED;
|
2008-03-07 00:22:02 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Validate the packet.
|
|
|
|
*/
|
|
|
|
if (!pskb_may_pull(skb, sizeof(struct udphdr)))
|
|
|
|
goto drop; /* No space for header. */
|
|
|
|
|
2009-02-05 23:05:45 +00:00
|
|
|
uh = udp_hdr(skb);
|
2008-03-07 00:22:02 +00:00
|
|
|
ulen = ntohs(uh->len);
|
2010-05-06 03:44:34 +00:00
|
|
|
saddr = ip_hdr(skb)->saddr;
|
|
|
|
daddr = ip_hdr(skb)->daddr;
|
|
|
|
|
2008-03-07 00:22:02 +00:00
|
|
|
if (ulen > skb->len)
|
|
|
|
goto short_packet;
|
|
|
|
|
|
|
|
if (proto == IPPROTO_UDP) {
|
|
|
|
/* UDP validates ulen. */
|
|
|
|
if (ulen < sizeof(*uh) || pskb_trim_rcsum(skb, ulen))
|
|
|
|
goto short_packet;
|
|
|
|
uh = udp_hdr(skb);
|
|
|
|
}
|
|
|
|
|
|
|
|
if (udp4_csum_init(skb, uh, proto))
|
|
|
|
goto csum_error;
|
|
|
|
|
bpf, net: Support SO_REUSEPORT sockets with bpf_sk_assign
Currently the bpf_sk_assign helper in tc BPF context refuses SO_REUSEPORT
sockets. This means we can't use the helper to steer traffic to Envoy,
which configures SO_REUSEPORT on its sockets. In turn, we're blocked
from removing TPROXY from our setup.
The reason that bpf_sk_assign refuses such sockets is that the
bpf_sk_lookup helpers don't execute SK_REUSEPORT programs. Instead,
one of the reuseport sockets is selected by hash. This could cause
dispatch to the "wrong" socket:
sk = bpf_sk_lookup_tcp(...) // select SO_REUSEPORT by hash
bpf_sk_assign(skb, sk) // SK_REUSEPORT wasn't executed
Fixing this isn't as simple as invoking SK_REUSEPORT from the lookup
helpers unfortunately. In the tc context, L2 headers are at the start
of the skb, while SK_REUSEPORT expects L3 headers instead.
Instead, we execute the SK_REUSEPORT program when the assigned socket
is pulled out of the skb, further up the stack. This creates some
trickiness with regards to refcounting as bpf_sk_assign will put both
refcounted and RCU freed sockets in skb->sk. reuseport sockets are RCU
freed. We can infer that the sk_assigned socket is RCU freed if the
reuseport lookup succeeds, but convincing yourself of this fact isn't
straight forward. Therefore we defensively check refcounting on the
sk_assign sock even though it's probably not required in practice.
Fixes: 8e368dc72e86 ("bpf: Fix use of sk->sk_reuseport from sk_assign")
Fixes: cf7fbe660f2d ("bpf: Add socket assign support")
Co-developed-by: Daniel Borkmann <daniel@iogearbox.net>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Cc: Joe Stringer <joe@cilium.io>
Link: https://lore.kernel.org/bpf/CACAyw98+qycmpQzKupquhkxbvWK4OFyDuuLMBNROnfWMZxUWeA@mail.gmail.com/
Reviewed-by: Kuniyuki Iwashima <kuniyu@amazon.com>
Signed-off-by: Lorenz Bauer <lmb@isovalent.com>
Link: https://lore.kernel.org/r/20230720-so-reuseport-v6-7-7021b683cdae@isovalent.com
Signed-off-by: Martin KaFai Lau <martin.lau@kernel.org>
2023-07-20 15:30:11 +00:00
|
|
|
sk = inet_steal_sock(net, skb, sizeof(struct udphdr), saddr, uh->source, daddr, uh->dest,
|
|
|
|
&refcounted, udp_ehashfn);
|
|
|
|
if (IS_ERR(sk))
|
|
|
|
goto no_sk;
|
|
|
|
|
2013-12-11 02:07:23 +00:00
|
|
|
if (sk) {
|
2013-12-11 22:46:51 +00:00
|
|
|
struct dst_entry *dst = skb_dst(skb);
|
2013-10-07 16:01:39 +00:00
|
|
|
int ret;
|
|
|
|
|
inet: fully convert sk->sk_rx_dst to RCU rules
syzbot reported various issues around early demux,
one being included in this changelog [1]
sk->sk_rx_dst is using RCU protection without clearly
documenting it.
And following sequences in tcp_v4_do_rcv()/tcp_v6_do_rcv()
are not following standard RCU rules.
[a] dst_release(dst);
[b] sk->sk_rx_dst = NULL;
They look wrong because a delete operation of RCU protected
pointer is supposed to clear the pointer before
the call_rcu()/synchronize_rcu() guarding actual memory freeing.
In some cases indeed, dst could be freed before [b] is done.
We could cheat by clearing sk_rx_dst before calling
dst_release(), but this seems the right time to stick
to standard RCU annotations and debugging facilities.
[1]
BUG: KASAN: use-after-free in dst_check include/net/dst.h:470 [inline]
BUG: KASAN: use-after-free in tcp_v4_early_demux+0x95b/0x960 net/ipv4/tcp_ipv4.c:1792
Read of size 2 at addr ffff88807f1cb73a by task syz-executor.5/9204
CPU: 0 PID: 9204 Comm: syz-executor.5 Not tainted 5.16.0-rc5-syzkaller #0
Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011
Call Trace:
<TASK>
__dump_stack lib/dump_stack.c:88 [inline]
dump_stack_lvl+0xcd/0x134 lib/dump_stack.c:106
print_address_description.constprop.0.cold+0x8d/0x320 mm/kasan/report.c:247
__kasan_report mm/kasan/report.c:433 [inline]
kasan_report.cold+0x83/0xdf mm/kasan/report.c:450
dst_check include/net/dst.h:470 [inline]
tcp_v4_early_demux+0x95b/0x960 net/ipv4/tcp_ipv4.c:1792
ip_rcv_finish_core.constprop.0+0x15de/0x1e80 net/ipv4/ip_input.c:340
ip_list_rcv_finish.constprop.0+0x1b2/0x6e0 net/ipv4/ip_input.c:583
ip_sublist_rcv net/ipv4/ip_input.c:609 [inline]
ip_list_rcv+0x34e/0x490 net/ipv4/ip_input.c:644
__netif_receive_skb_list_ptype net/core/dev.c:5508 [inline]
__netif_receive_skb_list_core+0x549/0x8e0 net/core/dev.c:5556
__netif_receive_skb_list net/core/dev.c:5608 [inline]
netif_receive_skb_list_internal+0x75e/0xd80 net/core/dev.c:5699
gro_normal_list net/core/dev.c:5853 [inline]
gro_normal_list net/core/dev.c:5849 [inline]
napi_complete_done+0x1f1/0x880 net/core/dev.c:6590
virtqueue_napi_complete drivers/net/virtio_net.c:339 [inline]
virtnet_poll+0xca2/0x11b0 drivers/net/virtio_net.c:1557
__napi_poll+0xaf/0x440 net/core/dev.c:7023
napi_poll net/core/dev.c:7090 [inline]
net_rx_action+0x801/0xb40 net/core/dev.c:7177
__do_softirq+0x29b/0x9c2 kernel/softirq.c:558
invoke_softirq kernel/softirq.c:432 [inline]
__irq_exit_rcu+0x123/0x180 kernel/softirq.c:637
irq_exit_rcu+0x5/0x20 kernel/softirq.c:649
common_interrupt+0x52/0xc0 arch/x86/kernel/irq.c:240
asm_common_interrupt+0x1e/0x40 arch/x86/include/asm/idtentry.h:629
RIP: 0033:0x7f5e972bfd57
Code: 39 d1 73 14 0f 1f 80 00 00 00 00 48 8b 50 f8 48 83 e8 08 48 39 ca 77 f3 48 39 c3 73 3e 48 89 13 48 8b 50 f8 48 89 38 49 8b 0e <48> 8b 3e 48 83 c3 08 48 83 c6 08 eb bc 48 39 d1 72 9e 48 39 d0 73
RSP: 002b:00007fff8a413210 EFLAGS: 00000283
RAX: 00007f5e97108990 RBX: 00007f5e97108338 RCX: ffffffff81d3aa45
RDX: ffffffff81d3aa45 RSI: 00007f5e97108340 RDI: ffffffff81d3aa45
RBP: 00007f5e97107eb8 R08: 00007f5e97108d88 R09: 0000000093c2e8d9
R10: 0000000000000000 R11: 0000000000000000 R12: 00007f5e97107eb0
R13: 00007f5e97108338 R14: 00007f5e97107ea8 R15: 0000000000000019
</TASK>
Allocated by task 13:
kasan_save_stack+0x1e/0x50 mm/kasan/common.c:38
kasan_set_track mm/kasan/common.c:46 [inline]
set_alloc_info mm/kasan/common.c:434 [inline]
__kasan_slab_alloc+0x90/0xc0 mm/kasan/common.c:467
kasan_slab_alloc include/linux/kasan.h:259 [inline]
slab_post_alloc_hook mm/slab.h:519 [inline]
slab_alloc_node mm/slub.c:3234 [inline]
slab_alloc mm/slub.c:3242 [inline]
kmem_cache_alloc+0x202/0x3a0 mm/slub.c:3247
dst_alloc+0x146/0x1f0 net/core/dst.c:92
rt_dst_alloc+0x73/0x430 net/ipv4/route.c:1613
ip_route_input_slow+0x1817/0x3a20 net/ipv4/route.c:2340
ip_route_input_rcu net/ipv4/route.c:2470 [inline]
ip_route_input_noref+0x116/0x2a0 net/ipv4/route.c:2415
ip_rcv_finish_core.constprop.0+0x288/0x1e80 net/ipv4/ip_input.c:354
ip_list_rcv_finish.constprop.0+0x1b2/0x6e0 net/ipv4/ip_input.c:583
ip_sublist_rcv net/ipv4/ip_input.c:609 [inline]
ip_list_rcv+0x34e/0x490 net/ipv4/ip_input.c:644
__netif_receive_skb_list_ptype net/core/dev.c:5508 [inline]
__netif_receive_skb_list_core+0x549/0x8e0 net/core/dev.c:5556
__netif_receive_skb_list net/core/dev.c:5608 [inline]
netif_receive_skb_list_internal+0x75e/0xd80 net/core/dev.c:5699
gro_normal_list net/core/dev.c:5853 [inline]
gro_normal_list net/core/dev.c:5849 [inline]
napi_complete_done+0x1f1/0x880 net/core/dev.c:6590
virtqueue_napi_complete drivers/net/virtio_net.c:339 [inline]
virtnet_poll+0xca2/0x11b0 drivers/net/virtio_net.c:1557
__napi_poll+0xaf/0x440 net/core/dev.c:7023
napi_poll net/core/dev.c:7090 [inline]
net_rx_action+0x801/0xb40 net/core/dev.c:7177
__do_softirq+0x29b/0x9c2 kernel/softirq.c:558
Freed by task 13:
kasan_save_stack+0x1e/0x50 mm/kasan/common.c:38
kasan_set_track+0x21/0x30 mm/kasan/common.c:46
kasan_set_free_info+0x20/0x30 mm/kasan/generic.c:370
____kasan_slab_free mm/kasan/common.c:366 [inline]
____kasan_slab_free mm/kasan/common.c:328 [inline]
__kasan_slab_free+0xff/0x130 mm/kasan/common.c:374
kasan_slab_free include/linux/kasan.h:235 [inline]
slab_free_hook mm/slub.c:1723 [inline]
slab_free_freelist_hook+0x8b/0x1c0 mm/slub.c:1749
slab_free mm/slub.c:3513 [inline]
kmem_cache_free+0xbd/0x5d0 mm/slub.c:3530
dst_destroy+0x2d6/0x3f0 net/core/dst.c:127
rcu_do_batch kernel/rcu/tree.c:2506 [inline]
rcu_core+0x7ab/0x1470 kernel/rcu/tree.c:2741
__do_softirq+0x29b/0x9c2 kernel/softirq.c:558
Last potentially related work creation:
kasan_save_stack+0x1e/0x50 mm/kasan/common.c:38
__kasan_record_aux_stack+0xf5/0x120 mm/kasan/generic.c:348
__call_rcu kernel/rcu/tree.c:2985 [inline]
call_rcu+0xb1/0x740 kernel/rcu/tree.c:3065
dst_release net/core/dst.c:177 [inline]
dst_release+0x79/0xe0 net/core/dst.c:167
tcp_v4_do_rcv+0x612/0x8d0 net/ipv4/tcp_ipv4.c:1712
sk_backlog_rcv include/net/sock.h:1030 [inline]
__release_sock+0x134/0x3b0 net/core/sock.c:2768
release_sock+0x54/0x1b0 net/core/sock.c:3300
tcp_sendmsg+0x36/0x40 net/ipv4/tcp.c:1441
inet_sendmsg+0x99/0xe0 net/ipv4/af_inet.c:819
sock_sendmsg_nosec net/socket.c:704 [inline]
sock_sendmsg+0xcf/0x120 net/socket.c:724
sock_write_iter+0x289/0x3c0 net/socket.c:1057
call_write_iter include/linux/fs.h:2162 [inline]
new_sync_write+0x429/0x660 fs/read_write.c:503
vfs_write+0x7cd/0xae0 fs/read_write.c:590
ksys_write+0x1ee/0x250 fs/read_write.c:643
do_syscall_x64 arch/x86/entry/common.c:50 [inline]
do_syscall_64+0x35/0xb0 arch/x86/entry/common.c:80
entry_SYSCALL_64_after_hwframe+0x44/0xae
The buggy address belongs to the object at ffff88807f1cb700
which belongs to the cache ip_dst_cache of size 176
The buggy address is located 58 bytes inside of
176-byte region [ffff88807f1cb700, ffff88807f1cb7b0)
The buggy address belongs to the page:
page:ffffea0001fc72c0 refcount:1 mapcount:0 mapping:0000000000000000 index:0x0 pfn:0x7f1cb
flags: 0xfff00000000200(slab|node=0|zone=1|lastcpupid=0x7ff)
raw: 00fff00000000200 dead000000000100 dead000000000122 ffff8881413bb780
raw: 0000000000000000 0000000000100010 00000001ffffffff 0000000000000000
page dumped because: kasan: bad access detected
page_owner tracks the page as allocated
page last allocated via order 0, migratetype Unmovable, gfp_mask 0x112a20(GFP_ATOMIC|__GFP_NOWARN|__GFP_NORETRY|__GFP_HARDWALL), pid 5, ts 108466983062, free_ts 108048976062
prep_new_page mm/page_alloc.c:2418 [inline]
get_page_from_freelist+0xa72/0x2f50 mm/page_alloc.c:4149
__alloc_pages+0x1b2/0x500 mm/page_alloc.c:5369
alloc_pages+0x1a7/0x300 mm/mempolicy.c:2191
alloc_slab_page mm/slub.c:1793 [inline]
allocate_slab mm/slub.c:1930 [inline]
new_slab+0x32d/0x4a0 mm/slub.c:1993
___slab_alloc+0x918/0xfe0 mm/slub.c:3022
__slab_alloc.constprop.0+0x4d/0xa0 mm/slub.c:3109
slab_alloc_node mm/slub.c:3200 [inline]
slab_alloc mm/slub.c:3242 [inline]
kmem_cache_alloc+0x35c/0x3a0 mm/slub.c:3247
dst_alloc+0x146/0x1f0 net/core/dst.c:92
rt_dst_alloc+0x73/0x430 net/ipv4/route.c:1613
__mkroute_output net/ipv4/route.c:2564 [inline]
ip_route_output_key_hash_rcu+0x921/0x2d00 net/ipv4/route.c:2791
ip_route_output_key_hash+0x18b/0x300 net/ipv4/route.c:2619
__ip_route_output_key include/net/route.h:126 [inline]
ip_route_output_flow+0x23/0x150 net/ipv4/route.c:2850
ip_route_output_key include/net/route.h:142 [inline]
geneve_get_v4_rt+0x3a6/0x830 drivers/net/geneve.c:809
geneve_xmit_skb drivers/net/geneve.c:899 [inline]
geneve_xmit+0xc4a/0x3540 drivers/net/geneve.c:1082
__netdev_start_xmit include/linux/netdevice.h:4994 [inline]
netdev_start_xmit include/linux/netdevice.h:5008 [inline]
xmit_one net/core/dev.c:3590 [inline]
dev_hard_start_xmit+0x1eb/0x920 net/core/dev.c:3606
__dev_queue_xmit+0x299a/0x3650 net/core/dev.c:4229
page last free stack trace:
reset_page_owner include/linux/page_owner.h:24 [inline]
free_pages_prepare mm/page_alloc.c:1338 [inline]
free_pcp_prepare+0x374/0x870 mm/page_alloc.c:1389
free_unref_page_prepare mm/page_alloc.c:3309 [inline]
free_unref_page+0x19/0x690 mm/page_alloc.c:3388
qlink_free mm/kasan/quarantine.c:146 [inline]
qlist_free_all+0x5a/0xc0 mm/kasan/quarantine.c:165
kasan_quarantine_reduce+0x180/0x200 mm/kasan/quarantine.c:272
__kasan_slab_alloc+0xa2/0xc0 mm/kasan/common.c:444
kasan_slab_alloc include/linux/kasan.h:259 [inline]
slab_post_alloc_hook mm/slab.h:519 [inline]
slab_alloc_node mm/slub.c:3234 [inline]
kmem_cache_alloc_node+0x255/0x3f0 mm/slub.c:3270
__alloc_skb+0x215/0x340 net/core/skbuff.c:414
alloc_skb include/linux/skbuff.h:1126 [inline]
alloc_skb_with_frags+0x93/0x620 net/core/skbuff.c:6078
sock_alloc_send_pskb+0x783/0x910 net/core/sock.c:2575
mld_newpack+0x1df/0x770 net/ipv6/mcast.c:1754
add_grhead+0x265/0x330 net/ipv6/mcast.c:1857
add_grec+0x1053/0x14e0 net/ipv6/mcast.c:1995
mld_send_initial_cr.part.0+0xf6/0x230 net/ipv6/mcast.c:2242
mld_send_initial_cr net/ipv6/mcast.c:1232 [inline]
mld_dad_work+0x1d3/0x690 net/ipv6/mcast.c:2268
process_one_work+0x9b2/0x1690 kernel/workqueue.c:2298
worker_thread+0x658/0x11f0 kernel/workqueue.c:2445
Memory state around the buggy address:
ffff88807f1cb600: fa fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb
ffff88807f1cb680: fb fb fb fb fb fb fc fc fc fc fc fc fc fc fc fc
>ffff88807f1cb700: fa fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb
^
ffff88807f1cb780: fb fb fb fb fb fb fc fc fc fc fc fc fc fc fc fc
ffff88807f1cb800: fa fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb
Fixes: 41063e9dd119 ("ipv4: Early TCP socket demux.")
Signed-off-by: Eric Dumazet <edumazet@google.com>
Link: https://lore.kernel.org/r/20211220143330.680945-1-eric.dumazet@gmail.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
2021-12-20 14:33:30 +00:00
|
|
|
if (unlikely(rcu_dereference(sk->sk_rx_dst) != dst))
|
2013-12-11 22:46:51 +00:00
|
|
|
udp_sk_rx_dst_set(sk, dst);
|
2008-03-07 00:22:02 +00:00
|
|
|
|
2018-09-13 14:27:20 +00:00
|
|
|
ret = udp_unicast_rcv_skb(sk, skb, uh);
|
2020-03-29 22:53:39 +00:00
|
|
|
if (refcounted)
|
|
|
|
sock_put(sk);
|
2018-09-13 14:27:20 +00:00
|
|
|
return ret;
|
2013-10-07 16:01:39 +00:00
|
|
|
}
|
2008-03-07 00:22:02 +00:00
|
|
|
|
2014-11-04 19:48:41 +00:00
|
|
|
if (rt->rt_flags & (RTCF_BROADCAST|RTCF_MULTICAST))
|
|
|
|
return __udp4_lib_mcast_deliver(net, skb, uh,
|
2014-11-06 18:37:54 +00:00
|
|
|
saddr, daddr, udptable, proto);
|
2014-11-04 19:48:41 +00:00
|
|
|
|
|
|
|
sk = __udp4_lib_lookup_skb(skb, uh->source, uh->dest, udptable);
|
2018-09-13 14:27:20 +00:00
|
|
|
if (sk)
|
|
|
|
return udp_unicast_rcv_skb(sk, skb, uh);
|
bpf, net: Support SO_REUSEPORT sockets with bpf_sk_assign
Currently the bpf_sk_assign helper in tc BPF context refuses SO_REUSEPORT
sockets. This means we can't use the helper to steer traffic to Envoy,
which configures SO_REUSEPORT on its sockets. In turn, we're blocked
from removing TPROXY from our setup.
The reason that bpf_sk_assign refuses such sockets is that the
bpf_sk_lookup helpers don't execute SK_REUSEPORT programs. Instead,
one of the reuseport sockets is selected by hash. This could cause
dispatch to the "wrong" socket:
sk = bpf_sk_lookup_tcp(...) // select SO_REUSEPORT by hash
bpf_sk_assign(skb, sk) // SK_REUSEPORT wasn't executed
Fixing this isn't as simple as invoking SK_REUSEPORT from the lookup
helpers unfortunately. In the tc context, L2 headers are at the start
of the skb, while SK_REUSEPORT expects L3 headers instead.
Instead, we execute the SK_REUSEPORT program when the assigned socket
is pulled out of the skb, further up the stack. This creates some
trickiness with regards to refcounting as bpf_sk_assign will put both
refcounted and RCU freed sockets in skb->sk. reuseport sockets are RCU
freed. We can infer that the sk_assigned socket is RCU freed if the
reuseport lookup succeeds, but convincing yourself of this fact isn't
straight forward. Therefore we defensively check refcounting on the
sk_assign sock even though it's probably not required in practice.
Fixes: 8e368dc72e86 ("bpf: Fix use of sk->sk_reuseport from sk_assign")
Fixes: cf7fbe660f2d ("bpf: Add socket assign support")
Co-developed-by: Daniel Borkmann <daniel@iogearbox.net>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Cc: Joe Stringer <joe@cilium.io>
Link: https://lore.kernel.org/bpf/CACAyw98+qycmpQzKupquhkxbvWK4OFyDuuLMBNROnfWMZxUWeA@mail.gmail.com/
Reviewed-by: Kuniyuki Iwashima <kuniyu@amazon.com>
Signed-off-by: Lorenz Bauer <lmb@isovalent.com>
Link: https://lore.kernel.org/r/20230720-so-reuseport-v6-7-7021b683cdae@isovalent.com
Signed-off-by: Martin KaFai Lau <martin.lau@kernel.org>
2023-07-20 15:30:11 +00:00
|
|
|
no_sk:
|
2008-03-07 00:22:02 +00:00
|
|
|
if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb))
|
|
|
|
goto drop;
|
2019-09-29 18:54:03 +00:00
|
|
|
nf_reset_ct(skb);
|
2008-03-07 00:22:02 +00:00
|
|
|
|
|
|
|
/* No socket. Drop packet silently, if checksum is wrong */
|
|
|
|
if (udp_lib_checksum_complete(skb))
|
|
|
|
goto csum_error;
|
|
|
|
|
2022-01-09 06:36:28 +00:00
|
|
|
drop_reason = SKB_DROP_REASON_NO_SOCKET;
|
2016-04-27 23:44:30 +00:00
|
|
|
__UDP_INC_STATS(net, UDP_MIB_NOPORTS, proto == IPPROTO_UDPLITE);
|
2008-03-07 00:22:02 +00:00
|
|
|
icmp_send(skb, ICMP_DEST_UNREACH, ICMP_PORT_UNREACH, 0);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Hmm. We got an UDP packet to a port to which we
|
|
|
|
* don't wanna listen. Ignore it.
|
|
|
|
*/
|
2022-01-09 06:36:28 +00:00
|
|
|
kfree_skb_reason(skb, drop_reason);
|
2008-03-07 00:22:02 +00:00
|
|
|
return 0;
|
|
|
|
|
|
|
|
short_packet:
|
2022-01-09 06:36:28 +00:00
|
|
|
drop_reason = SKB_DROP_REASON_PKT_TOO_SMALL;
|
2014-11-11 18:59:17 +00:00
|
|
|
net_dbg_ratelimited("UDP%s: short packet: From %pI4:%u %d/%d to %pI4:%u\n",
|
|
|
|
proto == IPPROTO_UDPLITE ? "Lite" : "",
|
|
|
|
&saddr, ntohs(uh->source),
|
|
|
|
ulen, skb->len,
|
|
|
|
&daddr, ntohs(uh->dest));
|
2008-03-07 00:22:02 +00:00
|
|
|
goto drop;
|
|
|
|
|
|
|
|
csum_error:
|
|
|
|
/*
|
|
|
|
* RFC1122: OK. Discards the bad packet silently (as far as
|
|
|
|
* the network is concerned, anyway) as per 4.1.3.4 (MUST).
|
|
|
|
*/
|
2022-01-09 06:36:28 +00:00
|
|
|
drop_reason = SKB_DROP_REASON_UDP_CSUM;
|
2014-11-11 18:59:17 +00:00
|
|
|
net_dbg_ratelimited("UDP%s: bad checksum. From %pI4:%u to %pI4:%u ulen %d\n",
|
|
|
|
proto == IPPROTO_UDPLITE ? "Lite" : "",
|
|
|
|
&saddr, ntohs(uh->source), &daddr, ntohs(uh->dest),
|
|
|
|
ulen);
|
2016-04-27 23:44:30 +00:00
|
|
|
__UDP_INC_STATS(net, UDP_MIB_CSUMERRORS, proto == IPPROTO_UDPLITE);
|
2008-03-07 00:22:02 +00:00
|
|
|
drop:
|
2016-04-27 23:44:30 +00:00
|
|
|
__UDP_INC_STATS(net, UDP_MIB_INERRORS, proto == IPPROTO_UDPLITE);
|
2022-01-09 06:36:28 +00:00
|
|
|
kfree_skb_reason(skb, drop_reason);
|
2008-03-07 00:22:02 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2013-10-07 16:01:39 +00:00
|
|
|
/* We can only early demux multicast if there is a single matching socket.
|
|
|
|
* If more than one socket found returns NULL
|
|
|
|
*/
|
|
|
|
static struct sock *__udp4_lib_mcast_demux_lookup(struct net *net,
|
|
|
|
__be16 loc_port, __be32 loc_addr,
|
|
|
|
__be16 rmt_port, __be32 rmt_addr,
|
2017-08-07 15:44:16 +00:00
|
|
|
int dif, int sdif)
|
2013-10-07 16:01:39 +00:00
|
|
|
{
|
2022-11-14 21:57:56 +00:00
|
|
|
struct udp_table *udptable = net->ipv4.udp_table;
|
2013-10-07 16:01:39 +00:00
|
|
|
unsigned short hnum = ntohs(loc_port);
|
2022-11-14 21:57:53 +00:00
|
|
|
struct sock *sk, *result;
|
|
|
|
struct udp_hslot *hslot;
|
|
|
|
unsigned int slot;
|
|
|
|
|
2022-11-14 21:57:56 +00:00
|
|
|
slot = udp_hashfn(net, hnum, udptable->mask);
|
|
|
|
hslot = &udptable->hash[slot];
|
2013-10-07 16:01:39 +00:00
|
|
|
|
2014-06-12 23:13:06 +00:00
|
|
|
/* Do not bother scanning a too big list */
|
|
|
|
if (hslot->count > 10)
|
|
|
|
return NULL;
|
|
|
|
|
2013-10-07 16:01:39 +00:00
|
|
|
result = NULL;
|
2016-04-01 15:52:13 +00:00
|
|
|
sk_for_each_rcu(sk, &hslot->head) {
|
|
|
|
if (__udp_is_mcast_sock(net, sk, loc_port, loc_addr,
|
2017-08-07 15:44:16 +00:00
|
|
|
rmt_port, rmt_addr, dif, sdif, hnum)) {
|
2016-04-01 15:52:13 +00:00
|
|
|
if (result)
|
|
|
|
return NULL;
|
2013-10-07 16:01:39 +00:00
|
|
|
result = sk;
|
|
|
|
}
|
|
|
|
}
|
2016-04-01 15:52:13 +00:00
|
|
|
|
2013-10-07 16:01:39 +00:00
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* For unicast we should only early demux connected sockets or we can
|
|
|
|
* break forwarding setups. The chains here can be long so only check
|
|
|
|
* if the first socket is an exact match and if not move on.
|
|
|
|
*/
|
|
|
|
static struct sock *__udp4_lib_demux_lookup(struct net *net,
|
|
|
|
__be16 loc_port, __be32 loc_addr,
|
|
|
|
__be16 rmt_port, __be32 rmt_addr,
|
2017-08-07 15:44:17 +00:00
|
|
|
int dif, int sdif)
|
2013-10-07 16:01:39 +00:00
|
|
|
{
|
2022-11-14 21:57:56 +00:00
|
|
|
struct udp_table *udptable = net->ipv4.udp_table;
|
2014-05-14 03:30:07 +00:00
|
|
|
INET_ADDR_COOKIE(acookie, rmt_addr, loc_addr);
|
2022-11-14 21:57:53 +00:00
|
|
|
unsigned short hnum = ntohs(loc_port);
|
|
|
|
unsigned int hash2, slot2;
|
|
|
|
struct udp_hslot *hslot2;
|
|
|
|
__portpair ports;
|
2016-04-01 15:52:13 +00:00
|
|
|
struct sock *sk;
|
2013-10-07 16:01:39 +00:00
|
|
|
|
2022-11-14 21:57:53 +00:00
|
|
|
hash2 = ipv4_portaddr_hash(net, loc_addr, hnum);
|
2022-11-14 21:57:56 +00:00
|
|
|
slot2 = hash2 & udptable->mask;
|
|
|
|
hslot2 = &udptable->hash2[slot2];
|
2022-11-14 21:57:53 +00:00
|
|
|
ports = INET_COMBINED_PORTS(rmt_port, hnum);
|
|
|
|
|
2016-04-01 15:52:13 +00:00
|
|
|
udp_portaddr_for_each_entry_rcu(sk, &hslot2->head) {
|
2022-05-13 18:55:50 +00:00
|
|
|
if (inet_match(net, sk, acookie, ports, dif, sdif))
|
2016-04-01 15:52:13 +00:00
|
|
|
return sk;
|
2013-10-07 16:01:39 +00:00
|
|
|
/* Only check first socket in chain */
|
|
|
|
break;
|
|
|
|
}
|
2016-04-01 15:52:13 +00:00
|
|
|
return NULL;
|
2013-10-07 16:01:39 +00:00
|
|
|
}
|
|
|
|
|
2017-09-28 13:51:36 +00:00
|
|
|
int udp_v4_early_demux(struct sk_buff *skb)
|
2013-10-07 16:01:39 +00:00
|
|
|
{
|
2013-12-11 16:10:05 +00:00
|
|
|
struct net *net = dev_net(skb->dev);
|
2017-09-28 13:51:37 +00:00
|
|
|
struct in_device *in_dev = NULL;
|
2013-12-11 16:10:05 +00:00
|
|
|
const struct iphdr *iph;
|
|
|
|
const struct udphdr *uh;
|
2016-04-01 15:52:13 +00:00
|
|
|
struct sock *sk = NULL;
|
2013-10-07 16:01:39 +00:00
|
|
|
struct dst_entry *dst;
|
|
|
|
int dif = skb->dev->ifindex;
|
2017-08-07 15:44:16 +00:00
|
|
|
int sdif = inet_sdif(skb);
|
2015-06-03 21:27:38 +00:00
|
|
|
int ours;
|
2013-10-07 16:01:39 +00:00
|
|
|
|
|
|
|
/* validate the packet */
|
|
|
|
if (!pskb_may_pull(skb, skb_transport_offset(skb) + sizeof(struct udphdr)))
|
2017-09-28 13:51:36 +00:00
|
|
|
return 0;
|
2013-10-07 16:01:39 +00:00
|
|
|
|
2013-12-11 16:10:05 +00:00
|
|
|
iph = ip_hdr(skb);
|
|
|
|
uh = udp_hdr(skb);
|
|
|
|
|
2017-10-09 12:52:10 +00:00
|
|
|
if (skb->pkt_type == PACKET_MULTICAST) {
|
2017-09-28 13:51:37 +00:00
|
|
|
in_dev = __in_dev_get_rcu(skb->dev);
|
2015-06-03 21:27:38 +00:00
|
|
|
|
|
|
|
if (!in_dev)
|
2017-09-28 13:51:36 +00:00
|
|
|
return 0;
|
2015-06-03 21:27:38 +00:00
|
|
|
|
2017-10-09 12:52:10 +00:00
|
|
|
ours = ip_check_mc_rcu(in_dev, iph->daddr, iph->saddr,
|
|
|
|
iph->protocol);
|
|
|
|
if (!ours)
|
|
|
|
return 0;
|
2016-03-22 08:19:38 +00:00
|
|
|
|
2013-10-07 16:01:39 +00:00
|
|
|
sk = __udp4_lib_mcast_demux_lookup(net, uh->dest, iph->daddr,
|
2017-08-07 15:44:16 +00:00
|
|
|
uh->source, iph->saddr,
|
|
|
|
dif, sdif);
|
2015-06-03 21:27:38 +00:00
|
|
|
} else if (skb->pkt_type == PACKET_HOST) {
|
2013-10-07 16:01:39 +00:00
|
|
|
sk = __udp4_lib_demux_lookup(net, uh->dest, iph->daddr,
|
2017-08-07 15:44:17 +00:00
|
|
|
uh->source, iph->saddr, dif, sdif);
|
2015-06-03 21:27:38 +00:00
|
|
|
}
|
2013-10-07 16:01:39 +00:00
|
|
|
|
2017-06-30 10:08:01 +00:00
|
|
|
if (!sk || !refcount_inc_not_zero(&sk->sk_refcnt))
|
2017-09-28 13:51:36 +00:00
|
|
|
return 0;
|
2013-10-07 16:01:39 +00:00
|
|
|
|
|
|
|
skb->sk = sk;
|
2014-09-04 17:32:11 +00:00
|
|
|
skb->destructor = sock_efree;
|
inet: fully convert sk->sk_rx_dst to RCU rules
syzbot reported various issues around early demux,
one being included in this changelog [1]
sk->sk_rx_dst is using RCU protection without clearly
documenting it.
And following sequences in tcp_v4_do_rcv()/tcp_v6_do_rcv()
are not following standard RCU rules.
[a] dst_release(dst);
[b] sk->sk_rx_dst = NULL;
They look wrong because a delete operation of RCU protected
pointer is supposed to clear the pointer before
the call_rcu()/synchronize_rcu() guarding actual memory freeing.
In some cases indeed, dst could be freed before [b] is done.
We could cheat by clearing sk_rx_dst before calling
dst_release(), but this seems the right time to stick
to standard RCU annotations and debugging facilities.
[1]
BUG: KASAN: use-after-free in dst_check include/net/dst.h:470 [inline]
BUG: KASAN: use-after-free in tcp_v4_early_demux+0x95b/0x960 net/ipv4/tcp_ipv4.c:1792
Read of size 2 at addr ffff88807f1cb73a by task syz-executor.5/9204
CPU: 0 PID: 9204 Comm: syz-executor.5 Not tainted 5.16.0-rc5-syzkaller #0
Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011
Call Trace:
<TASK>
__dump_stack lib/dump_stack.c:88 [inline]
dump_stack_lvl+0xcd/0x134 lib/dump_stack.c:106
print_address_description.constprop.0.cold+0x8d/0x320 mm/kasan/report.c:247
__kasan_report mm/kasan/report.c:433 [inline]
kasan_report.cold+0x83/0xdf mm/kasan/report.c:450
dst_check include/net/dst.h:470 [inline]
tcp_v4_early_demux+0x95b/0x960 net/ipv4/tcp_ipv4.c:1792
ip_rcv_finish_core.constprop.0+0x15de/0x1e80 net/ipv4/ip_input.c:340
ip_list_rcv_finish.constprop.0+0x1b2/0x6e0 net/ipv4/ip_input.c:583
ip_sublist_rcv net/ipv4/ip_input.c:609 [inline]
ip_list_rcv+0x34e/0x490 net/ipv4/ip_input.c:644
__netif_receive_skb_list_ptype net/core/dev.c:5508 [inline]
__netif_receive_skb_list_core+0x549/0x8e0 net/core/dev.c:5556
__netif_receive_skb_list net/core/dev.c:5608 [inline]
netif_receive_skb_list_internal+0x75e/0xd80 net/core/dev.c:5699
gro_normal_list net/core/dev.c:5853 [inline]
gro_normal_list net/core/dev.c:5849 [inline]
napi_complete_done+0x1f1/0x880 net/core/dev.c:6590
virtqueue_napi_complete drivers/net/virtio_net.c:339 [inline]
virtnet_poll+0xca2/0x11b0 drivers/net/virtio_net.c:1557
__napi_poll+0xaf/0x440 net/core/dev.c:7023
napi_poll net/core/dev.c:7090 [inline]
net_rx_action+0x801/0xb40 net/core/dev.c:7177
__do_softirq+0x29b/0x9c2 kernel/softirq.c:558
invoke_softirq kernel/softirq.c:432 [inline]
__irq_exit_rcu+0x123/0x180 kernel/softirq.c:637
irq_exit_rcu+0x5/0x20 kernel/softirq.c:649
common_interrupt+0x52/0xc0 arch/x86/kernel/irq.c:240
asm_common_interrupt+0x1e/0x40 arch/x86/include/asm/idtentry.h:629
RIP: 0033:0x7f5e972bfd57
Code: 39 d1 73 14 0f 1f 80 00 00 00 00 48 8b 50 f8 48 83 e8 08 48 39 ca 77 f3 48 39 c3 73 3e 48 89 13 48 8b 50 f8 48 89 38 49 8b 0e <48> 8b 3e 48 83 c3 08 48 83 c6 08 eb bc 48 39 d1 72 9e 48 39 d0 73
RSP: 002b:00007fff8a413210 EFLAGS: 00000283
RAX: 00007f5e97108990 RBX: 00007f5e97108338 RCX: ffffffff81d3aa45
RDX: ffffffff81d3aa45 RSI: 00007f5e97108340 RDI: ffffffff81d3aa45
RBP: 00007f5e97107eb8 R08: 00007f5e97108d88 R09: 0000000093c2e8d9
R10: 0000000000000000 R11: 0000000000000000 R12: 00007f5e97107eb0
R13: 00007f5e97108338 R14: 00007f5e97107ea8 R15: 0000000000000019
</TASK>
Allocated by task 13:
kasan_save_stack+0x1e/0x50 mm/kasan/common.c:38
kasan_set_track mm/kasan/common.c:46 [inline]
set_alloc_info mm/kasan/common.c:434 [inline]
__kasan_slab_alloc+0x90/0xc0 mm/kasan/common.c:467
kasan_slab_alloc include/linux/kasan.h:259 [inline]
slab_post_alloc_hook mm/slab.h:519 [inline]
slab_alloc_node mm/slub.c:3234 [inline]
slab_alloc mm/slub.c:3242 [inline]
kmem_cache_alloc+0x202/0x3a0 mm/slub.c:3247
dst_alloc+0x146/0x1f0 net/core/dst.c:92
rt_dst_alloc+0x73/0x430 net/ipv4/route.c:1613
ip_route_input_slow+0x1817/0x3a20 net/ipv4/route.c:2340
ip_route_input_rcu net/ipv4/route.c:2470 [inline]
ip_route_input_noref+0x116/0x2a0 net/ipv4/route.c:2415
ip_rcv_finish_core.constprop.0+0x288/0x1e80 net/ipv4/ip_input.c:354
ip_list_rcv_finish.constprop.0+0x1b2/0x6e0 net/ipv4/ip_input.c:583
ip_sublist_rcv net/ipv4/ip_input.c:609 [inline]
ip_list_rcv+0x34e/0x490 net/ipv4/ip_input.c:644
__netif_receive_skb_list_ptype net/core/dev.c:5508 [inline]
__netif_receive_skb_list_core+0x549/0x8e0 net/core/dev.c:5556
__netif_receive_skb_list net/core/dev.c:5608 [inline]
netif_receive_skb_list_internal+0x75e/0xd80 net/core/dev.c:5699
gro_normal_list net/core/dev.c:5853 [inline]
gro_normal_list net/core/dev.c:5849 [inline]
napi_complete_done+0x1f1/0x880 net/core/dev.c:6590
virtqueue_napi_complete drivers/net/virtio_net.c:339 [inline]
virtnet_poll+0xca2/0x11b0 drivers/net/virtio_net.c:1557
__napi_poll+0xaf/0x440 net/core/dev.c:7023
napi_poll net/core/dev.c:7090 [inline]
net_rx_action+0x801/0xb40 net/core/dev.c:7177
__do_softirq+0x29b/0x9c2 kernel/softirq.c:558
Freed by task 13:
kasan_save_stack+0x1e/0x50 mm/kasan/common.c:38
kasan_set_track+0x21/0x30 mm/kasan/common.c:46
kasan_set_free_info+0x20/0x30 mm/kasan/generic.c:370
____kasan_slab_free mm/kasan/common.c:366 [inline]
____kasan_slab_free mm/kasan/common.c:328 [inline]
__kasan_slab_free+0xff/0x130 mm/kasan/common.c:374
kasan_slab_free include/linux/kasan.h:235 [inline]
slab_free_hook mm/slub.c:1723 [inline]
slab_free_freelist_hook+0x8b/0x1c0 mm/slub.c:1749
slab_free mm/slub.c:3513 [inline]
kmem_cache_free+0xbd/0x5d0 mm/slub.c:3530
dst_destroy+0x2d6/0x3f0 net/core/dst.c:127
rcu_do_batch kernel/rcu/tree.c:2506 [inline]
rcu_core+0x7ab/0x1470 kernel/rcu/tree.c:2741
__do_softirq+0x29b/0x9c2 kernel/softirq.c:558
Last potentially related work creation:
kasan_save_stack+0x1e/0x50 mm/kasan/common.c:38
__kasan_record_aux_stack+0xf5/0x120 mm/kasan/generic.c:348
__call_rcu kernel/rcu/tree.c:2985 [inline]
call_rcu+0xb1/0x740 kernel/rcu/tree.c:3065
dst_release net/core/dst.c:177 [inline]
dst_release+0x79/0xe0 net/core/dst.c:167
tcp_v4_do_rcv+0x612/0x8d0 net/ipv4/tcp_ipv4.c:1712
sk_backlog_rcv include/net/sock.h:1030 [inline]
__release_sock+0x134/0x3b0 net/core/sock.c:2768
release_sock+0x54/0x1b0 net/core/sock.c:3300
tcp_sendmsg+0x36/0x40 net/ipv4/tcp.c:1441
inet_sendmsg+0x99/0xe0 net/ipv4/af_inet.c:819
sock_sendmsg_nosec net/socket.c:704 [inline]
sock_sendmsg+0xcf/0x120 net/socket.c:724
sock_write_iter+0x289/0x3c0 net/socket.c:1057
call_write_iter include/linux/fs.h:2162 [inline]
new_sync_write+0x429/0x660 fs/read_write.c:503
vfs_write+0x7cd/0xae0 fs/read_write.c:590
ksys_write+0x1ee/0x250 fs/read_write.c:643
do_syscall_x64 arch/x86/entry/common.c:50 [inline]
do_syscall_64+0x35/0xb0 arch/x86/entry/common.c:80
entry_SYSCALL_64_after_hwframe+0x44/0xae
The buggy address belongs to the object at ffff88807f1cb700
which belongs to the cache ip_dst_cache of size 176
The buggy address is located 58 bytes inside of
176-byte region [ffff88807f1cb700, ffff88807f1cb7b0)
The buggy address belongs to the page:
page:ffffea0001fc72c0 refcount:1 mapcount:0 mapping:0000000000000000 index:0x0 pfn:0x7f1cb
flags: 0xfff00000000200(slab|node=0|zone=1|lastcpupid=0x7ff)
raw: 00fff00000000200 dead000000000100 dead000000000122 ffff8881413bb780
raw: 0000000000000000 0000000000100010 00000001ffffffff 0000000000000000
page dumped because: kasan: bad access detected
page_owner tracks the page as allocated
page last allocated via order 0, migratetype Unmovable, gfp_mask 0x112a20(GFP_ATOMIC|__GFP_NOWARN|__GFP_NORETRY|__GFP_HARDWALL), pid 5, ts 108466983062, free_ts 108048976062
prep_new_page mm/page_alloc.c:2418 [inline]
get_page_from_freelist+0xa72/0x2f50 mm/page_alloc.c:4149
__alloc_pages+0x1b2/0x500 mm/page_alloc.c:5369
alloc_pages+0x1a7/0x300 mm/mempolicy.c:2191
alloc_slab_page mm/slub.c:1793 [inline]
allocate_slab mm/slub.c:1930 [inline]
new_slab+0x32d/0x4a0 mm/slub.c:1993
___slab_alloc+0x918/0xfe0 mm/slub.c:3022
__slab_alloc.constprop.0+0x4d/0xa0 mm/slub.c:3109
slab_alloc_node mm/slub.c:3200 [inline]
slab_alloc mm/slub.c:3242 [inline]
kmem_cache_alloc+0x35c/0x3a0 mm/slub.c:3247
dst_alloc+0x146/0x1f0 net/core/dst.c:92
rt_dst_alloc+0x73/0x430 net/ipv4/route.c:1613
__mkroute_output net/ipv4/route.c:2564 [inline]
ip_route_output_key_hash_rcu+0x921/0x2d00 net/ipv4/route.c:2791
ip_route_output_key_hash+0x18b/0x300 net/ipv4/route.c:2619
__ip_route_output_key include/net/route.h:126 [inline]
ip_route_output_flow+0x23/0x150 net/ipv4/route.c:2850
ip_route_output_key include/net/route.h:142 [inline]
geneve_get_v4_rt+0x3a6/0x830 drivers/net/geneve.c:809
geneve_xmit_skb drivers/net/geneve.c:899 [inline]
geneve_xmit+0xc4a/0x3540 drivers/net/geneve.c:1082
__netdev_start_xmit include/linux/netdevice.h:4994 [inline]
netdev_start_xmit include/linux/netdevice.h:5008 [inline]
xmit_one net/core/dev.c:3590 [inline]
dev_hard_start_xmit+0x1eb/0x920 net/core/dev.c:3606
__dev_queue_xmit+0x299a/0x3650 net/core/dev.c:4229
page last free stack trace:
reset_page_owner include/linux/page_owner.h:24 [inline]
free_pages_prepare mm/page_alloc.c:1338 [inline]
free_pcp_prepare+0x374/0x870 mm/page_alloc.c:1389
free_unref_page_prepare mm/page_alloc.c:3309 [inline]
free_unref_page+0x19/0x690 mm/page_alloc.c:3388
qlink_free mm/kasan/quarantine.c:146 [inline]
qlist_free_all+0x5a/0xc0 mm/kasan/quarantine.c:165
kasan_quarantine_reduce+0x180/0x200 mm/kasan/quarantine.c:272
__kasan_slab_alloc+0xa2/0xc0 mm/kasan/common.c:444
kasan_slab_alloc include/linux/kasan.h:259 [inline]
slab_post_alloc_hook mm/slab.h:519 [inline]
slab_alloc_node mm/slub.c:3234 [inline]
kmem_cache_alloc_node+0x255/0x3f0 mm/slub.c:3270
__alloc_skb+0x215/0x340 net/core/skbuff.c:414
alloc_skb include/linux/skbuff.h:1126 [inline]
alloc_skb_with_frags+0x93/0x620 net/core/skbuff.c:6078
sock_alloc_send_pskb+0x783/0x910 net/core/sock.c:2575
mld_newpack+0x1df/0x770 net/ipv6/mcast.c:1754
add_grhead+0x265/0x330 net/ipv6/mcast.c:1857
add_grec+0x1053/0x14e0 net/ipv6/mcast.c:1995
mld_send_initial_cr.part.0+0xf6/0x230 net/ipv6/mcast.c:2242
mld_send_initial_cr net/ipv6/mcast.c:1232 [inline]
mld_dad_work+0x1d3/0x690 net/ipv6/mcast.c:2268
process_one_work+0x9b2/0x1690 kernel/workqueue.c:2298
worker_thread+0x658/0x11f0 kernel/workqueue.c:2445
Memory state around the buggy address:
ffff88807f1cb600: fa fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb
ffff88807f1cb680: fb fb fb fb fb fb fc fc fc fc fc fc fc fc fc fc
>ffff88807f1cb700: fa fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb
^
ffff88807f1cb780: fb fb fb fb fb fb fc fc fc fc fc fc fc fc fc fc
ffff88807f1cb800: fa fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb
Fixes: 41063e9dd119 ("ipv4: Early TCP socket demux.")
Signed-off-by: Eric Dumazet <edumazet@google.com>
Link: https://lore.kernel.org/r/20211220143330.680945-1-eric.dumazet@gmail.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
2021-12-20 14:33:30 +00:00
|
|
|
dst = rcu_dereference(sk->sk_rx_dst);
|
2013-10-07 16:01:39 +00:00
|
|
|
|
|
|
|
if (dst)
|
|
|
|
dst = dst_check(dst, 0);
|
2015-08-01 10:14:33 +00:00
|
|
|
if (dst) {
|
2017-09-28 13:51:37 +00:00
|
|
|
u32 itag = 0;
|
|
|
|
|
2017-06-17 17:42:25 +00:00
|
|
|
/* set noref for now.
|
|
|
|
* any place which wants to hold dst has to call
|
|
|
|
* dst_hold_safe()
|
|
|
|
*/
|
|
|
|
skb_dst_set_noref(skb, dst);
|
2017-09-28 13:51:37 +00:00
|
|
|
|
|
|
|
/* for unconnected multicast sockets we need to validate
|
|
|
|
* the source on each packet
|
|
|
|
*/
|
|
|
|
if (!inet_sk(sk)->inet_daddr && in_dev)
|
|
|
|
return ip_mc_validate_source(skb, iph->daddr,
|
udp: mask TOS bits in udp_v4_early_demux()
udp_v4_early_demux() is the only function that calls
ip_mc_validate_source() with a TOS that hasn't been masked with
IPTOS_RT_MASK.
This results in different behaviours for incoming multicast UDPv4
packets, depending on if ip_mc_validate_source() is called from the
early-demux path (udp_v4_early_demux) or from the regular input path
(ip_route_input_noref).
ECN would normally not be used with UDP multicast packets, so the
practical consequences should be limited on that side. However,
IPTOS_RT_MASK is used to also masks the TOS' high order bits, to align
with the non-early-demux path behaviour.
Reproducer:
Setup two netns, connected with veth:
$ ip netns add ns0
$ ip netns add ns1
$ ip -netns ns0 link set dev lo up
$ ip -netns ns1 link set dev lo up
$ ip link add name veth01 netns ns0 type veth peer name veth10 netns ns1
$ ip -netns ns0 link set dev veth01 up
$ ip -netns ns1 link set dev veth10 up
$ ip -netns ns0 address add 192.0.2.10 peer 192.0.2.11/32 dev veth01
$ ip -netns ns1 address add 192.0.2.11 peer 192.0.2.10/32 dev veth10
In ns0, add route to multicast address 224.0.2.0/24 using source
address 198.51.100.10:
$ ip -netns ns0 address add 198.51.100.10/32 dev lo
$ ip -netns ns0 route add 224.0.2.0/24 dev veth01 src 198.51.100.10
In ns1, define route to 198.51.100.10, only for packets with TOS 4:
$ ip -netns ns1 route add 198.51.100.10/32 tos 4 dev veth10
Also activate rp_filter in ns1, so that incoming packets not matching
the above route get dropped:
$ ip netns exec ns1 sysctl -wq net.ipv4.conf.veth10.rp_filter=1
Now try to receive packets on 224.0.2.11:
$ ip netns exec ns1 socat UDP-RECVFROM:1111,ip-add-membership=224.0.2.11:veth10,ignoreeof -
In ns0, send packet to 224.0.2.11 with TOS 4 and ECT(0) (that is,
tos 6 for socat):
$ echo test0 | ip netns exec ns0 socat - UDP-DATAGRAM:224.0.2.11:1111,bind=:1111,tos=6
The "test0" message is properly received by socat in ns1, because
early-demux has no cached dst to use, so source address validation
is done by ip_route_input_mc(), which receives a TOS that has the
ECN bits masked.
Now send another packet to 224.0.2.11, still with TOS 4 and ECT(0):
$ echo test1 | ip netns exec ns0 socat - UDP-DATAGRAM:224.0.2.11:1111,bind=:1111,tos=6
The "test1" message isn't received by socat in ns1, because, now,
early-demux has a cached dst to use and calls ip_mc_validate_source()
immediately, without masking the ECN bits.
Fixes: bc044e8db796 ("udp: perform source validation for mcast early demux")
Signed-off-by: Guillaume Nault <gnault@redhat.com>
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
2021-01-16 10:44:22 +00:00
|
|
|
iph->saddr,
|
|
|
|
iph->tos & IPTOS_RT_MASK,
|
2017-09-28 13:51:37 +00:00
|
|
|
skb->dev, in_dev, &itag);
|
2015-08-01 10:14:33 +00:00
|
|
|
}
|
2017-09-28 13:51:36 +00:00
|
|
|
return 0;
|
2013-10-07 16:01:39 +00:00
|
|
|
}
|
|
|
|
|
2008-03-07 00:22:02 +00:00
|
|
|
int udp_rcv(struct sk_buff *skb)
|
|
|
|
{
|
2022-11-14 21:57:56 +00:00
|
|
|
return __udp4_lib_rcv(skb, dev_net(skb->dev)->ipv4.udp_table, IPPROTO_UDP);
|
2008-03-07 00:22:02 +00:00
|
|
|
}
|
|
|
|
|
2008-06-15 00:04:49 +00:00
|
|
|
void udp_destroy_sock(struct sock *sk)
|
2008-03-07 00:22:02 +00:00
|
|
|
{
|
2013-03-19 06:11:12 +00:00
|
|
|
struct udp_sock *up = udp_sk(sk);
|
2010-05-26 19:20:18 +00:00
|
|
|
bool slow = lock_sock_fast(sk);
|
2021-06-09 09:49:01 +00:00
|
|
|
|
|
|
|
/* protects from races with udp_abort() */
|
|
|
|
sock_set_flag(sk, SOCK_DEAD);
|
2008-03-07 00:22:02 +00:00
|
|
|
udp_flush_pending_frames(sk);
|
2010-05-26 19:20:18 +00:00
|
|
|
unlock_sock_fast(sk, slow);
|
2018-11-07 11:38:28 +00:00
|
|
|
if (static_branch_unlikely(&udp_encap_needed_key)) {
|
|
|
|
if (up->encap_type) {
|
|
|
|
void (*encap_destroy)(struct sock *sk);
|
|
|
|
encap_destroy = READ_ONCE(up->encap_destroy);
|
|
|
|
if (encap_destroy)
|
|
|
|
encap_destroy(sk);
|
|
|
|
}
|
2023-09-12 09:17:27 +00:00
|
|
|
if (udp_test_bit(ENCAP_ENABLED, sk))
|
2018-11-15 01:34:50 +00:00
|
|
|
static_branch_dec(&udp_encap_needed_key);
|
2013-03-19 06:11:12 +00:00
|
|
|
}
|
2008-03-07 00:22:02 +00:00
|
|
|
}
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
* Socket option code for UDP
|
|
|
|
*/
|
2006-11-27 17:29:59 +00:00
|
|
|
int udp_lib_setsockopt(struct sock *sk, int level, int optname,
|
2020-07-23 06:09:04 +00:00
|
|
|
sockptr_t optval, unsigned int optlen,
|
2006-11-27 17:29:59 +00:00
|
|
|
int (*push_pending_frames)(struct sock *))
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
struct udp_sock *up = udp_sk(sk);
|
2014-05-23 15:47:32 +00:00
|
|
|
int val, valbool;
|
2005-04-16 22:20:36 +00:00
|
|
|
int err = 0;
|
2007-12-03 11:34:16 +00:00
|
|
|
int is_udplite = IS_UDPLITE(sk);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2022-10-20 17:48:52 +00:00
|
|
|
if (level == SOL_SOCKET) {
|
|
|
|
err = sk_setsockopt(sk, level, optname, optval, optlen);
|
|
|
|
|
|
|
|
if (optname == SO_RCVBUF || optname == SO_RCVBUFFORCE) {
|
|
|
|
sockopt_lock_sock(sk);
|
|
|
|
/* paired with READ_ONCE in udp_rmem_release() */
|
|
|
|
WRITE_ONCE(up->forward_threshold, sk->sk_rcvbuf >> 2);
|
|
|
|
sockopt_release_sock(sk);
|
|
|
|
}
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
2009-07-17 00:26:32 +00:00
|
|
|
if (optlen < sizeof(int))
|
2005-04-16 22:20:36 +00:00
|
|
|
return -EINVAL;
|
|
|
|
|
2020-07-23 06:09:04 +00:00
|
|
|
if (copy_from_sockptr(&val, optval, sizeof(val)))
|
2005-04-16 22:20:36 +00:00
|
|
|
return -EFAULT;
|
|
|
|
|
2014-05-23 15:47:32 +00:00
|
|
|
valbool = val ? 1 : 0;
|
|
|
|
|
2007-03-09 04:41:55 +00:00
|
|
|
switch (optname) {
|
2005-04-16 22:20:36 +00:00
|
|
|
case UDP_CORK:
|
|
|
|
if (val != 0) {
|
2023-09-12 09:17:21 +00:00
|
|
|
udp_set_bit(CORK, sk);
|
2005-04-16 22:20:36 +00:00
|
|
|
} else {
|
2023-09-12 09:17:21 +00:00
|
|
|
udp_clear_bit(CORK, sk);
|
2005-04-16 22:20:36 +00:00
|
|
|
lock_sock(sk);
|
2014-11-12 05:59:20 +00:00
|
|
|
push_pending_frames(sk);
|
2005-04-16 22:20:36 +00:00
|
|
|
release_sock(sk);
|
|
|
|
}
|
|
|
|
break;
|
2007-02-09 14:24:47 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
case UDP_ENCAP:
|
|
|
|
switch (val) {
|
|
|
|
case 0:
|
2019-10-03 21:21:57 +00:00
|
|
|
#ifdef CONFIG_XFRM
|
2005-04-16 22:20:36 +00:00
|
|
|
case UDP_ENCAP_ESPINUDP:
|
|
|
|
case UDP_ENCAP_ESPINUDP_NON_IKE:
|
2020-04-27 15:59:34 +00:00
|
|
|
#if IS_ENABLED(CONFIG_IPV6)
|
|
|
|
if (sk->sk_family == AF_INET6)
|
2023-09-12 09:17:25 +00:00
|
|
|
WRITE_ONCE(up->encap_rcv,
|
|
|
|
ipv6_stub->xfrm6_udp_encap_rcv);
|
2020-04-27 15:59:34 +00:00
|
|
|
else
|
|
|
|
#endif
|
2023-09-12 09:17:25 +00:00
|
|
|
WRITE_ONCE(up->encap_rcv,
|
|
|
|
xfrm4_udp_encap_rcv);
|
2019-10-03 21:21:57 +00:00
|
|
|
#endif
|
2020-03-12 22:50:22 +00:00
|
|
|
fallthrough;
|
2007-06-27 22:37:46 +00:00
|
|
|
case UDP_ENCAP_L2TPINUDP:
|
2023-09-12 09:17:28 +00:00
|
|
|
WRITE_ONCE(up->encap_type, val);
|
2023-09-12 09:17:27 +00:00
|
|
|
udp_tunnel_encap_enable(sk);
|
2005-04-16 22:20:36 +00:00
|
|
|
break;
|
|
|
|
default:
|
|
|
|
err = -ENOPROTOOPT;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
|
2014-05-23 15:47:32 +00:00
|
|
|
case UDP_NO_CHECK6_TX:
|
2023-09-12 09:17:22 +00:00
|
|
|
udp_set_no_check6_tx(sk, valbool);
|
2014-05-23 15:47:32 +00:00
|
|
|
break;
|
|
|
|
|
|
|
|
case UDP_NO_CHECK6_RX:
|
2023-09-12 09:17:23 +00:00
|
|
|
udp_set_no_check6_rx(sk, valbool);
|
2014-05-23 15:47:32 +00:00
|
|
|
break;
|
|
|
|
|
udp: generate gso with UDP_SEGMENT
Support generic segmentation offload for udp datagrams. Callers can
concatenate and send at once the payload of multiple datagrams with
the same destination.
To set segment size, the caller sets socket option UDP_SEGMENT to the
length of each discrete payload. This value must be smaller than or
equal to the relevant MTU.
A follow-up patch adds cmsg UDP_SEGMENT to specify segment size on a
per send call basis.
Total byte length may then exceed MTU. If not an exact multiple of
segment size, the last segment will be shorter.
The implementation adds a gso_size field to the udp socket, ip(v6)
cmsg cookie and inet_cork structure to be able to set the value at
setsockopt or cmsg time and to work with both lockless and corked
paths.
Initial benchmark numbers show UDP GSO about as expensive as TCP GSO.
tcp tso
3197 MB/s 54232 msg/s 54232 calls/s
6,457,754,262 cycles
tcp gso
1765 MB/s 29939 msg/s 29939 calls/s
11,203,021,806 cycles
tcp without tso/gso *
739 MB/s 12548 msg/s 12548 calls/s
11,205,483,630 cycles
udp
876 MB/s 14873 msg/s 624666 calls/s
11,205,777,429 cycles
udp gso
2139 MB/s 36282 msg/s 36282 calls/s
11,204,374,561 cycles
[*] after reverting commit 0a6b2a1dc2a2
("tcp: switch to GSO being always on")
Measured total system cycles ('-a') for one core while pinning both
the network receive path and benchmark process to that core:
perf stat -a -C 12 -e cycles \
./udpgso_bench_tx -C 12 -4 -D "$DST" -l 4
Note the reduction in calls/s with GSO. Bytes per syscall drops
increases from 1470 to 61818.
Signed-off-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-04-26 17:42:17 +00:00
|
|
|
case UDP_SEGMENT:
|
|
|
|
if (val < 0 || val > USHRT_MAX)
|
|
|
|
return -EINVAL;
|
2021-06-30 16:42:44 +00:00
|
|
|
WRITE_ONCE(up->gso_size, val);
|
udp: generate gso with UDP_SEGMENT
Support generic segmentation offload for udp datagrams. Callers can
concatenate and send at once the payload of multiple datagrams with
the same destination.
To set segment size, the caller sets socket option UDP_SEGMENT to the
length of each discrete payload. This value must be smaller than or
equal to the relevant MTU.
A follow-up patch adds cmsg UDP_SEGMENT to specify segment size on a
per send call basis.
Total byte length may then exceed MTU. If not an exact multiple of
segment size, the last segment will be shorter.
The implementation adds a gso_size field to the udp socket, ip(v6)
cmsg cookie and inet_cork structure to be able to set the value at
setsockopt or cmsg time and to work with both lockless and corked
paths.
Initial benchmark numbers show UDP GSO about as expensive as TCP GSO.
tcp tso
3197 MB/s 54232 msg/s 54232 calls/s
6,457,754,262 cycles
tcp gso
1765 MB/s 29939 msg/s 29939 calls/s
11,203,021,806 cycles
tcp without tso/gso *
739 MB/s 12548 msg/s 12548 calls/s
11,205,483,630 cycles
udp
876 MB/s 14873 msg/s 624666 calls/s
11,205,777,429 cycles
udp gso
2139 MB/s 36282 msg/s 36282 calls/s
11,204,374,561 cycles
[*] after reverting commit 0a6b2a1dc2a2
("tcp: switch to GSO being always on")
Measured total system cycles ('-a') for one core while pinning both
the network receive path and benchmark process to that core:
perf stat -a -C 12 -e cycles \
./udpgso_bench_tx -C 12 -4 -D "$DST" -l 4
Note the reduction in calls/s with GSO. Bytes per syscall drops
increases from 1470 to 61818.
Signed-off-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-04-26 17:42:17 +00:00
|
|
|
break;
|
|
|
|
|
2018-11-07 11:38:29 +00:00
|
|
|
case UDP_GRO:
|
2021-03-30 10:28:52 +00:00
|
|
|
|
|
|
|
/* when enabling GRO, accept the related GSO packet type */
|
2018-11-07 11:38:29 +00:00
|
|
|
if (valbool)
|
2023-09-12 09:17:27 +00:00
|
|
|
udp_tunnel_encap_enable(sk);
|
2023-09-12 09:17:24 +00:00
|
|
|
udp_assign_bit(GRO_ENABLED, sk, valbool);
|
2023-09-12 09:17:26 +00:00
|
|
|
udp_assign_bit(ACCEPT_L4, sk, valbool);
|
2018-11-07 11:38:29 +00:00
|
|
|
break;
|
|
|
|
|
2006-11-27 19:10:57 +00:00
|
|
|
/*
|
|
|
|
* UDP-Lite's partial checksum coverage (RFC 3828).
|
|
|
|
*/
|
|
|
|
/* The sender sets actual checksum coverage length via this option.
|
|
|
|
* The case coverage > packet length is handled by send module. */
|
|
|
|
case UDPLITE_SEND_CSCOV:
|
2007-12-03 11:34:16 +00:00
|
|
|
if (!is_udplite) /* Disable the option on UDP sockets */
|
2006-11-27 19:10:57 +00:00
|
|
|
return -ENOPROTOOPT;
|
|
|
|
if (val != 0 && val < 8) /* Illegal coverage: use default (8) */
|
|
|
|
val = 8;
|
2010-05-24 21:33:03 +00:00
|
|
|
else if (val > USHRT_MAX)
|
|
|
|
val = USHRT_MAX;
|
2023-09-12 09:17:30 +00:00
|
|
|
WRITE_ONCE(up->pcslen, val);
|
|
|
|
udp_set_bit(UDPLITE_SEND_CC, sk);
|
2006-11-27 19:10:57 +00:00
|
|
|
break;
|
|
|
|
|
2007-02-09 14:24:47 +00:00
|
|
|
/* The receiver specifies a minimum checksum coverage value. To make
|
|
|
|
* sense, this should be set to at least 8 (as done below). If zero is
|
2006-11-27 19:10:57 +00:00
|
|
|
* used, this again means full checksum coverage. */
|
|
|
|
case UDPLITE_RECV_CSCOV:
|
2007-12-03 11:34:16 +00:00
|
|
|
if (!is_udplite) /* Disable the option on UDP sockets */
|
2006-11-27 19:10:57 +00:00
|
|
|
return -ENOPROTOOPT;
|
|
|
|
if (val != 0 && val < 8) /* Avoid silly minimal values. */
|
|
|
|
val = 8;
|
2010-05-24 21:33:03 +00:00
|
|
|
else if (val > USHRT_MAX)
|
|
|
|
val = USHRT_MAX;
|
2023-09-12 09:17:30 +00:00
|
|
|
WRITE_ONCE(up->pcrlen, val);
|
|
|
|
udp_set_bit(UDPLITE_RECV_CC, sk);
|
2006-11-27 19:10:57 +00:00
|
|
|
break;
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
default:
|
|
|
|
err = -ENOPROTOOPT;
|
|
|
|
break;
|
2007-03-09 04:41:55 +00:00
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
return err;
|
|
|
|
}
|
2009-07-17 00:26:32 +00:00
|
|
|
EXPORT_SYMBOL(udp_lib_setsockopt);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2020-07-23 06:09:07 +00:00
|
|
|
int udp_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval,
|
|
|
|
unsigned int optlen)
|
2008-03-07 00:22:02 +00:00
|
|
|
{
|
2022-10-20 17:48:52 +00:00
|
|
|
if (level == SOL_UDP || level == SOL_UDPLITE || level == SOL_SOCKET)
|
2020-07-23 06:09:04 +00:00
|
|
|
return udp_lib_setsockopt(sk, level, optname,
|
2020-07-23 06:09:07 +00:00
|
|
|
optval, optlen,
|
2008-03-07 00:22:02 +00:00
|
|
|
udp_push_pending_frames);
|
|
|
|
return ip_setsockopt(sk, level, optname, optval, optlen);
|
|
|
|
}
|
|
|
|
|
2006-11-27 17:29:59 +00:00
|
|
|
int udp_lib_getsockopt(struct sock *sk, int level, int optname,
|
|
|
|
char __user *optval, int __user *optlen)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
struct udp_sock *up = udp_sk(sk);
|
|
|
|
int val, len;
|
|
|
|
|
2009-07-17 00:26:32 +00:00
|
|
|
if (get_user(len, optlen))
|
2005-04-16 22:20:36 +00:00
|
|
|
return -EFAULT;
|
|
|
|
|
|
|
|
len = min_t(unsigned int, len, sizeof(int));
|
2007-02-09 14:24:47 +00:00
|
|
|
|
2007-03-09 04:41:55 +00:00
|
|
|
if (len < 0)
|
2005-04-16 22:20:36 +00:00
|
|
|
return -EINVAL;
|
|
|
|
|
2007-03-09 04:41:55 +00:00
|
|
|
switch (optname) {
|
2005-04-16 22:20:36 +00:00
|
|
|
case UDP_CORK:
|
2023-09-12 09:17:21 +00:00
|
|
|
val = udp_test_bit(CORK, sk);
|
2005-04-16 22:20:36 +00:00
|
|
|
break;
|
|
|
|
|
|
|
|
case UDP_ENCAP:
|
2023-09-12 09:17:28 +00:00
|
|
|
val = READ_ONCE(up->encap_type);
|
2005-04-16 22:20:36 +00:00
|
|
|
break;
|
|
|
|
|
2014-05-23 15:47:32 +00:00
|
|
|
case UDP_NO_CHECK6_TX:
|
2023-09-12 09:17:22 +00:00
|
|
|
val = udp_get_no_check6_tx(sk);
|
2014-05-23 15:47:32 +00:00
|
|
|
break;
|
|
|
|
|
|
|
|
case UDP_NO_CHECK6_RX:
|
2023-09-12 09:17:23 +00:00
|
|
|
val = udp_get_no_check6_rx(sk);
|
2014-05-23 15:47:32 +00:00
|
|
|
break;
|
|
|
|
|
udp: generate gso with UDP_SEGMENT
Support generic segmentation offload for udp datagrams. Callers can
concatenate and send at once the payload of multiple datagrams with
the same destination.
To set segment size, the caller sets socket option UDP_SEGMENT to the
length of each discrete payload. This value must be smaller than or
equal to the relevant MTU.
A follow-up patch adds cmsg UDP_SEGMENT to specify segment size on a
per send call basis.
Total byte length may then exceed MTU. If not an exact multiple of
segment size, the last segment will be shorter.
The implementation adds a gso_size field to the udp socket, ip(v6)
cmsg cookie and inet_cork structure to be able to set the value at
setsockopt or cmsg time and to work with both lockless and corked
paths.
Initial benchmark numbers show UDP GSO about as expensive as TCP GSO.
tcp tso
3197 MB/s 54232 msg/s 54232 calls/s
6,457,754,262 cycles
tcp gso
1765 MB/s 29939 msg/s 29939 calls/s
11,203,021,806 cycles
tcp without tso/gso *
739 MB/s 12548 msg/s 12548 calls/s
11,205,483,630 cycles
udp
876 MB/s 14873 msg/s 624666 calls/s
11,205,777,429 cycles
udp gso
2139 MB/s 36282 msg/s 36282 calls/s
11,204,374,561 cycles
[*] after reverting commit 0a6b2a1dc2a2
("tcp: switch to GSO being always on")
Measured total system cycles ('-a') for one core while pinning both
the network receive path and benchmark process to that core:
perf stat -a -C 12 -e cycles \
./udpgso_bench_tx -C 12 -4 -D "$DST" -l 4
Note the reduction in calls/s with GSO. Bytes per syscall drops
increases from 1470 to 61818.
Signed-off-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-04-26 17:42:17 +00:00
|
|
|
case UDP_SEGMENT:
|
2021-06-30 16:42:44 +00:00
|
|
|
val = READ_ONCE(up->gso_size);
|
udp: generate gso with UDP_SEGMENT
Support generic segmentation offload for udp datagrams. Callers can
concatenate and send at once the payload of multiple datagrams with
the same destination.
To set segment size, the caller sets socket option UDP_SEGMENT to the
length of each discrete payload. This value must be smaller than or
equal to the relevant MTU.
A follow-up patch adds cmsg UDP_SEGMENT to specify segment size on a
per send call basis.
Total byte length may then exceed MTU. If not an exact multiple of
segment size, the last segment will be shorter.
The implementation adds a gso_size field to the udp socket, ip(v6)
cmsg cookie and inet_cork structure to be able to set the value at
setsockopt or cmsg time and to work with both lockless and corked
paths.
Initial benchmark numbers show UDP GSO about as expensive as TCP GSO.
tcp tso
3197 MB/s 54232 msg/s 54232 calls/s
6,457,754,262 cycles
tcp gso
1765 MB/s 29939 msg/s 29939 calls/s
11,203,021,806 cycles
tcp without tso/gso *
739 MB/s 12548 msg/s 12548 calls/s
11,205,483,630 cycles
udp
876 MB/s 14873 msg/s 624666 calls/s
11,205,777,429 cycles
udp gso
2139 MB/s 36282 msg/s 36282 calls/s
11,204,374,561 cycles
[*] after reverting commit 0a6b2a1dc2a2
("tcp: switch to GSO being always on")
Measured total system cycles ('-a') for one core while pinning both
the network receive path and benchmark process to that core:
perf stat -a -C 12 -e cycles \
./udpgso_bench_tx -C 12 -4 -D "$DST" -l 4
Note the reduction in calls/s with GSO. Bytes per syscall drops
increases from 1470 to 61818.
Signed-off-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-04-26 17:42:17 +00:00
|
|
|
break;
|
|
|
|
|
2021-04-01 06:59:17 +00:00
|
|
|
case UDP_GRO:
|
2023-09-12 09:17:24 +00:00
|
|
|
val = udp_test_bit(GRO_ENABLED, sk);
|
2021-04-01 06:59:17 +00:00
|
|
|
break;
|
|
|
|
|
2006-11-27 19:10:57 +00:00
|
|
|
/* The following two cannot be changed on UDP sockets, the return is
|
|
|
|
* always 0 (which corresponds to the full checksum coverage of UDP). */
|
|
|
|
case UDPLITE_SEND_CSCOV:
|
2023-09-12 09:17:30 +00:00
|
|
|
val = READ_ONCE(up->pcslen);
|
2006-11-27 19:10:57 +00:00
|
|
|
break;
|
|
|
|
|
|
|
|
case UDPLITE_RECV_CSCOV:
|
2023-09-12 09:17:30 +00:00
|
|
|
val = READ_ONCE(up->pcrlen);
|
2006-11-27 19:10:57 +00:00
|
|
|
break;
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
default:
|
|
|
|
return -ENOPROTOOPT;
|
2007-03-09 04:41:55 +00:00
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2007-03-09 04:41:55 +00:00
|
|
|
if (put_user(len, optlen))
|
2007-02-09 14:24:47 +00:00
|
|
|
return -EFAULT;
|
2009-07-17 00:26:32 +00:00
|
|
|
if (copy_to_user(optval, &val, len))
|
2005-04-16 22:20:36 +00:00
|
|
|
return -EFAULT;
|
2007-02-09 14:24:47 +00:00
|
|
|
return 0;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2009-07-17 00:26:32 +00:00
|
|
|
EXPORT_SYMBOL(udp_lib_getsockopt);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2008-03-07 00:22:02 +00:00
|
|
|
int udp_getsockopt(struct sock *sk, int level, int optname,
|
|
|
|
char __user *optval, int __user *optlen)
|
|
|
|
{
|
|
|
|
if (level == SOL_UDP || level == SOL_UDPLITE)
|
|
|
|
return udp_lib_getsockopt(sk, level, optname, optval, optlen);
|
|
|
|
return ip_getsockopt(sk, level, optname, optval, optlen);
|
|
|
|
}
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/**
|
|
|
|
* udp_poll - wait for a UDP event.
|
2020-07-12 23:15:02 +00:00
|
|
|
* @file: - file struct
|
|
|
|
* @sock: - socket
|
|
|
|
* @wait: - poll table
|
2005-04-16 22:20:36 +00:00
|
|
|
*
|
2007-02-09 14:24:47 +00:00
|
|
|
* This is same as datagram poll, except for the special case of
|
2005-04-16 22:20:36 +00:00
|
|
|
* blocking sockets. If application is using a blocking fd
|
|
|
|
* and a packet with checksum error is in the queue;
|
|
|
|
* then it could get return from select indicating data available
|
|
|
|
* but then block when reading it. Add special case code
|
|
|
|
* to work around these arguably broken applications.
|
|
|
|
*/
|
2018-06-28 16:43:44 +00:00
|
|
|
__poll_t udp_poll(struct file *file, struct socket *sock, poll_table *wait)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2018-06-28 16:43:44 +00:00
|
|
|
__poll_t mask = datagram_poll(file, sock, wait);
|
2005-04-16 22:20:36 +00:00
|
|
|
struct sock *sk = sock->sk;
|
2006-11-27 19:10:57 +00:00
|
|
|
|
2019-10-24 05:44:50 +00:00
|
|
|
if (!skb_queue_empty_lockless(&udp_sk(sk)->reader_queue))
|
2018-02-11 22:34:03 +00:00
|
|
|
mask |= EPOLLIN | EPOLLRDNORM;
|
2017-05-16 09:20:14 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/* Check for false positives due to checksum errors */
|
2018-06-28 16:43:44 +00:00
|
|
|
if ((mask & EPOLLRDNORM) && !(file->f_flags & O_NONBLOCK) &&
|
2016-08-23 20:59:33 +00:00
|
|
|
!(sk->sk_shutdown & RCV_SHUTDOWN) && first_packet_length(sk) == -1)
|
2018-02-11 22:34:03 +00:00
|
|
|
mask &= ~(EPOLLIN | EPOLLRDNORM);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2021-10-08 20:33:05 +00:00
|
|
|
/* psock ingress_msg queue should not contain any bad checksum frames */
|
|
|
|
if (sk_is_readable(sk))
|
|
|
|
mask |= EPOLLIN | EPOLLRDNORM;
|
2005-04-16 22:20:36 +00:00
|
|
|
return mask;
|
2007-02-09 14:24:47 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2018-06-28 16:43:44 +00:00
|
|
|
EXPORT_SYMBOL(udp_poll);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2016-08-24 04:06:33 +00:00
|
|
|
int udp_abort(struct sock *sk, int err)
|
|
|
|
{
|
bpf: Add bpf_sock_destroy kfunc
The socket destroy kfunc is used to forcefully terminate sockets from
certain BPF contexts. We plan to use the capability in Cilium
load-balancing to terminate client sockets that continue to connect to
deleted backends. The other use case is on-the-fly policy enforcement
where existing socket connections prevented by policies need to be
forcefully terminated. The kfunc also allows terminating sockets that may
or may not be actively sending traffic.
The kfunc can currently be called only from BPF TCP and UDP iterators
where users can filter, and terminate selected sockets. More
specifically, it can only be called from BPF contexts that ensure
socket locking in order to allow synchronous execution of protocol
specific `diag_destroy` handlers. The previous commit that batches UDP
sockets during iteration facilitated a synchronous invocation of the UDP
destroy callback from BPF context by skipping socket locks in
`udp_abort`. TCP iterator already supported batching of sockets being
iterated. To that end, `tracing_iter_filter` callback filter is added so
that verifier can restrict the kfunc to programs with `BPF_TRACE_ITER`
attach type, and reject other programs.
The kfunc takes `sock_common` type argument, even though it expects, and
casts them to a `sock` pointer. This enables the verifier to allow the
sock_destroy kfunc to be called for TCP with `sock_common` and UDP with
`sock` structs. Furthermore, as `sock_common` only has a subset of
certain fields of `sock`, casting pointer to the latter type might not
always be safe for certain sockets like request sockets, but these have a
special handling in the diag_destroy handlers.
Additionally, the kfunc is defined with `KF_TRUSTED_ARGS` flag to avoid the
cases where a `PTR_TO_BTF_ID` sk is obtained by following another pointer.
eg. getting a sk pointer (may be even NULL) by following another sk
pointer. The pointer socket argument passed in TCP and UDP iterators is
tagged as `PTR_TRUSTED` in {tcp,udp}_reg_info. The TRUSTED arg changes
are contributed by Martin KaFai Lau <martin.lau@kernel.org>.
Signed-off-by: Aditi Ghag <aditi.ghag@isovalent.com>
Link: https://lore.kernel.org/r/20230519225157.760788-8-aditi.ghag@isovalent.com
Signed-off-by: Martin KaFai Lau <martin.lau@kernel.org>
2023-05-19 22:51:55 +00:00
|
|
|
if (!has_current_bpf_ctx())
|
|
|
|
lock_sock(sk);
|
2016-08-24 04:06:33 +00:00
|
|
|
|
2021-06-09 09:49:01 +00:00
|
|
|
/* udp{v6}_destroy_sock() sets it under the sk lock, avoid racing
|
|
|
|
* with close()
|
|
|
|
*/
|
|
|
|
if (sock_flag(sk, SOCK_DEAD))
|
|
|
|
goto out;
|
|
|
|
|
2016-08-24 04:06:33 +00:00
|
|
|
sk->sk_err = err;
|
2021-06-27 22:48:21 +00:00
|
|
|
sk_error_report(sk);
|
2016-10-20 16:39:40 +00:00
|
|
|
__udp_disconnect(sk, 0);
|
2016-08-24 04:06:33 +00:00
|
|
|
|
2021-06-09 09:49:01 +00:00
|
|
|
out:
|
bpf: Add bpf_sock_destroy kfunc
The socket destroy kfunc is used to forcefully terminate sockets from
certain BPF contexts. We plan to use the capability in Cilium
load-balancing to terminate client sockets that continue to connect to
deleted backends. The other use case is on-the-fly policy enforcement
where existing socket connections prevented by policies need to be
forcefully terminated. The kfunc also allows terminating sockets that may
or may not be actively sending traffic.
The kfunc can currently be called only from BPF TCP and UDP iterators
where users can filter, and terminate selected sockets. More
specifically, it can only be called from BPF contexts that ensure
socket locking in order to allow synchronous execution of protocol
specific `diag_destroy` handlers. The previous commit that batches UDP
sockets during iteration facilitated a synchronous invocation of the UDP
destroy callback from BPF context by skipping socket locks in
`udp_abort`. TCP iterator already supported batching of sockets being
iterated. To that end, `tracing_iter_filter` callback filter is added so
that verifier can restrict the kfunc to programs with `BPF_TRACE_ITER`
attach type, and reject other programs.
The kfunc takes `sock_common` type argument, even though it expects, and
casts them to a `sock` pointer. This enables the verifier to allow the
sock_destroy kfunc to be called for TCP with `sock_common` and UDP with
`sock` structs. Furthermore, as `sock_common` only has a subset of
certain fields of `sock`, casting pointer to the latter type might not
always be safe for certain sockets like request sockets, but these have a
special handling in the diag_destroy handlers.
Additionally, the kfunc is defined with `KF_TRUSTED_ARGS` flag to avoid the
cases where a `PTR_TO_BTF_ID` sk is obtained by following another pointer.
eg. getting a sk pointer (may be even NULL) by following another sk
pointer. The pointer socket argument passed in TCP and UDP iterators is
tagged as `PTR_TRUSTED` in {tcp,udp}_reg_info. The TRUSTED arg changes
are contributed by Martin KaFai Lau <martin.lau@kernel.org>.
Signed-off-by: Aditi Ghag <aditi.ghag@isovalent.com>
Link: https://lore.kernel.org/r/20230519225157.760788-8-aditi.ghag@isovalent.com
Signed-off-by: Martin KaFai Lau <martin.lau@kernel.org>
2023-05-19 22:51:55 +00:00
|
|
|
if (!has_current_bpf_ctx())
|
|
|
|
release_sock(sk);
|
2016-08-24 04:06:33 +00:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(udp_abort);
|
|
|
|
|
2008-03-07 00:22:02 +00:00
|
|
|
struct proto udp_prot = {
|
2018-03-14 04:57:16 +00:00
|
|
|
.name = "UDP",
|
|
|
|
.owner = THIS_MODULE,
|
|
|
|
.close = udp_lib_close,
|
2018-03-30 22:08:05 +00:00
|
|
|
.pre_connect = udp_pre_connect,
|
2018-03-14 04:57:16 +00:00
|
|
|
.connect = ip4_datagram_connect,
|
|
|
|
.disconnect = udp_disconnect,
|
|
|
|
.ioctl = udp_ioctl,
|
|
|
|
.init = udp_init_sock,
|
|
|
|
.destroy = udp_destroy_sock,
|
|
|
|
.setsockopt = udp_setsockopt,
|
|
|
|
.getsockopt = udp_getsockopt,
|
|
|
|
.sendmsg = udp_sendmsg,
|
|
|
|
.recvmsg = udp_recvmsg,
|
2023-06-07 18:19:13 +00:00
|
|
|
.splice_eof = udp_splice_eof,
|
2018-03-14 04:57:16 +00:00
|
|
|
.release_cb = ip4_datagram_release_cb,
|
|
|
|
.hash = udp_lib_hash,
|
|
|
|
.unhash = udp_lib_unhash,
|
|
|
|
.rehash = udp_v4_rehash,
|
|
|
|
.get_port = udp_v4_get_port,
|
net: bpf: Handle return value of BPF_CGROUP_RUN_PROG_INET{4,6}_POST_BIND()
The return value of BPF_CGROUP_RUN_PROG_INET{4,6}_POST_BIND() in
__inet_bind() is not handled properly. While the return value
is non-zero, it will set inet_saddr and inet_rcv_saddr to 0 and
exit:
err = BPF_CGROUP_RUN_PROG_INET4_POST_BIND(sk);
if (err) {
inet->inet_saddr = inet->inet_rcv_saddr = 0;
goto out_release_sock;
}
Let's take UDP for example and see what will happen. For UDP
socket, it will be added to 'udp_prot.h.udp_table->hash' and
'udp_prot.h.udp_table->hash2' after the sk->sk_prot->get_port()
called success. If 'inet->inet_rcv_saddr' is specified here,
then 'sk' will be in the 'hslot2' of 'hash2' that it don't belong
to (because inet_saddr is changed to 0), and UDP packet received
will not be passed to this sock. If 'inet->inet_rcv_saddr' is not
specified here, the sock will work fine, as it can receive packet
properly, which is wired, as the 'bind()' is already failed.
To undo the get_port() operation, introduce the 'put_port' field
for 'struct proto'. For TCP proto, it is inet_put_port(); For UDP
proto, it is udp_lib_unhash(); For icmp proto, it is
ping_unhash().
Therefore, after sys_bind() fail caused by
BPF_CGROUP_RUN_PROG_INET4_POST_BIND(), it will be unbinded, which
means that it can try to be binded to another port.
Signed-off-by: Menglong Dong <imagedong@tencent.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Link: https://lore.kernel.org/bpf/20220106132022.3470772-2-imagedong@tencent.com
2022-01-06 13:20:20 +00:00
|
|
|
.put_port = udp_lib_unhash,
|
2021-03-31 02:32:31 +00:00
|
|
|
#ifdef CONFIG_BPF_SYSCALL
|
|
|
|
.psock_update_sk_prot = udp_bpf_update_proto,
|
|
|
|
#endif
|
2018-03-14 04:57:16 +00:00
|
|
|
.memory_allocated = &udp_memory_allocated,
|
2022-06-09 06:34:08 +00:00
|
|
|
.per_cpu_fw_alloc = &udp_memory_per_cpu_fw_alloc,
|
|
|
|
|
2018-03-14 04:57:16 +00:00
|
|
|
.sysctl_mem = sysctl_udp_mem,
|
|
|
|
.sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_udp_wmem_min),
|
|
|
|
.sysctl_rmem_offset = offsetof(struct net, ipv4.sysctl_udp_rmem_min),
|
|
|
|
.obj_size = sizeof(struct udp_sock),
|
2022-11-14 21:57:54 +00:00
|
|
|
.h.udp_table = NULL,
|
2018-03-14 04:57:16 +00:00
|
|
|
.diag_destroy = udp_abort,
|
2008-03-07 00:22:02 +00:00
|
|
|
};
|
2009-07-17 00:26:32 +00:00
|
|
|
EXPORT_SYMBOL(udp_prot);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
/* ------------------------------------------------------------------------ */
|
|
|
|
#ifdef CONFIG_PROC_FS
|
|
|
|
|
2023-05-19 22:51:50 +00:00
|
|
|
static unsigned short seq_file_family(const struct seq_file *seq);
|
|
|
|
static bool seq_sk_match(struct seq_file *seq, const struct sock *sk)
|
|
|
|
{
|
|
|
|
unsigned short family = seq_file_family(seq);
|
|
|
|
|
|
|
|
/* AF_UNSPEC is used as a match all */
|
|
|
|
return ((family == AF_UNSPEC || family == sk->sk_family) &&
|
|
|
|
net_eq(sock_net(sk), seq_file_net(seq)));
|
|
|
|
}
|
|
|
|
|
2023-05-19 22:51:52 +00:00
|
|
|
#ifdef CONFIG_BPF_SYSCALL
|
|
|
|
static const struct seq_operations bpf_iter_udp_seq_ops;
|
|
|
|
#endif
|
2023-05-19 22:51:51 +00:00
|
|
|
static struct udp_table *udp_get_table_seq(struct seq_file *seq,
|
|
|
|
struct net *net)
|
2022-11-14 21:57:55 +00:00
|
|
|
{
|
2023-05-19 22:51:51 +00:00
|
|
|
const struct udp_seq_afinfo *afinfo;
|
|
|
|
|
2023-05-19 22:51:52 +00:00
|
|
|
#ifdef CONFIG_BPF_SYSCALL
|
|
|
|
if (seq->op == &bpf_iter_udp_seq_ops)
|
2023-05-19 22:51:51 +00:00
|
|
|
return net->ipv4.udp_table;
|
2023-05-19 22:51:52 +00:00
|
|
|
#endif
|
2023-05-19 22:51:51 +00:00
|
|
|
|
|
|
|
afinfo = pde_data(file_inode(seq->file));
|
2022-11-14 21:57:55 +00:00
|
|
|
return afinfo->udp_table ? : net->ipv4.udp_table;
|
|
|
|
}
|
|
|
|
|
2008-10-29 08:41:45 +00:00
|
|
|
static struct sock *udp_get_first(struct seq_file *seq, int start)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
struct udp_iter_state *state = seq->private;
|
2008-03-29 01:23:33 +00:00
|
|
|
struct net *net = seq_file_net(seq);
|
2022-11-14 21:57:55 +00:00
|
|
|
struct udp_table *udptable;
|
2022-11-14 21:57:53 +00:00
|
|
|
struct sock *sk;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2023-05-19 22:51:51 +00:00
|
|
|
udptable = udp_get_table_seq(seq, net);
|
2022-11-14 21:57:55 +00:00
|
|
|
|
|
|
|
for (state->bucket = start; state->bucket <= udptable->mask;
|
2009-10-07 00:37:59 +00:00
|
|
|
++state->bucket) {
|
2022-11-14 21:57:55 +00:00
|
|
|
struct udp_hslot *hslot = &udptable->hash[state->bucket];
|
2009-10-07 00:37:59 +00:00
|
|
|
|
2016-04-01 15:52:13 +00:00
|
|
|
if (hlist_empty(&hslot->head))
|
2009-10-07 00:37:59 +00:00
|
|
|
continue;
|
|
|
|
|
2008-10-29 08:41:45 +00:00
|
|
|
spin_lock_bh(&hslot->lock);
|
2016-04-01 15:52:13 +00:00
|
|
|
sk_for_each(sk, &hslot->head) {
|
2023-05-19 22:51:50 +00:00
|
|
|
if (seq_sk_match(seq, sk))
|
2005-04-16 22:20:36 +00:00
|
|
|
goto found;
|
|
|
|
}
|
2008-10-29 08:41:45 +00:00
|
|
|
spin_unlock_bh(&hslot->lock);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
sk = NULL;
|
|
|
|
found:
|
|
|
|
return sk;
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct sock *udp_get_next(struct seq_file *seq, struct sock *sk)
|
|
|
|
{
|
|
|
|
struct udp_iter_state *state = seq->private;
|
2008-03-29 01:23:33 +00:00
|
|
|
struct net *net = seq_file_net(seq);
|
2022-11-14 21:57:55 +00:00
|
|
|
struct udp_table *udptable;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
do {
|
2016-04-01 15:52:13 +00:00
|
|
|
sk = sk_next(sk);
|
2023-05-19 22:51:50 +00:00
|
|
|
} while (sk && !seq_sk_match(seq, sk));
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2008-10-29 08:41:45 +00:00
|
|
|
if (!sk) {
|
2023-05-19 22:51:51 +00:00
|
|
|
udptable = udp_get_table_seq(seq, net);
|
2022-11-14 21:57:55 +00:00
|
|
|
|
|
|
|
if (state->bucket <= udptable->mask)
|
|
|
|
spin_unlock_bh(&udptable->hash[state->bucket].lock);
|
|
|
|
|
2008-10-29 08:41:45 +00:00
|
|
|
return udp_get_first(seq, state->bucket + 1);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
return sk;
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct sock *udp_get_idx(struct seq_file *seq, loff_t pos)
|
|
|
|
{
|
2008-10-29 08:41:45 +00:00
|
|
|
struct sock *sk = udp_get_first(seq, 0);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
if (sk)
|
2007-03-09 04:41:55 +00:00
|
|
|
while (pos && (sk = udp_get_next(seq, sk)) != NULL)
|
2005-04-16 22:20:36 +00:00
|
|
|
--pos;
|
|
|
|
return pos ? NULL : sk;
|
|
|
|
}
|
|
|
|
|
2018-04-10 19:31:50 +00:00
|
|
|
void *udp_seq_start(struct seq_file *seq, loff_t *pos)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2009-03-23 22:22:33 +00:00
|
|
|
struct udp_iter_state *state = seq->private;
|
2009-10-07 00:37:59 +00:00
|
|
|
state->bucket = MAX_UDP_PORTS;
|
2009-03-23 22:22:33 +00:00
|
|
|
|
2008-04-01 02:38:15 +00:00
|
|
|
return *pos ? udp_get_idx(seq, *pos-1) : SEQ_START_TOKEN;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2018-04-10 19:31:50 +00:00
|
|
|
EXPORT_SYMBOL(udp_seq_start);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2018-04-10 19:31:50 +00:00
|
|
|
void *udp_seq_next(struct seq_file *seq, void *v, loff_t *pos)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
struct sock *sk;
|
|
|
|
|
2008-04-01 02:38:15 +00:00
|
|
|
if (v == SEQ_START_TOKEN)
|
2005-04-16 22:20:36 +00:00
|
|
|
sk = udp_get_idx(seq, 0);
|
|
|
|
else
|
|
|
|
sk = udp_get_next(seq, v);
|
|
|
|
|
|
|
|
++*pos;
|
|
|
|
return sk;
|
|
|
|
}
|
2018-04-10 19:31:50 +00:00
|
|
|
EXPORT_SYMBOL(udp_seq_next);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2018-04-10 19:31:50 +00:00
|
|
|
void udp_seq_stop(struct seq_file *seq, void *v)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2008-10-29 08:41:45 +00:00
|
|
|
struct udp_iter_state *state = seq->private;
|
2022-11-14 21:57:55 +00:00
|
|
|
struct udp_table *udptable;
|
2008-10-29 08:41:45 +00:00
|
|
|
|
2023-05-19 22:51:51 +00:00
|
|
|
udptable = udp_get_table_seq(seq, seq_file_net(seq));
|
2022-11-14 21:57:55 +00:00
|
|
|
|
|
|
|
if (state->bucket <= udptable->mask)
|
|
|
|
spin_unlock_bh(&udptable->hash[state->bucket].lock);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2018-04-10 19:31:50 +00:00
|
|
|
EXPORT_SYMBOL(udp_seq_stop);
|
2008-03-07 00:22:02 +00:00
|
|
|
|
|
|
|
/* ------------------------------------------------------------------------ */
|
2008-04-24 08:02:16 +00:00
|
|
|
static void udp4_format_sock(struct sock *sp, struct seq_file *f,
|
2013-11-14 22:31:57 +00:00
|
|
|
int bucket)
|
2008-03-07 00:22:02 +00:00
|
|
|
{
|
|
|
|
struct inet_sock *inet = inet_sk(sp);
|
2009-10-15 06:30:45 +00:00
|
|
|
__be32 dest = inet->inet_daddr;
|
|
|
|
__be32 src = inet->inet_rcv_saddr;
|
|
|
|
__u16 destp = ntohs(inet->inet_dport);
|
|
|
|
__u16 srcp = ntohs(inet->inet_sport);
|
2008-03-07 00:22:02 +00:00
|
|
|
|
2009-10-07 00:37:59 +00:00
|
|
|
seq_printf(f, "%5d: %08X:%04X %08X:%04X"
|
2019-05-17 15:11:28 +00:00
|
|
|
" %02X %08X:%08X %02X:%08lX %08X %5u %8d %lu %d %pK %u",
|
2008-03-07 00:22:02 +00:00
|
|
|
bucket, src, srcp, dest, destp, sp->sk_state,
|
2009-06-18 02:05:41 +00:00
|
|
|
sk_wmem_alloc_get(sp),
|
2018-06-08 09:35:40 +00:00
|
|
|
udp_rqueue_get(sp),
|
2012-05-24 07:10:10 +00:00
|
|
|
0, 0L, 0,
|
|
|
|
from_kuid_munged(seq_user_ns(f), sock_i_uid(sp)),
|
|
|
|
0, sock_i_ino(sp),
|
2017-06-30 10:08:01 +00:00
|
|
|
refcount_read(&sp->sk_refcnt), sp,
|
2013-11-14 22:31:57 +00:00
|
|
|
atomic_read(&sp->sk_drops));
|
2008-03-07 00:22:02 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
int udp4_seq_show(struct seq_file *seq, void *v)
|
|
|
|
{
|
2013-11-14 22:31:57 +00:00
|
|
|
seq_setwidth(seq, 127);
|
2008-03-07 00:22:02 +00:00
|
|
|
if (v == SEQ_START_TOKEN)
|
2021-12-27 08:29:51 +00:00
|
|
|
seq_puts(seq, " sl local_address rem_address st tx_queue "
|
2008-03-07 00:22:02 +00:00
|
|
|
"rx_queue tr tm->when retrnsmt uid timeout "
|
2008-06-18 04:04:56 +00:00
|
|
|
"inode ref pointer drops");
|
2008-03-07 00:22:02 +00:00
|
|
|
else {
|
|
|
|
struct udp_iter_state *state = seq->private;
|
|
|
|
|
2013-11-14 22:31:57 +00:00
|
|
|
udp4_format_sock(v, seq, state->bucket);
|
2008-03-07 00:22:02 +00:00
|
|
|
}
|
2013-11-14 22:31:57 +00:00
|
|
|
seq_pad(seq, '\n');
|
2008-03-07 00:22:02 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2020-06-23 23:08:13 +00:00
|
|
|
#ifdef CONFIG_BPF_SYSCALL
|
|
|
|
struct bpf_iter__udp {
|
|
|
|
__bpf_md_ptr(struct bpf_iter_meta *, meta);
|
|
|
|
__bpf_md_ptr(struct udp_sock *, udp_sk);
|
|
|
|
uid_t uid __aligned(8);
|
|
|
|
int bucket __aligned(8);
|
|
|
|
};
|
|
|
|
|
2023-05-19 22:51:53 +00:00
|
|
|
struct bpf_udp_iter_state {
|
|
|
|
struct udp_iter_state state;
|
|
|
|
unsigned int cur_sk;
|
|
|
|
unsigned int end_sk;
|
|
|
|
unsigned int max_sk;
|
|
|
|
int offset;
|
|
|
|
struct sock **batch;
|
|
|
|
bool st_bucket_done;
|
|
|
|
};
|
|
|
|
|
|
|
|
static int bpf_iter_udp_realloc_batch(struct bpf_udp_iter_state *iter,
|
|
|
|
unsigned int new_batch_sz);
|
|
|
|
static struct sock *bpf_iter_udp_batch(struct seq_file *seq)
|
|
|
|
{
|
|
|
|
struct bpf_udp_iter_state *iter = seq->private;
|
|
|
|
struct udp_iter_state *state = &iter->state;
|
|
|
|
struct net *net = seq_file_net(seq);
|
|
|
|
struct udp_table *udptable;
|
|
|
|
unsigned int batch_sks = 0;
|
|
|
|
bool resized = false;
|
|
|
|
struct sock *sk;
|
|
|
|
|
|
|
|
/* The current batch is done, so advance the bucket. */
|
|
|
|
if (iter->st_bucket_done) {
|
|
|
|
state->bucket++;
|
|
|
|
iter->offset = 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
udptable = udp_get_table_seq(seq, net);
|
|
|
|
|
|
|
|
again:
|
|
|
|
/* New batch for the next bucket.
|
|
|
|
* Iterate over the hash table to find a bucket with sockets matching
|
|
|
|
* the iterator attributes, and return the first matching socket from
|
|
|
|
* the bucket. The remaining matched sockets from the bucket are batched
|
|
|
|
* before releasing the bucket lock. This allows BPF programs that are
|
|
|
|
* called in seq_show to acquire the bucket lock if needed.
|
|
|
|
*/
|
|
|
|
iter->cur_sk = 0;
|
|
|
|
iter->end_sk = 0;
|
|
|
|
iter->st_bucket_done = false;
|
|
|
|
batch_sks = 0;
|
|
|
|
|
|
|
|
for (; state->bucket <= udptable->mask; state->bucket++) {
|
|
|
|
struct udp_hslot *hslot2 = &udptable->hash2[state->bucket];
|
|
|
|
|
|
|
|
if (hlist_empty(&hslot2->head)) {
|
|
|
|
iter->offset = 0;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
spin_lock_bh(&hslot2->lock);
|
|
|
|
udp_portaddr_for_each_entry(sk, &hslot2->head) {
|
|
|
|
if (seq_sk_match(seq, sk)) {
|
|
|
|
/* Resume from the last iterated socket at the
|
|
|
|
* offset in the bucket before iterator was stopped.
|
|
|
|
*/
|
|
|
|
if (iter->offset) {
|
|
|
|
--iter->offset;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
if (iter->end_sk < iter->max_sk) {
|
|
|
|
sock_hold(sk);
|
|
|
|
iter->batch[iter->end_sk++] = sk;
|
|
|
|
}
|
|
|
|
batch_sks++;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
spin_unlock_bh(&hslot2->lock);
|
|
|
|
|
|
|
|
if (iter->end_sk)
|
|
|
|
break;
|
|
|
|
|
|
|
|
/* Reset the current bucket's offset before moving to the next bucket. */
|
|
|
|
iter->offset = 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* All done: no batch made. */
|
|
|
|
if (!iter->end_sk)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
if (iter->end_sk == batch_sks) {
|
|
|
|
/* Batching is done for the current bucket; return the first
|
|
|
|
* socket to be iterated from the batch.
|
|
|
|
*/
|
|
|
|
iter->st_bucket_done = true;
|
|
|
|
goto done;
|
|
|
|
}
|
|
|
|
if (!resized && !bpf_iter_udp_realloc_batch(iter, batch_sks * 3 / 2)) {
|
|
|
|
resized = true;
|
|
|
|
/* After allocating a larger batch, retry one more time to grab
|
|
|
|
* the whole bucket.
|
|
|
|
*/
|
|
|
|
state->bucket--;
|
|
|
|
goto again;
|
|
|
|
}
|
|
|
|
done:
|
|
|
|
return iter->batch[0];
|
|
|
|
}
|
|
|
|
|
|
|
|
static void *bpf_iter_udp_seq_next(struct seq_file *seq, void *v, loff_t *pos)
|
|
|
|
{
|
|
|
|
struct bpf_udp_iter_state *iter = seq->private;
|
|
|
|
struct sock *sk;
|
|
|
|
|
|
|
|
/* Whenever seq_next() is called, the iter->cur_sk is
|
|
|
|
* done with seq_show(), so unref the iter->cur_sk.
|
|
|
|
*/
|
|
|
|
if (iter->cur_sk < iter->end_sk) {
|
|
|
|
sock_put(iter->batch[iter->cur_sk++]);
|
|
|
|
++iter->offset;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* After updating iter->cur_sk, check if there are more sockets
|
|
|
|
* available in the current bucket batch.
|
|
|
|
*/
|
|
|
|
if (iter->cur_sk < iter->end_sk)
|
|
|
|
sk = iter->batch[iter->cur_sk];
|
|
|
|
else
|
|
|
|
/* Prepare a new batch. */
|
|
|
|
sk = bpf_iter_udp_batch(seq);
|
|
|
|
|
|
|
|
++*pos;
|
|
|
|
return sk;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void *bpf_iter_udp_seq_start(struct seq_file *seq, loff_t *pos)
|
|
|
|
{
|
|
|
|
/* bpf iter does not support lseek, so it always
|
|
|
|
* continue from where it was stop()-ped.
|
|
|
|
*/
|
|
|
|
if (*pos)
|
|
|
|
return bpf_iter_udp_batch(seq);
|
|
|
|
|
|
|
|
return SEQ_START_TOKEN;
|
|
|
|
}
|
|
|
|
|
2020-06-23 23:08:13 +00:00
|
|
|
static int udp_prog_seq_show(struct bpf_prog *prog, struct bpf_iter_meta *meta,
|
|
|
|
struct udp_sock *udp_sk, uid_t uid, int bucket)
|
|
|
|
{
|
|
|
|
struct bpf_iter__udp ctx;
|
|
|
|
|
|
|
|
meta->seq_num--; /* skip SEQ_START_TOKEN */
|
|
|
|
ctx.meta = meta;
|
|
|
|
ctx.udp_sk = udp_sk;
|
|
|
|
ctx.uid = uid;
|
|
|
|
ctx.bucket = bucket;
|
|
|
|
return bpf_iter_run_prog(prog, &ctx);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int bpf_iter_udp_seq_show(struct seq_file *seq, void *v)
|
|
|
|
{
|
|
|
|
struct udp_iter_state *state = seq->private;
|
|
|
|
struct bpf_iter_meta meta;
|
|
|
|
struct bpf_prog *prog;
|
|
|
|
struct sock *sk = v;
|
|
|
|
uid_t uid;
|
2023-05-19 22:51:53 +00:00
|
|
|
int ret;
|
2020-06-23 23:08:13 +00:00
|
|
|
|
|
|
|
if (v == SEQ_START_TOKEN)
|
|
|
|
return 0;
|
|
|
|
|
2023-05-19 22:51:53 +00:00
|
|
|
lock_sock(sk);
|
|
|
|
|
|
|
|
if (unlikely(sk_unhashed(sk))) {
|
|
|
|
ret = SEQ_SKIP;
|
|
|
|
goto unlock;
|
|
|
|
}
|
|
|
|
|
2020-06-23 23:08:13 +00:00
|
|
|
uid = from_kuid_munged(seq_user_ns(seq), sock_i_uid(sk));
|
|
|
|
meta.seq = seq;
|
|
|
|
prog = bpf_iter_get_info(&meta, false);
|
2023-05-19 22:51:53 +00:00
|
|
|
ret = udp_prog_seq_show(prog, &meta, v, uid, state->bucket);
|
|
|
|
|
|
|
|
unlock:
|
|
|
|
release_sock(sk);
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void bpf_iter_udp_put_batch(struct bpf_udp_iter_state *iter)
|
|
|
|
{
|
|
|
|
while (iter->cur_sk < iter->end_sk)
|
|
|
|
sock_put(iter->batch[iter->cur_sk++]);
|
2020-06-23 23:08:13 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void bpf_iter_udp_seq_stop(struct seq_file *seq, void *v)
|
|
|
|
{
|
2023-05-19 22:51:53 +00:00
|
|
|
struct bpf_udp_iter_state *iter = seq->private;
|
2020-06-23 23:08:13 +00:00
|
|
|
struct bpf_iter_meta meta;
|
|
|
|
struct bpf_prog *prog;
|
|
|
|
|
|
|
|
if (!v) {
|
|
|
|
meta.seq = seq;
|
|
|
|
prog = bpf_iter_get_info(&meta, true);
|
|
|
|
if (prog)
|
|
|
|
(void)udp_prog_seq_show(prog, &meta, v, 0, 0);
|
|
|
|
}
|
|
|
|
|
2023-05-19 22:51:53 +00:00
|
|
|
if (iter->cur_sk < iter->end_sk) {
|
|
|
|
bpf_iter_udp_put_batch(iter);
|
|
|
|
iter->st_bucket_done = false;
|
|
|
|
}
|
2020-06-23 23:08:13 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static const struct seq_operations bpf_iter_udp_seq_ops = {
|
2023-05-19 22:51:53 +00:00
|
|
|
.start = bpf_iter_udp_seq_start,
|
|
|
|
.next = bpf_iter_udp_seq_next,
|
2020-06-23 23:08:13 +00:00
|
|
|
.stop = bpf_iter_udp_seq_stop,
|
|
|
|
.show = bpf_iter_udp_seq_show,
|
|
|
|
};
|
|
|
|
#endif
|
|
|
|
|
2023-05-19 22:51:50 +00:00
|
|
|
static unsigned short seq_file_family(const struct seq_file *seq)
|
|
|
|
{
|
|
|
|
const struct udp_seq_afinfo *afinfo;
|
|
|
|
|
|
|
|
#ifdef CONFIG_BPF_SYSCALL
|
|
|
|
/* BPF iterator: bpf programs to filter sockets. */
|
|
|
|
if (seq->op == &bpf_iter_udp_seq_ops)
|
|
|
|
return AF_UNSPEC;
|
|
|
|
#endif
|
|
|
|
|
|
|
|
/* Proc fs iterator */
|
|
|
|
afinfo = pde_data(file_inode(seq->file));
|
|
|
|
return afinfo->family;
|
|
|
|
}
|
|
|
|
|
2018-04-10 17:42:55 +00:00
|
|
|
const struct seq_operations udp_seq_ops = {
|
2018-04-10 19:31:50 +00:00
|
|
|
.start = udp_seq_start,
|
|
|
|
.next = udp_seq_next,
|
|
|
|
.stop = udp_seq_stop,
|
|
|
|
.show = udp4_seq_show,
|
|
|
|
};
|
2018-04-10 17:42:55 +00:00
|
|
|
EXPORT_SYMBOL(udp_seq_ops);
|
2011-10-30 06:46:30 +00:00
|
|
|
|
2008-03-07 00:22:02 +00:00
|
|
|
static struct udp_seq_afinfo udp4_seq_afinfo = {
|
|
|
|
.family = AF_INET,
|
2022-11-14 21:57:55 +00:00
|
|
|
.udp_table = NULL,
|
2008-03-07 00:22:02 +00:00
|
|
|
};
|
|
|
|
|
2010-01-17 03:35:32 +00:00
|
|
|
static int __net_init udp4_proc_init_net(struct net *net)
|
2008-03-24 21:53:49 +00:00
|
|
|
{
|
2018-04-10 17:42:55 +00:00
|
|
|
if (!proc_create_net_data("udp", 0444, net->proc_net, &udp_seq_ops,
|
|
|
|
sizeof(struct udp_iter_state), &udp4_seq_afinfo))
|
2018-04-10 19:31:50 +00:00
|
|
|
return -ENOMEM;
|
|
|
|
return 0;
|
2008-03-24 21:53:49 +00:00
|
|
|
}
|
|
|
|
|
2010-01-17 03:35:32 +00:00
|
|
|
static void __net_exit udp4_proc_exit_net(struct net *net)
|
2008-03-24 21:53:49 +00:00
|
|
|
{
|
2018-04-10 19:31:50 +00:00
|
|
|
remove_proc_entry("udp", net->proc_net);
|
2008-03-24 21:53:49 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static struct pernet_operations udp4_net_ops = {
|
|
|
|
.init = udp4_proc_init_net,
|
|
|
|
.exit = udp4_proc_exit_net,
|
|
|
|
};
|
|
|
|
|
2008-03-07 00:22:02 +00:00
|
|
|
int __init udp4_proc_init(void)
|
|
|
|
{
|
2008-03-24 21:53:49 +00:00
|
|
|
return register_pernet_subsys(&udp4_net_ops);
|
2008-03-07 00:22:02 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void udp4_proc_exit(void)
|
|
|
|
{
|
2008-03-24 21:53:49 +00:00
|
|
|
unregister_pernet_subsys(&udp4_net_ops);
|
2008-03-07 00:22:02 +00:00
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
#endif /* CONFIG_PROC_FS */
|
|
|
|
|
2009-10-07 00:37:59 +00:00
|
|
|
static __initdata unsigned long uhash_entries;
|
|
|
|
static int __init set_uhash_entries(char *str)
|
2008-10-29 08:41:45 +00:00
|
|
|
{
|
2012-05-19 14:13:18 +00:00
|
|
|
ssize_t ret;
|
|
|
|
|
2009-10-07 00:37:59 +00:00
|
|
|
if (!str)
|
|
|
|
return 0;
|
2012-05-19 14:13:18 +00:00
|
|
|
|
|
|
|
ret = kstrtoul(str, 0, &uhash_entries);
|
|
|
|
if (ret)
|
|
|
|
return 0;
|
|
|
|
|
2009-10-07 00:37:59 +00:00
|
|
|
if (uhash_entries && uhash_entries < UDP_HTABLE_SIZE_MIN)
|
|
|
|
uhash_entries = UDP_HTABLE_SIZE_MIN;
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
__setup("uhash_entries=", set_uhash_entries);
|
2008-10-29 08:41:45 +00:00
|
|
|
|
2009-10-07 00:37:59 +00:00
|
|
|
void __init udp_table_init(struct udp_table *table, const char *name)
|
|
|
|
{
|
|
|
|
unsigned int i;
|
|
|
|
|
2012-05-23 13:33:35 +00:00
|
|
|
table->hash = alloc_large_system_hash(name,
|
|
|
|
2 * sizeof(struct udp_hslot),
|
|
|
|
uhash_entries,
|
|
|
|
21, /* one slot per 2 MB */
|
|
|
|
0,
|
|
|
|
&table->log,
|
|
|
|
&table->mask,
|
|
|
|
UDP_HTABLE_SIZE_MIN,
|
udp: Introduce optional per-netns hash table.
The maximum hash table size is 64K due to the nature of the protocol. [0]
It's smaller than TCP, and fewer sockets can cause a performance drop.
On an EC2 c5.24xlarge instance (192 GiB memory), after running iperf3 in
different netns, creating 32Mi sockets without data transfer in the root
netns causes regression for the iperf3's connection.
uhash_entries sockets length Gbps
64K 1 1 5.69
1Mi 16 5.27
2Mi 32 4.90
4Mi 64 4.09
8Mi 128 2.96
16Mi 256 2.06
32Mi 512 1.12
The per-netns hash table breaks the lengthy lists into shorter ones. It is
useful on a multi-tenant system with thousands of netns. With smaller hash
tables, we can look up sockets faster, isolate noisy neighbours, and reduce
lock contention.
The max size of the per-netns table is 64K as well. This is because the
possible hash range by udp_hashfn() always fits in 64K within the same
netns and we cannot make full use of the whole buckets larger than 64K.
/* 0 < num < 64K -> X < hash < X + 64K */
(num + net_hash_mix(net)) & mask;
Also, the min size is 128. We use a bitmap to search for an available
port in udp_lib_get_port(). To keep the bitmap on the stack and not
fire the CONFIG_FRAME_WARN error at build time, we round up the table
size to 128.
The sysctl usage is the same with TCP:
$ dmesg | cut -d ' ' -f 6- | grep "UDP hash"
UDP hash table entries: 65536 (order: 9, 2097152 bytes, vmalloc)
# sysctl net.ipv4.udp_hash_entries
net.ipv4.udp_hash_entries = 65536 # can be changed by uhash_entries
# sysctl net.ipv4.udp_child_hash_entries
net.ipv4.udp_child_hash_entries = 0 # disabled by default
# ip netns add test1
# ip netns exec test1 sysctl net.ipv4.udp_hash_entries
net.ipv4.udp_hash_entries = -65536 # share the global table
# sysctl -w net.ipv4.udp_child_hash_entries=100
net.ipv4.udp_child_hash_entries = 100
# ip netns add test2
# ip netns exec test2 sysctl net.ipv4.udp_hash_entries
net.ipv4.udp_hash_entries = 128 # own a per-netns table with 2^n buckets
We could optimise the hash table lookup/iteration further by removing
the netns comparison for the per-netns one in the future. Also, we
could optimise the sparse udp_hslot layout by putting it in udp_table.
[0]: https://lore.kernel.org/netdev/4ACC2815.7010101@gmail.com/
Signed-off-by: Kuniyuki Iwashima <kuniyu@amazon.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2022-11-14 21:57:57 +00:00
|
|
|
UDP_HTABLE_SIZE_MAX);
|
2012-05-23 13:33:35 +00:00
|
|
|
|
2009-11-08 10:17:58 +00:00
|
|
|
table->hash2 = table->hash + (table->mask + 1);
|
2009-10-07 00:37:59 +00:00
|
|
|
for (i = 0; i <= table->mask; i++) {
|
2016-04-01 15:52:13 +00:00
|
|
|
INIT_HLIST_HEAD(&table->hash[i].head);
|
2009-11-08 10:17:05 +00:00
|
|
|
table->hash[i].count = 0;
|
2008-10-29 08:41:45 +00:00
|
|
|
spin_lock_init(&table->hash[i].lock);
|
|
|
|
}
|
2009-11-08 10:17:58 +00:00
|
|
|
for (i = 0; i <= table->mask; i++) {
|
2016-04-01 15:52:13 +00:00
|
|
|
INIT_HLIST_HEAD(&table->hash2[i].head);
|
2009-11-08 10:17:58 +00:00
|
|
|
table->hash2[i].count = 0;
|
|
|
|
spin_lock_init(&table->hash2[i].lock);
|
|
|
|
}
|
2008-10-29 08:41:45 +00:00
|
|
|
}
|
|
|
|
|
2015-02-24 17:17:31 +00:00
|
|
|
u32 udp_flow_hashrnd(void)
|
|
|
|
{
|
|
|
|
static u32 hashrnd __read_mostly;
|
|
|
|
|
|
|
|
net_get_random_once(&hashrnd, sizeof(hashrnd));
|
|
|
|
|
|
|
|
return hashrnd;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(udp_flow_hashrnd);
|
|
|
|
|
udp: Introduce optional per-netns hash table.
The maximum hash table size is 64K due to the nature of the protocol. [0]
It's smaller than TCP, and fewer sockets can cause a performance drop.
On an EC2 c5.24xlarge instance (192 GiB memory), after running iperf3 in
different netns, creating 32Mi sockets without data transfer in the root
netns causes regression for the iperf3's connection.
uhash_entries sockets length Gbps
64K 1 1 5.69
1Mi 16 5.27
2Mi 32 4.90
4Mi 64 4.09
8Mi 128 2.96
16Mi 256 2.06
32Mi 512 1.12
The per-netns hash table breaks the lengthy lists into shorter ones. It is
useful on a multi-tenant system with thousands of netns. With smaller hash
tables, we can look up sockets faster, isolate noisy neighbours, and reduce
lock contention.
The max size of the per-netns table is 64K as well. This is because the
possible hash range by udp_hashfn() always fits in 64K within the same
netns and we cannot make full use of the whole buckets larger than 64K.
/* 0 < num < 64K -> X < hash < X + 64K */
(num + net_hash_mix(net)) & mask;
Also, the min size is 128. We use a bitmap to search for an available
port in udp_lib_get_port(). To keep the bitmap on the stack and not
fire the CONFIG_FRAME_WARN error at build time, we round up the table
size to 128.
The sysctl usage is the same with TCP:
$ dmesg | cut -d ' ' -f 6- | grep "UDP hash"
UDP hash table entries: 65536 (order: 9, 2097152 bytes, vmalloc)
# sysctl net.ipv4.udp_hash_entries
net.ipv4.udp_hash_entries = 65536 # can be changed by uhash_entries
# sysctl net.ipv4.udp_child_hash_entries
net.ipv4.udp_child_hash_entries = 0 # disabled by default
# ip netns add test1
# ip netns exec test1 sysctl net.ipv4.udp_hash_entries
net.ipv4.udp_hash_entries = -65536 # share the global table
# sysctl -w net.ipv4.udp_child_hash_entries=100
net.ipv4.udp_child_hash_entries = 100
# ip netns add test2
# ip netns exec test2 sysctl net.ipv4.udp_hash_entries
net.ipv4.udp_hash_entries = 128 # own a per-netns table with 2^n buckets
We could optimise the hash table lookup/iteration further by removing
the netns comparison for the per-netns one in the future. Also, we
could optimise the sparse udp_hslot layout by putting it in udp_table.
[0]: https://lore.kernel.org/netdev/4ACC2815.7010101@gmail.com/
Signed-off-by: Kuniyuki Iwashima <kuniyu@amazon.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2022-11-14 21:57:57 +00:00
|
|
|
static void __net_init udp_sysctl_init(struct net *net)
|
2018-03-14 04:57:16 +00:00
|
|
|
{
|
2022-06-09 06:34:07 +00:00
|
|
|
net->ipv4.sysctl_udp_rmem_min = PAGE_SIZE;
|
|
|
|
net->ipv4.sysctl_udp_wmem_min = PAGE_SIZE;
|
2018-03-14 04:57:16 +00:00
|
|
|
|
|
|
|
#ifdef CONFIG_NET_L3_MASTER_DEV
|
|
|
|
net->ipv4.sysctl_udp_l3mdev_accept = 0;
|
|
|
|
#endif
|
udp: Introduce optional per-netns hash table.
The maximum hash table size is 64K due to the nature of the protocol. [0]
It's smaller than TCP, and fewer sockets can cause a performance drop.
On an EC2 c5.24xlarge instance (192 GiB memory), after running iperf3 in
different netns, creating 32Mi sockets without data transfer in the root
netns causes regression for the iperf3's connection.
uhash_entries sockets length Gbps
64K 1 1 5.69
1Mi 16 5.27
2Mi 32 4.90
4Mi 64 4.09
8Mi 128 2.96
16Mi 256 2.06
32Mi 512 1.12
The per-netns hash table breaks the lengthy lists into shorter ones. It is
useful on a multi-tenant system with thousands of netns. With smaller hash
tables, we can look up sockets faster, isolate noisy neighbours, and reduce
lock contention.
The max size of the per-netns table is 64K as well. This is because the
possible hash range by udp_hashfn() always fits in 64K within the same
netns and we cannot make full use of the whole buckets larger than 64K.
/* 0 < num < 64K -> X < hash < X + 64K */
(num + net_hash_mix(net)) & mask;
Also, the min size is 128. We use a bitmap to search for an available
port in udp_lib_get_port(). To keep the bitmap on the stack and not
fire the CONFIG_FRAME_WARN error at build time, we round up the table
size to 128.
The sysctl usage is the same with TCP:
$ dmesg | cut -d ' ' -f 6- | grep "UDP hash"
UDP hash table entries: 65536 (order: 9, 2097152 bytes, vmalloc)
# sysctl net.ipv4.udp_hash_entries
net.ipv4.udp_hash_entries = 65536 # can be changed by uhash_entries
# sysctl net.ipv4.udp_child_hash_entries
net.ipv4.udp_child_hash_entries = 0 # disabled by default
# ip netns add test1
# ip netns exec test1 sysctl net.ipv4.udp_hash_entries
net.ipv4.udp_hash_entries = -65536 # share the global table
# sysctl -w net.ipv4.udp_child_hash_entries=100
net.ipv4.udp_child_hash_entries = 100
# ip netns add test2
# ip netns exec test2 sysctl net.ipv4.udp_hash_entries
net.ipv4.udp_hash_entries = 128 # own a per-netns table with 2^n buckets
We could optimise the hash table lookup/iteration further by removing
the netns comparison for the per-netns one in the future. Also, we
could optimise the sparse udp_hslot layout by putting it in udp_table.
[0]: https://lore.kernel.org/netdev/4ACC2815.7010101@gmail.com/
Signed-off-by: Kuniyuki Iwashima <kuniyu@amazon.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2022-11-14 21:57:57 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static struct udp_table __net_init *udp_pernet_table_alloc(unsigned int hash_entries)
|
|
|
|
{
|
|
|
|
struct udp_table *udptable;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
udptable = kmalloc(sizeof(*udptable), GFP_KERNEL);
|
|
|
|
if (!udptable)
|
|
|
|
goto out;
|
|
|
|
|
|
|
|
udptable->hash = vmalloc_huge(hash_entries * 2 * sizeof(struct udp_hslot),
|
|
|
|
GFP_KERNEL_ACCOUNT);
|
|
|
|
if (!udptable->hash)
|
|
|
|
goto free_table;
|
|
|
|
|
|
|
|
udptable->hash2 = udptable->hash + hash_entries;
|
|
|
|
udptable->mask = hash_entries - 1;
|
|
|
|
udptable->log = ilog2(hash_entries);
|
|
|
|
|
|
|
|
for (i = 0; i < hash_entries; i++) {
|
|
|
|
INIT_HLIST_HEAD(&udptable->hash[i].head);
|
|
|
|
udptable->hash[i].count = 0;
|
|
|
|
spin_lock_init(&udptable->hash[i].lock);
|
|
|
|
|
|
|
|
INIT_HLIST_HEAD(&udptable->hash2[i].head);
|
|
|
|
udptable->hash2[i].count = 0;
|
|
|
|
spin_lock_init(&udptable->hash2[i].lock);
|
|
|
|
}
|
|
|
|
|
|
|
|
return udptable;
|
|
|
|
|
|
|
|
free_table:
|
|
|
|
kfree(udptable);
|
|
|
|
out:
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void __net_exit udp_pernet_table_free(struct net *net)
|
|
|
|
{
|
|
|
|
struct udp_table *udptable = net->ipv4.udp_table;
|
|
|
|
|
|
|
|
if (udptable == &udp_table)
|
|
|
|
return;
|
|
|
|
|
|
|
|
kvfree(udptable->hash);
|
|
|
|
kfree(udptable);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void __net_init udp_set_table(struct net *net)
|
|
|
|
{
|
|
|
|
struct udp_table *udptable;
|
|
|
|
unsigned int hash_entries;
|
|
|
|
struct net *old_net;
|
|
|
|
|
|
|
|
if (net_eq(net, &init_net))
|
|
|
|
goto fallback;
|
|
|
|
|
|
|
|
old_net = current->nsproxy->net_ns;
|
|
|
|
hash_entries = READ_ONCE(old_net->ipv4.sysctl_udp_child_hash_entries);
|
|
|
|
if (!hash_entries)
|
|
|
|
goto fallback;
|
|
|
|
|
|
|
|
/* Set min to keep the bitmap on stack in udp_lib_get_port() */
|
|
|
|
if (hash_entries < UDP_HTABLE_SIZE_MIN_PERNET)
|
|
|
|
hash_entries = UDP_HTABLE_SIZE_MIN_PERNET;
|
|
|
|
else
|
|
|
|
hash_entries = roundup_pow_of_two(hash_entries);
|
|
|
|
|
|
|
|
udptable = udp_pernet_table_alloc(hash_entries);
|
|
|
|
if (udptable) {
|
|
|
|
net->ipv4.udp_table = udptable;
|
|
|
|
} else {
|
|
|
|
pr_warn("Failed to allocate UDP hash table (entries: %u) "
|
|
|
|
"for a netns, fallback to the global one\n",
|
|
|
|
hash_entries);
|
|
|
|
fallback:
|
|
|
|
net->ipv4.udp_table = &udp_table;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static int __net_init udp_pernet_init(struct net *net)
|
|
|
|
{
|
|
|
|
udp_sysctl_init(net);
|
|
|
|
udp_set_table(net);
|
2018-03-14 04:57:16 +00:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
udp: Introduce optional per-netns hash table.
The maximum hash table size is 64K due to the nature of the protocol. [0]
It's smaller than TCP, and fewer sockets can cause a performance drop.
On an EC2 c5.24xlarge instance (192 GiB memory), after running iperf3 in
different netns, creating 32Mi sockets without data transfer in the root
netns causes regression for the iperf3's connection.
uhash_entries sockets length Gbps
64K 1 1 5.69
1Mi 16 5.27
2Mi 32 4.90
4Mi 64 4.09
8Mi 128 2.96
16Mi 256 2.06
32Mi 512 1.12
The per-netns hash table breaks the lengthy lists into shorter ones. It is
useful on a multi-tenant system with thousands of netns. With smaller hash
tables, we can look up sockets faster, isolate noisy neighbours, and reduce
lock contention.
The max size of the per-netns table is 64K as well. This is because the
possible hash range by udp_hashfn() always fits in 64K within the same
netns and we cannot make full use of the whole buckets larger than 64K.
/* 0 < num < 64K -> X < hash < X + 64K */
(num + net_hash_mix(net)) & mask;
Also, the min size is 128. We use a bitmap to search for an available
port in udp_lib_get_port(). To keep the bitmap on the stack and not
fire the CONFIG_FRAME_WARN error at build time, we round up the table
size to 128.
The sysctl usage is the same with TCP:
$ dmesg | cut -d ' ' -f 6- | grep "UDP hash"
UDP hash table entries: 65536 (order: 9, 2097152 bytes, vmalloc)
# sysctl net.ipv4.udp_hash_entries
net.ipv4.udp_hash_entries = 65536 # can be changed by uhash_entries
# sysctl net.ipv4.udp_child_hash_entries
net.ipv4.udp_child_hash_entries = 0 # disabled by default
# ip netns add test1
# ip netns exec test1 sysctl net.ipv4.udp_hash_entries
net.ipv4.udp_hash_entries = -65536 # share the global table
# sysctl -w net.ipv4.udp_child_hash_entries=100
net.ipv4.udp_child_hash_entries = 100
# ip netns add test2
# ip netns exec test2 sysctl net.ipv4.udp_hash_entries
net.ipv4.udp_hash_entries = 128 # own a per-netns table with 2^n buckets
We could optimise the hash table lookup/iteration further by removing
the netns comparison for the per-netns one in the future. Also, we
could optimise the sparse udp_hslot layout by putting it in udp_table.
[0]: https://lore.kernel.org/netdev/4ACC2815.7010101@gmail.com/
Signed-off-by: Kuniyuki Iwashima <kuniyu@amazon.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2022-11-14 21:57:57 +00:00
|
|
|
static void __net_exit udp_pernet_exit(struct net *net)
|
|
|
|
{
|
|
|
|
udp_pernet_table_free(net);
|
|
|
|
}
|
|
|
|
|
2018-03-14 04:57:16 +00:00
|
|
|
static struct pernet_operations __net_initdata udp_sysctl_ops = {
|
udp: Introduce optional per-netns hash table.
The maximum hash table size is 64K due to the nature of the protocol. [0]
It's smaller than TCP, and fewer sockets can cause a performance drop.
On an EC2 c5.24xlarge instance (192 GiB memory), after running iperf3 in
different netns, creating 32Mi sockets without data transfer in the root
netns causes regression for the iperf3's connection.
uhash_entries sockets length Gbps
64K 1 1 5.69
1Mi 16 5.27
2Mi 32 4.90
4Mi 64 4.09
8Mi 128 2.96
16Mi 256 2.06
32Mi 512 1.12
The per-netns hash table breaks the lengthy lists into shorter ones. It is
useful on a multi-tenant system with thousands of netns. With smaller hash
tables, we can look up sockets faster, isolate noisy neighbours, and reduce
lock contention.
The max size of the per-netns table is 64K as well. This is because the
possible hash range by udp_hashfn() always fits in 64K within the same
netns and we cannot make full use of the whole buckets larger than 64K.
/* 0 < num < 64K -> X < hash < X + 64K */
(num + net_hash_mix(net)) & mask;
Also, the min size is 128. We use a bitmap to search for an available
port in udp_lib_get_port(). To keep the bitmap on the stack and not
fire the CONFIG_FRAME_WARN error at build time, we round up the table
size to 128.
The sysctl usage is the same with TCP:
$ dmesg | cut -d ' ' -f 6- | grep "UDP hash"
UDP hash table entries: 65536 (order: 9, 2097152 bytes, vmalloc)
# sysctl net.ipv4.udp_hash_entries
net.ipv4.udp_hash_entries = 65536 # can be changed by uhash_entries
# sysctl net.ipv4.udp_child_hash_entries
net.ipv4.udp_child_hash_entries = 0 # disabled by default
# ip netns add test1
# ip netns exec test1 sysctl net.ipv4.udp_hash_entries
net.ipv4.udp_hash_entries = -65536 # share the global table
# sysctl -w net.ipv4.udp_child_hash_entries=100
net.ipv4.udp_child_hash_entries = 100
# ip netns add test2
# ip netns exec test2 sysctl net.ipv4.udp_hash_entries
net.ipv4.udp_hash_entries = 128 # own a per-netns table with 2^n buckets
We could optimise the hash table lookup/iteration further by removing
the netns comparison for the per-netns one in the future. Also, we
could optimise the sparse udp_hslot layout by putting it in udp_table.
[0]: https://lore.kernel.org/netdev/4ACC2815.7010101@gmail.com/
Signed-off-by: Kuniyuki Iwashima <kuniyu@amazon.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2022-11-14 21:57:57 +00:00
|
|
|
.init = udp_pernet_init,
|
|
|
|
.exit = udp_pernet_exit,
|
2018-03-14 04:57:16 +00:00
|
|
|
};
|
|
|
|
|
2020-06-23 23:08:13 +00:00
|
|
|
#if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
|
|
|
|
DEFINE_BPF_ITER_FUNC(udp, struct bpf_iter_meta *meta,
|
|
|
|
struct udp_sock *udp_sk, uid_t uid, int bucket)
|
|
|
|
|
2023-05-19 22:51:53 +00:00
|
|
|
static int bpf_iter_udp_realloc_batch(struct bpf_udp_iter_state *iter,
|
|
|
|
unsigned int new_batch_sz)
|
2020-06-23 23:08:13 +00:00
|
|
|
{
|
2023-05-19 22:51:53 +00:00
|
|
|
struct sock **new_batch;
|
2020-06-23 23:08:13 +00:00
|
|
|
|
2023-05-19 22:51:53 +00:00
|
|
|
new_batch = kvmalloc_array(new_batch_sz, sizeof(*new_batch),
|
|
|
|
GFP_USER | __GFP_NOWARN);
|
|
|
|
if (!new_batch)
|
2020-06-23 23:08:13 +00:00
|
|
|
return -ENOMEM;
|
|
|
|
|
2023-05-19 22:51:53 +00:00
|
|
|
bpf_iter_udp_put_batch(iter);
|
|
|
|
kvfree(iter->batch);
|
|
|
|
iter->batch = new_batch;
|
|
|
|
iter->max_sk = new_batch_sz;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
#define INIT_BATCH_SZ 16
|
|
|
|
|
2020-07-23 18:41:10 +00:00
|
|
|
static int bpf_iter_init_udp(void *priv_data, struct bpf_iter_aux_info *aux)
|
2020-06-23 23:08:13 +00:00
|
|
|
{
|
2023-05-19 22:51:53 +00:00
|
|
|
struct bpf_udp_iter_state *iter = priv_data;
|
|
|
|
int ret;
|
|
|
|
|
2020-07-23 18:41:10 +00:00
|
|
|
ret = bpf_iter_init_seq_net(priv_data, aux);
|
2020-06-23 23:08:13 +00:00
|
|
|
if (ret)
|
2023-05-19 22:51:53 +00:00
|
|
|
return ret;
|
|
|
|
|
|
|
|
ret = bpf_iter_udp_realloc_batch(iter, INIT_BATCH_SZ);
|
|
|
|
if (ret)
|
|
|
|
bpf_iter_fini_seq_net(priv_data);
|
|
|
|
|
2020-06-23 23:08:13 +00:00
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void bpf_iter_fini_udp(void *priv_data)
|
|
|
|
{
|
2023-05-19 22:51:53 +00:00
|
|
|
struct bpf_udp_iter_state *iter = priv_data;
|
2020-06-23 23:08:13 +00:00
|
|
|
|
|
|
|
bpf_iter_fini_seq_net(priv_data);
|
2023-05-19 22:51:53 +00:00
|
|
|
kvfree(iter->batch);
|
2020-06-23 23:08:13 +00:00
|
|
|
}
|
|
|
|
|
2020-07-23 18:41:09 +00:00
|
|
|
static const struct bpf_iter_seq_info udp_seq_info = {
|
2020-06-23 23:08:13 +00:00
|
|
|
.seq_ops = &bpf_iter_udp_seq_ops,
|
|
|
|
.init_seq_private = bpf_iter_init_udp,
|
|
|
|
.fini_seq_private = bpf_iter_fini_udp,
|
2023-05-19 22:51:53 +00:00
|
|
|
.seq_priv_size = sizeof(struct bpf_udp_iter_state),
|
2020-07-23 18:41:09 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
static struct bpf_iter_reg udp_reg_info = {
|
|
|
|
.target = "udp",
|
2020-06-23 23:08:13 +00:00
|
|
|
.ctx_arg_info_size = 1,
|
|
|
|
.ctx_arg_info = {
|
|
|
|
{ offsetof(struct bpf_iter__udp, udp_sk),
|
bpf: Add bpf_sock_destroy kfunc
The socket destroy kfunc is used to forcefully terminate sockets from
certain BPF contexts. We plan to use the capability in Cilium
load-balancing to terminate client sockets that continue to connect to
deleted backends. The other use case is on-the-fly policy enforcement
where existing socket connections prevented by policies need to be
forcefully terminated. The kfunc also allows terminating sockets that may
or may not be actively sending traffic.
The kfunc can currently be called only from BPF TCP and UDP iterators
where users can filter, and terminate selected sockets. More
specifically, it can only be called from BPF contexts that ensure
socket locking in order to allow synchronous execution of protocol
specific `diag_destroy` handlers. The previous commit that batches UDP
sockets during iteration facilitated a synchronous invocation of the UDP
destroy callback from BPF context by skipping socket locks in
`udp_abort`. TCP iterator already supported batching of sockets being
iterated. To that end, `tracing_iter_filter` callback filter is added so
that verifier can restrict the kfunc to programs with `BPF_TRACE_ITER`
attach type, and reject other programs.
The kfunc takes `sock_common` type argument, even though it expects, and
casts them to a `sock` pointer. This enables the verifier to allow the
sock_destroy kfunc to be called for TCP with `sock_common` and UDP with
`sock` structs. Furthermore, as `sock_common` only has a subset of
certain fields of `sock`, casting pointer to the latter type might not
always be safe for certain sockets like request sockets, but these have a
special handling in the diag_destroy handlers.
Additionally, the kfunc is defined with `KF_TRUSTED_ARGS` flag to avoid the
cases where a `PTR_TO_BTF_ID` sk is obtained by following another pointer.
eg. getting a sk pointer (may be even NULL) by following another sk
pointer. The pointer socket argument passed in TCP and UDP iterators is
tagged as `PTR_TRUSTED` in {tcp,udp}_reg_info. The TRUSTED arg changes
are contributed by Martin KaFai Lau <martin.lau@kernel.org>.
Signed-off-by: Aditi Ghag <aditi.ghag@isovalent.com>
Link: https://lore.kernel.org/r/20230519225157.760788-8-aditi.ghag@isovalent.com
Signed-off-by: Martin KaFai Lau <martin.lau@kernel.org>
2023-05-19 22:51:55 +00:00
|
|
|
PTR_TO_BTF_ID_OR_NULL | PTR_TRUSTED },
|
2020-06-23 23:08:13 +00:00
|
|
|
},
|
2020-07-23 18:41:09 +00:00
|
|
|
.seq_info = &udp_seq_info,
|
2020-06-23 23:08:13 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
static void __init bpf_iter_register(void)
|
|
|
|
{
|
2020-07-20 16:34:03 +00:00
|
|
|
udp_reg_info.ctx_arg_info[0].btf_id = btf_sock_ids[BTF_SOCK_TYPE_UDP];
|
2020-06-23 23:08:13 +00:00
|
|
|
if (bpf_iter_reg_target(&udp_reg_info))
|
|
|
|
pr_warn("Warning: could not register bpf iterator udp\n");
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2007-12-31 08:29:24 +00:00
|
|
|
void __init udp_init(void)
|
|
|
|
{
|
2011-07-07 07:27:05 +00:00
|
|
|
unsigned long limit;
|
2016-12-08 19:41:54 +00:00
|
|
|
unsigned int i;
|
2007-12-31 08:29:24 +00:00
|
|
|
|
2009-10-07 00:37:59 +00:00
|
|
|
udp_table_init(&udp_table, "UDP");
|
2011-07-07 07:27:05 +00:00
|
|
|
limit = nr_free_buffer_pages() / 8;
|
2007-12-31 08:29:24 +00:00
|
|
|
limit = max(limit, 128UL);
|
|
|
|
sysctl_udp_mem[0] = limit / 4 * 3;
|
|
|
|
sysctl_udp_mem[1] = limit;
|
|
|
|
sysctl_udp_mem[2] = sysctl_udp_mem[0] * 2;
|
|
|
|
|
2016-12-08 19:41:54 +00:00
|
|
|
/* 16 spinlocks per cpu */
|
|
|
|
udp_busylocks_log = ilog2(nr_cpu_ids) + 4;
|
|
|
|
udp_busylocks = kmalloc(sizeof(spinlock_t) << udp_busylocks_log,
|
|
|
|
GFP_KERNEL);
|
|
|
|
if (!udp_busylocks)
|
|
|
|
panic("UDP: failed to alloc udp_busylocks\n");
|
|
|
|
for (i = 0; i < (1U << udp_busylocks_log); i++)
|
|
|
|
spin_lock_init(udp_busylocks + i);
|
2018-03-14 04:57:16 +00:00
|
|
|
|
|
|
|
if (register_pernet_subsys(&udp_sysctl_ops))
|
|
|
|
panic("UDP: failed to init sysctl parameters.\n");
|
2020-06-23 23:08:13 +00:00
|
|
|
|
|
|
|
#if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
|
|
|
|
bpf_iter_register();
|
|
|
|
#endif
|
2007-12-31 08:29:24 +00:00
|
|
|
}
|