diff --git a/include/linux/udp.h b/include/linux/udp.h index d04188714dca..3748e82b627b 100644 --- a/include/linux/udp.h +++ b/include/linux/udp.h @@ -92,6 +92,9 @@ struct udp_sock { /* This fields follows rcvbuf value, and is touched by udp_recvmsg */ int forward_threshold; + + /* Cache friendly copy of sk->sk_peek_off >= 0 */ + bool peeking_with_offset; }; #define udp_test_bit(nr, sk) \ @@ -109,6 +112,13 @@ struct udp_sock { #define udp_sk(ptr) container_of_const(ptr, struct udp_sock, inet.sk) +static inline int udp_set_peek_off(struct sock *sk, int val) +{ + sk_set_peek_off(sk, val); + WRITE_ONCE(udp_sk(sk)->peeking_with_offset, val >= 0); + return 0; +} + static inline void udp_set_no_check6_tx(struct sock *sk, bool val) { udp_assign_bit(NO_CHECK6_TX, sk, val); diff --git a/net/ipv4/af_inet.c b/net/ipv4/af_inet.c index ad278009e469..5daebdcbca32 100644 --- a/net/ipv4/af_inet.c +++ b/net/ipv4/af_inet.c @@ -1103,7 +1103,7 @@ const struct proto_ops inet_dgram_ops = { .recvmsg = inet_recvmsg, .mmap = sock_no_mmap, .splice_eof = inet_splice_eof, - .set_peek_off = sk_set_peek_off, + .set_peek_off = udp_set_peek_off, #ifdef CONFIG_COMPAT .compat_ioctl = inet_compat_ioctl, #endif diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c index f631b0a21af4..38cce7cc51f6 100644 --- a/net/ipv4/udp.c +++ b/net/ipv4/udp.c @@ -1589,7 +1589,7 @@ int udp_init_sock(struct sock *sk) void skb_consume_udp(struct sock *sk, struct sk_buff *skb, int len) { - if (unlikely(READ_ONCE(sk->sk_peek_off) >= 0)) { + if (unlikely(READ_ONCE(udp_sk(sk)->peeking_with_offset))) { bool slow = lock_sock_fast(sk); sk_peek_offset_bwd(sk, len); diff --git a/net/ipv6/af_inet6.c b/net/ipv6/af_inet6.c index 959bfd9f6344..b90d46533cdc 100644 --- a/net/ipv6/af_inet6.c +++ b/net/ipv6/af_inet6.c @@ -736,7 +736,7 @@ const struct proto_ops inet6_dgram_ops = { .recvmsg = inet6_recvmsg, /* retpoline's sake */ .read_skb = udp_read_skb, .mmap = sock_no_mmap, - .set_peek_off = sk_set_peek_off, + .set_peek_off = udp_set_peek_off, #ifdef CONFIG_COMPAT .compat_ioctl = inet6_compat_ioctl, #endif