mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-09-05 10:29:37 +00:00
Merge branch 'gro-remove-redundant-rcu_read_lock'
Eric Dumazet says: ==================== gro: remove redundant rcu_read_lock Recent trees got an increase of rcu_read_{lock|unlock} costs, it is time to get rid of the not needed pairs. ==================== Link: https://lore.kernel.org/r/20211123225608.2155163-1-eric.dumazet@gmail.com Signed-off-by: Jakub Kicinski <kuba@kernel.org>
This commit is contained in:
commit
80690a85f5
9 changed files with 22 additions and 76 deletions
|
@ -517,18 +517,15 @@ static struct sk_buff *geneve_gro_receive(struct sock *sk,
|
|||
|
||||
type = gh->proto_type;
|
||||
|
||||
rcu_read_lock();
|
||||
ptype = gro_find_receive_by_type(type);
|
||||
if (!ptype)
|
||||
goto out_unlock;
|
||||
goto out;
|
||||
|
||||
skb_gro_pull(skb, gh_len);
|
||||
skb_gro_postpull_rcsum(skb, gh, gh_len);
|
||||
pp = call_gro_receive(ptype->callbacks.gro_receive, head, skb);
|
||||
flush = 0;
|
||||
|
||||
out_unlock:
|
||||
rcu_read_unlock();
|
||||
out:
|
||||
skb_gro_flush_final(skb, pp, flush);
|
||||
|
||||
|
@ -548,13 +545,10 @@ static int geneve_gro_complete(struct sock *sk, struct sk_buff *skb,
|
|||
gh_len = geneve_hlen(gh);
|
||||
type = gh->proto_type;
|
||||
|
||||
rcu_read_lock();
|
||||
ptype = gro_find_complete_by_type(type);
|
||||
if (ptype)
|
||||
err = ptype->callbacks.gro_complete(skb, nhoff + gh_len);
|
||||
|
||||
rcu_read_unlock();
|
||||
|
||||
skb_set_inner_mac_header(skb, nhoff + gh_len);
|
||||
|
||||
return err;
|
||||
|
|
|
@ -476,10 +476,9 @@ static struct sk_buff *vlan_gro_receive(struct list_head *head,
|
|||
|
||||
type = vhdr->h_vlan_encapsulated_proto;
|
||||
|
||||
rcu_read_lock();
|
||||
ptype = gro_find_receive_by_type(type);
|
||||
if (!ptype)
|
||||
goto out_unlock;
|
||||
goto out;
|
||||
|
||||
flush = 0;
|
||||
|
||||
|
@ -501,8 +500,6 @@ static struct sk_buff *vlan_gro_receive(struct list_head *head,
|
|||
ipv6_gro_receive, inet_gro_receive,
|
||||
head, skb);
|
||||
|
||||
out_unlock:
|
||||
rcu_read_unlock();
|
||||
out:
|
||||
skb_gro_flush_final(skb, pp, flush);
|
||||
|
||||
|
@ -516,14 +513,12 @@ static int vlan_gro_complete(struct sk_buff *skb, int nhoff)
|
|||
struct packet_offload *ptype;
|
||||
int err = -ENOENT;
|
||||
|
||||
rcu_read_lock();
|
||||
ptype = gro_find_complete_by_type(type);
|
||||
if (ptype)
|
||||
err = INDIRECT_CALL_INET(ptype->callbacks.gro_complete,
|
||||
ipv6_gro_complete, inet_gro_complete,
|
||||
skb, nhoff + sizeof(*vhdr));
|
||||
|
||||
rcu_read_unlock();
|
||||
return err;
|
||||
}
|
||||
|
||||
|
|
|
@ -436,11 +436,10 @@ struct sk_buff *eth_gro_receive(struct list_head *head, struct sk_buff *skb)
|
|||
|
||||
type = eh->h_proto;
|
||||
|
||||
rcu_read_lock();
|
||||
ptype = gro_find_receive_by_type(type);
|
||||
if (ptype == NULL) {
|
||||
flush = 1;
|
||||
goto out_unlock;
|
||||
goto out;
|
||||
}
|
||||
|
||||
skb_gro_pull(skb, sizeof(*eh));
|
||||
|
@ -450,8 +449,6 @@ struct sk_buff *eth_gro_receive(struct list_head *head, struct sk_buff *skb)
|
|||
ipv6_gro_receive, inet_gro_receive,
|
||||
head, skb);
|
||||
|
||||
out_unlock:
|
||||
rcu_read_unlock();
|
||||
out:
|
||||
skb_gro_flush_final(skb, pp, flush);
|
||||
|
||||
|
@ -469,14 +466,12 @@ int eth_gro_complete(struct sk_buff *skb, int nhoff)
|
|||
if (skb->encapsulation)
|
||||
skb_set_inner_mac_header(skb, nhoff);
|
||||
|
||||
rcu_read_lock();
|
||||
ptype = gro_find_complete_by_type(type);
|
||||
if (ptype != NULL)
|
||||
err = INDIRECT_CALL_INET(ptype->callbacks.gro_complete,
|
||||
ipv6_gro_complete, inet_gro_complete,
|
||||
skb, nhoff + sizeof(*eh));
|
||||
|
||||
rcu_read_unlock();
|
||||
return err;
|
||||
}
|
||||
EXPORT_SYMBOL(eth_gro_complete);
|
||||
|
|
|
@ -1452,19 +1452,18 @@ struct sk_buff *inet_gro_receive(struct list_head *head, struct sk_buff *skb)
|
|||
|
||||
proto = iph->protocol;
|
||||
|
||||
rcu_read_lock();
|
||||
ops = rcu_dereference(inet_offloads[proto]);
|
||||
if (!ops || !ops->callbacks.gro_receive)
|
||||
goto out_unlock;
|
||||
goto out;
|
||||
|
||||
if (*(u8 *)iph != 0x45)
|
||||
goto out_unlock;
|
||||
goto out;
|
||||
|
||||
if (ip_is_fragment(iph))
|
||||
goto out_unlock;
|
||||
goto out;
|
||||
|
||||
if (unlikely(ip_fast_csum((u8 *)iph, 5)))
|
||||
goto out_unlock;
|
||||
goto out;
|
||||
|
||||
id = ntohl(*(__be32 *)&iph->id);
|
||||
flush = (u16)((ntohl(*(__be32 *)iph) ^ skb_gro_len(skb)) | (id & ~IP_DF));
|
||||
|
@ -1541,9 +1540,6 @@ struct sk_buff *inet_gro_receive(struct list_head *head, struct sk_buff *skb)
|
|||
pp = indirect_call_gro_receive(tcp4_gro_receive, udp4_gro_receive,
|
||||
ops->callbacks.gro_receive, head, skb);
|
||||
|
||||
out_unlock:
|
||||
rcu_read_unlock();
|
||||
|
||||
out:
|
||||
skb_gro_flush_final(skb, pp, flush);
|
||||
|
||||
|
@ -1616,10 +1612,9 @@ int inet_gro_complete(struct sk_buff *skb, int nhoff)
|
|||
csum_replace2(&iph->check, iph->tot_len, newlen);
|
||||
iph->tot_len = newlen;
|
||||
|
||||
rcu_read_lock();
|
||||
ops = rcu_dereference(inet_offloads[proto]);
|
||||
if (WARN_ON(!ops || !ops->callbacks.gro_complete))
|
||||
goto out_unlock;
|
||||
goto out;
|
||||
|
||||
/* Only need to add sizeof(*iph) to get to the next hdr below
|
||||
* because any hdr with option will have been flushed in
|
||||
|
@ -1629,9 +1624,7 @@ int inet_gro_complete(struct sk_buff *skb, int nhoff)
|
|||
tcp4_gro_complete, udp4_gro_complete,
|
||||
skb, nhoff + sizeof(*iph));
|
||||
|
||||
out_unlock:
|
||||
rcu_read_unlock();
|
||||
|
||||
out:
|
||||
return err;
|
||||
}
|
||||
|
||||
|
|
|
@ -247,17 +247,14 @@ static struct sk_buff *fou_gro_receive(struct sock *sk,
|
|||
/* Flag this frame as already having an outer encap header */
|
||||
NAPI_GRO_CB(skb)->is_fou = 1;
|
||||
|
||||
rcu_read_lock();
|
||||
offloads = NAPI_GRO_CB(skb)->is_ipv6 ? inet6_offloads : inet_offloads;
|
||||
ops = rcu_dereference(offloads[proto]);
|
||||
if (!ops || !ops->callbacks.gro_receive)
|
||||
goto out_unlock;
|
||||
goto out;
|
||||
|
||||
pp = call_gro_receive(ops->callbacks.gro_receive, head, skb);
|
||||
|
||||
out_unlock:
|
||||
rcu_read_unlock();
|
||||
|
||||
out:
|
||||
return pp;
|
||||
}
|
||||
|
||||
|
@ -269,19 +266,16 @@ static int fou_gro_complete(struct sock *sk, struct sk_buff *skb,
|
|||
const struct net_offload *ops;
|
||||
int err = -ENOSYS;
|
||||
|
||||
rcu_read_lock();
|
||||
offloads = NAPI_GRO_CB(skb)->is_ipv6 ? inet6_offloads : inet_offloads;
|
||||
ops = rcu_dereference(offloads[proto]);
|
||||
if (WARN_ON(!ops || !ops->callbacks.gro_complete))
|
||||
goto out_unlock;
|
||||
goto out;
|
||||
|
||||
err = ops->callbacks.gro_complete(skb, nhoff);
|
||||
|
||||
skb_set_inner_mac_header(skb, nhoff);
|
||||
|
||||
out_unlock:
|
||||
rcu_read_unlock();
|
||||
|
||||
out:
|
||||
return err;
|
||||
}
|
||||
|
||||
|
@ -439,17 +433,14 @@ static struct sk_buff *gue_gro_receive(struct sock *sk,
|
|||
/* Flag this frame as already having an outer encap header */
|
||||
NAPI_GRO_CB(skb)->is_fou = 1;
|
||||
|
||||
rcu_read_lock();
|
||||
offloads = NAPI_GRO_CB(skb)->is_ipv6 ? inet6_offloads : inet_offloads;
|
||||
ops = rcu_dereference(offloads[proto]);
|
||||
if (WARN_ON_ONCE(!ops || !ops->callbacks.gro_receive))
|
||||
goto out_unlock;
|
||||
goto out;
|
||||
|
||||
pp = call_gro_receive(ops->callbacks.gro_receive, head, skb);
|
||||
flush = 0;
|
||||
|
||||
out_unlock:
|
||||
rcu_read_unlock();
|
||||
out:
|
||||
skb_gro_flush_final_remcsum(skb, pp, flush, &grc);
|
||||
|
||||
|
@ -486,18 +477,16 @@ static int gue_gro_complete(struct sock *sk, struct sk_buff *skb, int nhoff)
|
|||
return err;
|
||||
}
|
||||
|
||||
rcu_read_lock();
|
||||
offloads = NAPI_GRO_CB(skb)->is_ipv6 ? inet6_offloads : inet_offloads;
|
||||
ops = rcu_dereference(offloads[proto]);
|
||||
if (WARN_ON(!ops || !ops->callbacks.gro_complete))
|
||||
goto out_unlock;
|
||||
goto out;
|
||||
|
||||
err = ops->callbacks.gro_complete(skb, nhoff + guehlen);
|
||||
|
||||
skb_set_inner_mac_header(skb, nhoff + guehlen);
|
||||
|
||||
out_unlock:
|
||||
rcu_read_unlock();
|
||||
out:
|
||||
return err;
|
||||
}
|
||||
|
||||
|
|
|
@ -163,10 +163,9 @@ static struct sk_buff *gre_gro_receive(struct list_head *head,
|
|||
|
||||
type = greh->protocol;
|
||||
|
||||
rcu_read_lock();
|
||||
ptype = gro_find_receive_by_type(type);
|
||||
if (!ptype)
|
||||
goto out_unlock;
|
||||
goto out;
|
||||
|
||||
grehlen = GRE_HEADER_SECTION;
|
||||
|
||||
|
@ -180,13 +179,13 @@ static struct sk_buff *gre_gro_receive(struct list_head *head,
|
|||
if (skb_gro_header_hard(skb, hlen)) {
|
||||
greh = skb_gro_header_slow(skb, hlen, off);
|
||||
if (unlikely(!greh))
|
||||
goto out_unlock;
|
||||
goto out;
|
||||
}
|
||||
|
||||
/* Don't bother verifying checksum if we're going to flush anyway. */
|
||||
if ((greh->flags & GRE_CSUM) && !NAPI_GRO_CB(skb)->flush) {
|
||||
if (skb_gro_checksum_simple_validate(skb))
|
||||
goto out_unlock;
|
||||
goto out;
|
||||
|
||||
skb_gro_checksum_try_convert(skb, IPPROTO_GRE,
|
||||
null_compute_pseudo);
|
||||
|
@ -230,8 +229,6 @@ static struct sk_buff *gre_gro_receive(struct list_head *head,
|
|||
pp = call_gro_receive(ptype->callbacks.gro_receive, head, skb);
|
||||
flush = 0;
|
||||
|
||||
out_unlock:
|
||||
rcu_read_unlock();
|
||||
out:
|
||||
skb_gro_flush_final(skb, pp, flush);
|
||||
|
||||
|
@ -256,13 +253,10 @@ static int gre_gro_complete(struct sk_buff *skb, int nhoff)
|
|||
if (greh->flags & GRE_CSUM)
|
||||
grehlen += GRE_HEADER_SECTION;
|
||||
|
||||
rcu_read_lock();
|
||||
ptype = gro_find_complete_by_type(type);
|
||||
if (ptype)
|
||||
err = ptype->callbacks.gro_complete(skb, nhoff + grehlen);
|
||||
|
||||
rcu_read_unlock();
|
||||
|
||||
skb_set_inner_mac_header(skb, nhoff + grehlen);
|
||||
|
||||
return err;
|
||||
|
|
|
@ -628,13 +628,11 @@ struct sk_buff *udp4_gro_receive(struct list_head *head, struct sk_buff *skb)
|
|||
inet_gro_compute_pseudo);
|
||||
skip:
|
||||
NAPI_GRO_CB(skb)->is_ipv6 = 0;
|
||||
rcu_read_lock();
|
||||
|
||||
if (static_branch_unlikely(&udp_encap_needed_key))
|
||||
sk = udp4_gro_lookup_skb(skb, uh->source, uh->dest);
|
||||
|
||||
pp = udp_gro_receive(head, skb, uh, sk);
|
||||
rcu_read_unlock();
|
||||
return pp;
|
||||
|
||||
flush:
|
||||
|
@ -669,7 +667,6 @@ int udp_gro_complete(struct sk_buff *skb, int nhoff,
|
|||
|
||||
uh->len = newlen;
|
||||
|
||||
rcu_read_lock();
|
||||
sk = INDIRECT_CALL_INET(lookup, udp6_lib_lookup_skb,
|
||||
udp4_lib_lookup_skb, skb, uh->source, uh->dest);
|
||||
if (sk && udp_sk(sk)->gro_complete) {
|
||||
|
@ -690,7 +687,6 @@ int udp_gro_complete(struct sk_buff *skb, int nhoff,
|
|||
} else {
|
||||
err = udp_gro_complete_segment(skb);
|
||||
}
|
||||
rcu_read_unlock();
|
||||
|
||||
if (skb->remcsum_offload)
|
||||
skb_shinfo(skb)->gso_type |= SKB_GSO_TUNNEL_REMCSUM;
|
||||
|
|
|
@ -208,7 +208,6 @@ INDIRECT_CALLABLE_SCOPE struct sk_buff *ipv6_gro_receive(struct list_head *head,
|
|||
|
||||
flush += ntohs(iph->payload_len) != skb_gro_len(skb);
|
||||
|
||||
rcu_read_lock();
|
||||
proto = iph->nexthdr;
|
||||
ops = rcu_dereference(inet6_offloads[proto]);
|
||||
if (!ops || !ops->callbacks.gro_receive) {
|
||||
|
@ -221,7 +220,7 @@ INDIRECT_CALLABLE_SCOPE struct sk_buff *ipv6_gro_receive(struct list_head *head,
|
|||
|
||||
ops = rcu_dereference(inet6_offloads[proto]);
|
||||
if (!ops || !ops->callbacks.gro_receive)
|
||||
goto out_unlock;
|
||||
goto out;
|
||||
|
||||
iph = ipv6_hdr(skb);
|
||||
}
|
||||
|
@ -279,9 +278,6 @@ INDIRECT_CALLABLE_SCOPE struct sk_buff *ipv6_gro_receive(struct list_head *head,
|
|||
pp = indirect_call_gro_receive_l4(tcp6_gro_receive, udp6_gro_receive,
|
||||
ops->callbacks.gro_receive, head, skb);
|
||||
|
||||
out_unlock:
|
||||
rcu_read_unlock();
|
||||
|
||||
out:
|
||||
skb_gro_flush_final(skb, pp, flush);
|
||||
|
||||
|
@ -331,18 +327,14 @@ INDIRECT_CALLABLE_SCOPE int ipv6_gro_complete(struct sk_buff *skb, int nhoff)
|
|||
|
||||
iph->payload_len = htons(skb->len - nhoff - sizeof(*iph));
|
||||
|
||||
rcu_read_lock();
|
||||
|
||||
nhoff += sizeof(*iph) + ipv6_exthdrs_len(iph, &ops);
|
||||
if (WARN_ON(!ops || !ops->callbacks.gro_complete))
|
||||
goto out_unlock;
|
||||
goto out;
|
||||
|
||||
err = INDIRECT_CALL_L4(ops->callbacks.gro_complete, tcp6_gro_complete,
|
||||
udp6_gro_complete, skb, nhoff);
|
||||
|
||||
out_unlock:
|
||||
rcu_read_unlock();
|
||||
|
||||
out:
|
||||
return err;
|
||||
}
|
||||
|
||||
|
|
|
@ -145,13 +145,11 @@ struct sk_buff *udp6_gro_receive(struct list_head *head, struct sk_buff *skb)
|
|||
|
||||
skip:
|
||||
NAPI_GRO_CB(skb)->is_ipv6 = 1;
|
||||
rcu_read_lock();
|
||||
|
||||
if (static_branch_unlikely(&udpv6_encap_needed_key))
|
||||
sk = udp6_gro_lookup_skb(skb, uh->source, uh->dest);
|
||||
|
||||
pp = udp_gro_receive(head, skb, uh, sk);
|
||||
rcu_read_unlock();
|
||||
return pp;
|
||||
|
||||
flush:
|
||||
|
|
Loading…
Reference in a new issue