mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-08-28 20:02:17 +00:00
![Daniel Borkmann](/assets/img/avatar_default.png)
__skb_get_poff() returns the offset to the payload as far as it could be dissected. The main user is currently BPF, so that we can dynamically truncate packets without needing to push actual payload to the user space and instead can analyze headers only. Suggested-by: Eric Dumazet <edumazet@google.com> Signed-off-by: Daniel Borkmann <dborkman@redhat.com> Acked-by: Eric Dumazet <edumazet@google.com> Signed-off-by: David S. Miller <davem@davemloft.net>
388 lines
8.8 KiB
C
388 lines
8.8 KiB
C
#include <linux/skbuff.h>
|
|
#include <linux/export.h>
|
|
#include <linux/ip.h>
|
|
#include <linux/ipv6.h>
|
|
#include <linux/if_vlan.h>
|
|
#include <net/ip.h>
|
|
#include <net/ipv6.h>
|
|
#include <linux/igmp.h>
|
|
#include <linux/icmp.h>
|
|
#include <linux/sctp.h>
|
|
#include <linux/dccp.h>
|
|
#include <linux/if_tunnel.h>
|
|
#include <linux/if_pppox.h>
|
|
#include <linux/ppp_defs.h>
|
|
#include <net/flow_keys.h>
|
|
|
|
/* copy saddr & daddr, possibly using 64bit load/store
|
|
* Equivalent to : flow->src = iph->saddr;
|
|
* flow->dst = iph->daddr;
|
|
*/
|
|
static void iph_to_flow_copy_addrs(struct flow_keys *flow, const struct iphdr *iph)
|
|
{
|
|
BUILD_BUG_ON(offsetof(typeof(*flow), dst) !=
|
|
offsetof(typeof(*flow), src) + sizeof(flow->src));
|
|
memcpy(&flow->src, &iph->saddr, sizeof(flow->src) + sizeof(flow->dst));
|
|
}
|
|
|
|
bool skb_flow_dissect(const struct sk_buff *skb, struct flow_keys *flow)
|
|
{
|
|
int poff, nhoff = skb_network_offset(skb);
|
|
u8 ip_proto;
|
|
__be16 proto = skb->protocol;
|
|
|
|
memset(flow, 0, sizeof(*flow));
|
|
|
|
again:
|
|
switch (proto) {
|
|
case __constant_htons(ETH_P_IP): {
|
|
const struct iphdr *iph;
|
|
struct iphdr _iph;
|
|
ip:
|
|
iph = skb_header_pointer(skb, nhoff, sizeof(_iph), &_iph);
|
|
if (!iph)
|
|
return false;
|
|
|
|
if (ip_is_fragment(iph))
|
|
ip_proto = 0;
|
|
else
|
|
ip_proto = iph->protocol;
|
|
iph_to_flow_copy_addrs(flow, iph);
|
|
nhoff += iph->ihl * 4;
|
|
break;
|
|
}
|
|
case __constant_htons(ETH_P_IPV6): {
|
|
const struct ipv6hdr *iph;
|
|
struct ipv6hdr _iph;
|
|
ipv6:
|
|
iph = skb_header_pointer(skb, nhoff, sizeof(_iph), &_iph);
|
|
if (!iph)
|
|
return false;
|
|
|
|
ip_proto = iph->nexthdr;
|
|
flow->src = (__force __be32)ipv6_addr_hash(&iph->saddr);
|
|
flow->dst = (__force __be32)ipv6_addr_hash(&iph->daddr);
|
|
nhoff += sizeof(struct ipv6hdr);
|
|
break;
|
|
}
|
|
case __constant_htons(ETH_P_8021Q): {
|
|
const struct vlan_hdr *vlan;
|
|
struct vlan_hdr _vlan;
|
|
|
|
vlan = skb_header_pointer(skb, nhoff, sizeof(_vlan), &_vlan);
|
|
if (!vlan)
|
|
return false;
|
|
|
|
proto = vlan->h_vlan_encapsulated_proto;
|
|
nhoff += sizeof(*vlan);
|
|
goto again;
|
|
}
|
|
case __constant_htons(ETH_P_PPP_SES): {
|
|
struct {
|
|
struct pppoe_hdr hdr;
|
|
__be16 proto;
|
|
} *hdr, _hdr;
|
|
hdr = skb_header_pointer(skb, nhoff, sizeof(_hdr), &_hdr);
|
|
if (!hdr)
|
|
return false;
|
|
proto = hdr->proto;
|
|
nhoff += PPPOE_SES_HLEN;
|
|
switch (proto) {
|
|
case __constant_htons(PPP_IP):
|
|
goto ip;
|
|
case __constant_htons(PPP_IPV6):
|
|
goto ipv6;
|
|
default:
|
|
return false;
|
|
}
|
|
}
|
|
default:
|
|
return false;
|
|
}
|
|
|
|
switch (ip_proto) {
|
|
case IPPROTO_GRE: {
|
|
struct gre_hdr {
|
|
__be16 flags;
|
|
__be16 proto;
|
|
} *hdr, _hdr;
|
|
|
|
hdr = skb_header_pointer(skb, nhoff, sizeof(_hdr), &_hdr);
|
|
if (!hdr)
|
|
return false;
|
|
/*
|
|
* Only look inside GRE if version zero and no
|
|
* routing
|
|
*/
|
|
if (!(hdr->flags & (GRE_VERSION|GRE_ROUTING))) {
|
|
proto = hdr->proto;
|
|
nhoff += 4;
|
|
if (hdr->flags & GRE_CSUM)
|
|
nhoff += 4;
|
|
if (hdr->flags & GRE_KEY)
|
|
nhoff += 4;
|
|
if (hdr->flags & GRE_SEQ)
|
|
nhoff += 4;
|
|
if (proto == htons(ETH_P_TEB)) {
|
|
const struct ethhdr *eth;
|
|
struct ethhdr _eth;
|
|
|
|
eth = skb_header_pointer(skb, nhoff,
|
|
sizeof(_eth), &_eth);
|
|
if (!eth)
|
|
return false;
|
|
proto = eth->h_proto;
|
|
nhoff += sizeof(*eth);
|
|
}
|
|
goto again;
|
|
}
|
|
break;
|
|
}
|
|
case IPPROTO_IPIP:
|
|
goto again;
|
|
default:
|
|
break;
|
|
}
|
|
|
|
flow->ip_proto = ip_proto;
|
|
poff = proto_ports_offset(ip_proto);
|
|
if (poff >= 0) {
|
|
__be32 *ports, _ports;
|
|
|
|
nhoff += poff;
|
|
ports = skb_header_pointer(skb, nhoff, sizeof(_ports), &_ports);
|
|
if (ports)
|
|
flow->ports = *ports;
|
|
}
|
|
|
|
flow->thoff = (u16) nhoff;
|
|
|
|
return true;
|
|
}
|
|
EXPORT_SYMBOL(skb_flow_dissect);
|
|
|
|
static u32 hashrnd __read_mostly;
|
|
|
|
/*
|
|
* __skb_get_rxhash: calculate a flow hash based on src/dst addresses
|
|
* and src/dst port numbers. Sets rxhash in skb to non-zero hash value
|
|
* on success, zero indicates no valid hash. Also, sets l4_rxhash in skb
|
|
* if hash is a canonical 4-tuple hash over transport ports.
|
|
*/
|
|
void __skb_get_rxhash(struct sk_buff *skb)
|
|
{
|
|
struct flow_keys keys;
|
|
u32 hash;
|
|
|
|
if (!skb_flow_dissect(skb, &keys))
|
|
return;
|
|
|
|
if (keys.ports)
|
|
skb->l4_rxhash = 1;
|
|
|
|
/* get a consistent hash (same value on both flow directions) */
|
|
if (((__force u32)keys.dst < (__force u32)keys.src) ||
|
|
(((__force u32)keys.dst == (__force u32)keys.src) &&
|
|
((__force u16)keys.port16[1] < (__force u16)keys.port16[0]))) {
|
|
swap(keys.dst, keys.src);
|
|
swap(keys.port16[0], keys.port16[1]);
|
|
}
|
|
|
|
hash = jhash_3words((__force u32)keys.dst,
|
|
(__force u32)keys.src,
|
|
(__force u32)keys.ports, hashrnd);
|
|
if (!hash)
|
|
hash = 1;
|
|
|
|
skb->rxhash = hash;
|
|
}
|
|
EXPORT_SYMBOL(__skb_get_rxhash);
|
|
|
|
/*
|
|
* Returns a Tx hash based on the given packet descriptor a Tx queues' number
|
|
* to be used as a distribution range.
|
|
*/
|
|
u16 __skb_tx_hash(const struct net_device *dev, const struct sk_buff *skb,
|
|
unsigned int num_tx_queues)
|
|
{
|
|
u32 hash;
|
|
u16 qoffset = 0;
|
|
u16 qcount = num_tx_queues;
|
|
|
|
if (skb_rx_queue_recorded(skb)) {
|
|
hash = skb_get_rx_queue(skb);
|
|
while (unlikely(hash >= num_tx_queues))
|
|
hash -= num_tx_queues;
|
|
return hash;
|
|
}
|
|
|
|
if (dev->num_tc) {
|
|
u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
|
|
qoffset = dev->tc_to_txq[tc].offset;
|
|
qcount = dev->tc_to_txq[tc].count;
|
|
}
|
|
|
|
if (skb->sk && skb->sk->sk_hash)
|
|
hash = skb->sk->sk_hash;
|
|
else
|
|
hash = (__force u16) skb->protocol;
|
|
hash = jhash_1word(hash, hashrnd);
|
|
|
|
return (u16) (((u64) hash * qcount) >> 32) + qoffset;
|
|
}
|
|
EXPORT_SYMBOL(__skb_tx_hash);
|
|
|
|
/* __skb_get_poff() returns the offset to the payload as far as it could
|
|
* be dissected. The main user is currently BPF, so that we can dynamically
|
|
* truncate packets without needing to push actual payload to the user
|
|
* space and can analyze headers only, instead.
|
|
*/
|
|
u32 __skb_get_poff(const struct sk_buff *skb)
|
|
{
|
|
struct flow_keys keys;
|
|
u32 poff = 0;
|
|
|
|
if (!skb_flow_dissect(skb, &keys))
|
|
return 0;
|
|
|
|
poff += keys.thoff;
|
|
switch (keys.ip_proto) {
|
|
case IPPROTO_TCP: {
|
|
const struct tcphdr *tcph;
|
|
struct tcphdr _tcph;
|
|
|
|
tcph = skb_header_pointer(skb, poff, sizeof(_tcph), &_tcph);
|
|
if (!tcph)
|
|
return poff;
|
|
|
|
poff += max_t(u32, sizeof(struct tcphdr), tcph->doff * 4);
|
|
break;
|
|
}
|
|
case IPPROTO_UDP:
|
|
case IPPROTO_UDPLITE:
|
|
poff += sizeof(struct udphdr);
|
|
break;
|
|
/* For the rest, we do not really care about header
|
|
* extensions at this point for now.
|
|
*/
|
|
case IPPROTO_ICMP:
|
|
poff += sizeof(struct icmphdr);
|
|
break;
|
|
case IPPROTO_ICMPV6:
|
|
poff += sizeof(struct icmp6hdr);
|
|
break;
|
|
case IPPROTO_IGMP:
|
|
poff += sizeof(struct igmphdr);
|
|
break;
|
|
case IPPROTO_DCCP:
|
|
poff += sizeof(struct dccp_hdr);
|
|
break;
|
|
case IPPROTO_SCTP:
|
|
poff += sizeof(struct sctphdr);
|
|
break;
|
|
}
|
|
|
|
return poff;
|
|
}
|
|
|
|
static inline u16 dev_cap_txqueue(struct net_device *dev, u16 queue_index)
|
|
{
|
|
if (unlikely(queue_index >= dev->real_num_tx_queues)) {
|
|
net_warn_ratelimited("%s selects TX queue %d, but real number of TX queues is %d\n",
|
|
dev->name, queue_index,
|
|
dev->real_num_tx_queues);
|
|
return 0;
|
|
}
|
|
return queue_index;
|
|
}
|
|
|
|
static inline int get_xps_queue(struct net_device *dev, struct sk_buff *skb)
|
|
{
|
|
#ifdef CONFIG_XPS
|
|
struct xps_dev_maps *dev_maps;
|
|
struct xps_map *map;
|
|
int queue_index = -1;
|
|
|
|
rcu_read_lock();
|
|
dev_maps = rcu_dereference(dev->xps_maps);
|
|
if (dev_maps) {
|
|
map = rcu_dereference(
|
|
dev_maps->cpu_map[raw_smp_processor_id()]);
|
|
if (map) {
|
|
if (map->len == 1)
|
|
queue_index = map->queues[0];
|
|
else {
|
|
u32 hash;
|
|
if (skb->sk && skb->sk->sk_hash)
|
|
hash = skb->sk->sk_hash;
|
|
else
|
|
hash = (__force u16) skb->protocol ^
|
|
skb->rxhash;
|
|
hash = jhash_1word(hash, hashrnd);
|
|
queue_index = map->queues[
|
|
((u64)hash * map->len) >> 32];
|
|
}
|
|
if (unlikely(queue_index >= dev->real_num_tx_queues))
|
|
queue_index = -1;
|
|
}
|
|
}
|
|
rcu_read_unlock();
|
|
|
|
return queue_index;
|
|
#else
|
|
return -1;
|
|
#endif
|
|
}
|
|
|
|
u16 __netdev_pick_tx(struct net_device *dev, struct sk_buff *skb)
|
|
{
|
|
struct sock *sk = skb->sk;
|
|
int queue_index = sk_tx_queue_get(sk);
|
|
|
|
if (queue_index < 0 || skb->ooo_okay ||
|
|
queue_index >= dev->real_num_tx_queues) {
|
|
int new_index = get_xps_queue(dev, skb);
|
|
if (new_index < 0)
|
|
new_index = skb_tx_hash(dev, skb);
|
|
|
|
if (queue_index != new_index && sk) {
|
|
struct dst_entry *dst =
|
|
rcu_dereference_check(sk->sk_dst_cache, 1);
|
|
|
|
if (dst && skb_dst(skb) == dst)
|
|
sk_tx_queue_set(sk, queue_index);
|
|
|
|
}
|
|
|
|
queue_index = new_index;
|
|
}
|
|
|
|
return queue_index;
|
|
}
|
|
EXPORT_SYMBOL(__netdev_pick_tx);
|
|
|
|
struct netdev_queue *netdev_pick_tx(struct net_device *dev,
|
|
struct sk_buff *skb)
|
|
{
|
|
int queue_index = 0;
|
|
|
|
if (dev->real_num_tx_queues != 1) {
|
|
const struct net_device_ops *ops = dev->netdev_ops;
|
|
if (ops->ndo_select_queue)
|
|
queue_index = ops->ndo_select_queue(dev, skb);
|
|
else
|
|
queue_index = __netdev_pick_tx(dev, skb);
|
|
queue_index = dev_cap_txqueue(dev, queue_index);
|
|
}
|
|
|
|
skb_set_queue_mapping(skb, queue_index);
|
|
return netdev_get_tx_queue(dev, queue_index);
|
|
}
|
|
|
|
static int __init initialize_hashrnd(void)
|
|
{
|
|
get_random_bytes(&hashrnd, sizeof(hashrnd));
|
|
return 0;
|
|
}
|
|
|
|
late_initcall_sync(initialize_hashrnd);
|