2019-11-11 23:29:56 +00:00
|
|
|
#include <linux/kernel.h>
|
|
|
|
#include <linux/init.h>
|
|
|
|
#include <linux/module.h>
|
|
|
|
#include <linux/netfilter.h>
|
|
|
|
#include <linux/rhashtable.h>
|
|
|
|
#include <linux/netdevice.h>
|
|
|
|
#include <linux/tc_act/tc_csum.h>
|
|
|
|
#include <net/flow_offload.h>
|
|
|
|
#include <net/netfilter/nf_flow_table.h>
|
2020-02-24 05:22:53 +00:00
|
|
|
#include <net/netfilter/nf_tables.h>
|
2019-11-11 23:29:56 +00:00
|
|
|
#include <net/netfilter/nf_conntrack.h>
|
2020-03-28 00:57:54 +00:00
|
|
|
#include <net/netfilter/nf_conntrack_acct.h>
|
2019-11-11 23:29:56 +00:00
|
|
|
#include <net/netfilter/nf_conntrack_core.h>
|
|
|
|
#include <net/netfilter/nf_conntrack_tuple.h>
|
|
|
|
|
2021-03-03 12:59:53 +00:00
|
|
|
static struct workqueue_struct *nf_flow_offload_add_wq;
|
|
|
|
static struct workqueue_struct *nf_flow_offload_del_wq;
|
|
|
|
static struct workqueue_struct *nf_flow_offload_stats_wq;
|
2019-11-11 23:29:56 +00:00
|
|
|
|
|
|
|
struct flow_offload_work {
|
|
|
|
struct list_head list;
|
|
|
|
enum flow_cls_command cmd;
|
|
|
|
struct nf_flowtable *flowtable;
|
|
|
|
struct flow_offload *flow;
|
2020-03-27 09:12:30 +00:00
|
|
|
struct work_struct work;
|
2019-11-11 23:29:56 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
#define NF_FLOW_DISSECTOR(__match, __type, __field) \
|
|
|
|
(__match)->dissector.offset[__type] = \
|
|
|
|
offsetof(struct nf_flow_key, __field)
|
|
|
|
|
2020-02-24 04:22:54 +00:00
|
|
|
static void nf_flow_rule_lwt_match(struct nf_flow_match *match,
|
|
|
|
struct ip_tunnel_info *tun_info)
|
|
|
|
{
|
|
|
|
struct nf_flow_key *mask = &match->mask;
|
|
|
|
struct nf_flow_key *key = &match->key;
|
|
|
|
unsigned int enc_keys;
|
|
|
|
|
|
|
|
if (!tun_info || !(tun_info->mode & IP_TUNNEL_INFO_TX))
|
|
|
|
return;
|
|
|
|
|
|
|
|
NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_ENC_CONTROL, enc_control);
|
|
|
|
NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_ENC_KEYID, enc_key_id);
|
|
|
|
key->enc_key_id.keyid = tunnel_id_to_key32(tun_info->key.tun_id);
|
|
|
|
mask->enc_key_id.keyid = 0xffffffff;
|
|
|
|
enc_keys = BIT(FLOW_DISSECTOR_KEY_ENC_KEYID) |
|
|
|
|
BIT(FLOW_DISSECTOR_KEY_ENC_CONTROL);
|
|
|
|
|
|
|
|
if (ip_tunnel_info_af(tun_info) == AF_INET) {
|
|
|
|
NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS,
|
|
|
|
enc_ipv4);
|
|
|
|
key->enc_ipv4.src = tun_info->key.u.ipv4.dst;
|
|
|
|
key->enc_ipv4.dst = tun_info->key.u.ipv4.src;
|
|
|
|
if (key->enc_ipv4.src)
|
|
|
|
mask->enc_ipv4.src = 0xffffffff;
|
|
|
|
if (key->enc_ipv4.dst)
|
|
|
|
mask->enc_ipv4.dst = 0xffffffff;
|
|
|
|
enc_keys |= BIT(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS);
|
|
|
|
key->enc_control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
|
|
|
|
} else {
|
|
|
|
memcpy(&key->enc_ipv6.src, &tun_info->key.u.ipv6.dst,
|
|
|
|
sizeof(struct in6_addr));
|
|
|
|
memcpy(&key->enc_ipv6.dst, &tun_info->key.u.ipv6.src,
|
|
|
|
sizeof(struct in6_addr));
|
|
|
|
if (memcmp(&key->enc_ipv6.src, &in6addr_any,
|
|
|
|
sizeof(struct in6_addr)))
|
2021-11-07 01:28:21 +00:00
|
|
|
memset(&mask->enc_ipv6.src, 0xff,
|
2020-02-24 04:22:54 +00:00
|
|
|
sizeof(struct in6_addr));
|
|
|
|
if (memcmp(&key->enc_ipv6.dst, &in6addr_any,
|
|
|
|
sizeof(struct in6_addr)))
|
2021-11-07 01:28:21 +00:00
|
|
|
memset(&mask->enc_ipv6.dst, 0xff,
|
2020-02-24 04:22:54 +00:00
|
|
|
sizeof(struct in6_addr));
|
|
|
|
enc_keys |= BIT(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS);
|
|
|
|
key->enc_control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
|
|
|
|
}
|
|
|
|
|
|
|
|
match->dissector.used_keys |= enc_keys;
|
|
|
|
}
|
|
|
|
|
2021-04-03 13:59:42 +00:00
|
|
|
static void nf_flow_rule_vlan_match(struct flow_dissector_key_vlan *key,
|
|
|
|
struct flow_dissector_key_vlan *mask,
|
|
|
|
u16 vlan_id, __be16 proto)
|
|
|
|
{
|
|
|
|
key->vlan_id = vlan_id;
|
|
|
|
mask->vlan_id = VLAN_VID_MASK;
|
|
|
|
key->vlan_tpid = proto;
|
|
|
|
mask->vlan_tpid = 0xffff;
|
|
|
|
}
|
|
|
|
|
2019-11-11 23:29:56 +00:00
|
|
|
static int nf_flow_rule_match(struct nf_flow_match *match,
|
2020-02-24 04:22:54 +00:00
|
|
|
const struct flow_offload_tuple *tuple,
|
|
|
|
struct dst_entry *other_dst)
|
2019-11-11 23:29:56 +00:00
|
|
|
{
|
|
|
|
struct nf_flow_key *mask = &match->mask;
|
|
|
|
struct nf_flow_key *key = &match->key;
|
2020-02-24 04:22:54 +00:00
|
|
|
struct ip_tunnel_info *tun_info;
|
2021-04-03 13:59:42 +00:00
|
|
|
bool vlan_encap = false;
|
2019-11-11 23:29:56 +00:00
|
|
|
|
2020-01-06 11:42:55 +00:00
|
|
|
NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_META, meta);
|
2019-11-11 23:29:56 +00:00
|
|
|
NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_CONTROL, control);
|
|
|
|
NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_BASIC, basic);
|
|
|
|
NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_IPV4_ADDRS, ipv4);
|
2019-11-29 09:07:01 +00:00
|
|
|
NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_IPV6_ADDRS, ipv6);
|
2019-11-11 23:29:56 +00:00
|
|
|
NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_TCP, tcp);
|
|
|
|
NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_PORTS, tp);
|
|
|
|
|
2020-03-19 04:52:45 +00:00
|
|
|
if (other_dst && other_dst->lwtstate) {
|
2020-02-24 04:22:54 +00:00
|
|
|
tun_info = lwt_tun_info(other_dst->lwtstate);
|
|
|
|
nf_flow_rule_lwt_match(match, tun_info);
|
|
|
|
}
|
|
|
|
|
2022-02-28 09:23:49 +00:00
|
|
|
if (tuple->xmit_type == FLOW_OFFLOAD_XMIT_TC)
|
|
|
|
key->meta.ingress_ifindex = tuple->tc.iifidx;
|
|
|
|
else
|
|
|
|
key->meta.ingress_ifindex = tuple->iifidx;
|
|
|
|
|
2020-01-06 11:42:55 +00:00
|
|
|
mask->meta.ingress_ifindex = 0xffffffff;
|
|
|
|
|
2021-04-03 13:59:42 +00:00
|
|
|
if (tuple->encap_num > 0 && !(tuple->in_vlan_ingress & BIT(0)) &&
|
|
|
|
tuple->encap[0].proto == htons(ETH_P_8021Q)) {
|
|
|
|
NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_VLAN, vlan);
|
|
|
|
nf_flow_rule_vlan_match(&key->vlan, &mask->vlan,
|
|
|
|
tuple->encap[0].id,
|
|
|
|
tuple->encap[0].proto);
|
|
|
|
vlan_encap = true;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (tuple->encap_num > 1 && !(tuple->in_vlan_ingress & BIT(1)) &&
|
|
|
|
tuple->encap[1].proto == htons(ETH_P_8021Q)) {
|
|
|
|
if (vlan_encap) {
|
|
|
|
NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_CVLAN,
|
|
|
|
cvlan);
|
|
|
|
nf_flow_rule_vlan_match(&key->cvlan, &mask->cvlan,
|
|
|
|
tuple->encap[1].id,
|
|
|
|
tuple->encap[1].proto);
|
|
|
|
} else {
|
|
|
|
NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_VLAN,
|
|
|
|
vlan);
|
|
|
|
nf_flow_rule_vlan_match(&key->vlan, &mask->vlan,
|
|
|
|
tuple->encap[1].id,
|
|
|
|
tuple->encap[1].proto);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-11-11 23:29:56 +00:00
|
|
|
switch (tuple->l3proto) {
|
|
|
|
case AF_INET:
|
|
|
|
key->control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
|
|
|
|
key->basic.n_proto = htons(ETH_P_IP);
|
|
|
|
key->ipv4.src = tuple->src_v4.s_addr;
|
|
|
|
mask->ipv4.src = 0xffffffff;
|
|
|
|
key->ipv4.dst = tuple->dst_v4.s_addr;
|
|
|
|
mask->ipv4.dst = 0xffffffff;
|
|
|
|
break;
|
2019-11-29 09:07:01 +00:00
|
|
|
case AF_INET6:
|
|
|
|
key->control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
|
|
|
|
key->basic.n_proto = htons(ETH_P_IPV6);
|
|
|
|
key->ipv6.src = tuple->src_v6;
|
|
|
|
memset(&mask->ipv6.src, 0xff, sizeof(mask->ipv6.src));
|
|
|
|
key->ipv6.dst = tuple->dst_v6;
|
|
|
|
memset(&mask->ipv6.dst, 0xff, sizeof(mask->ipv6.dst));
|
|
|
|
break;
|
2019-11-11 23:29:56 +00:00
|
|
|
default:
|
|
|
|
return -EOPNOTSUPP;
|
|
|
|
}
|
2020-03-19 19:37:21 +00:00
|
|
|
mask->control.addr_type = 0xffff;
|
2019-11-29 09:07:01 +00:00
|
|
|
match->dissector.used_keys |= BIT(key->control.addr_type);
|
2019-11-11 23:29:56 +00:00
|
|
|
mask->basic.n_proto = 0xffff;
|
|
|
|
|
|
|
|
switch (tuple->l4proto) {
|
|
|
|
case IPPROTO_TCP:
|
|
|
|
key->tcp.flags = 0;
|
2019-12-10 20:24:28 +00:00
|
|
|
mask->tcp.flags = cpu_to_be16(be32_to_cpu(TCP_FLAG_RST | TCP_FLAG_FIN) >> 16);
|
2019-11-11 23:29:56 +00:00
|
|
|
match->dissector.used_keys |= BIT(FLOW_DISSECTOR_KEY_TCP);
|
|
|
|
break;
|
|
|
|
case IPPROTO_UDP:
|
2022-02-25 01:53:07 +00:00
|
|
|
case IPPROTO_GRE:
|
2019-11-11 23:29:56 +00:00
|
|
|
break;
|
|
|
|
default:
|
|
|
|
return -EOPNOTSUPP;
|
|
|
|
}
|
|
|
|
|
|
|
|
key->basic.ip_proto = tuple->l4proto;
|
|
|
|
mask->basic.ip_proto = 0xff;
|
|
|
|
|
2020-01-06 11:42:55 +00:00
|
|
|
match->dissector.used_keys |= BIT(FLOW_DISSECTOR_KEY_META) |
|
|
|
|
BIT(FLOW_DISSECTOR_KEY_CONTROL) |
|
2022-02-25 01:53:07 +00:00
|
|
|
BIT(FLOW_DISSECTOR_KEY_BASIC);
|
|
|
|
|
|
|
|
switch (tuple->l4proto) {
|
|
|
|
case IPPROTO_TCP:
|
|
|
|
case IPPROTO_UDP:
|
|
|
|
key->tp.src = tuple->src_port;
|
|
|
|
mask->tp.src = 0xffff;
|
|
|
|
key->tp.dst = tuple->dst_port;
|
|
|
|
mask->tp.dst = 0xffff;
|
|
|
|
|
|
|
|
match->dissector.used_keys |= BIT(FLOW_DISSECTOR_KEY_PORTS);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2019-11-11 23:29:56 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void flow_offload_mangle(struct flow_action_entry *entry,
|
2019-12-07 17:38:12 +00:00
|
|
|
enum flow_action_mangle_base htype, u32 offset,
|
|
|
|
const __be32 *value, const __be32 *mask)
|
2019-11-11 23:29:56 +00:00
|
|
|
{
|
|
|
|
entry->id = FLOW_ACTION_MANGLE;
|
|
|
|
entry->mangle.htype = htype;
|
|
|
|
entry->mangle.offset = offset;
|
|
|
|
memcpy(&entry->mangle.mask, mask, sizeof(u32));
|
|
|
|
memcpy(&entry->mangle.val, value, sizeof(u32));
|
|
|
|
}
|
|
|
|
|
2019-11-13 13:08:00 +00:00
|
|
|
static inline struct flow_action_entry *
|
|
|
|
flow_action_entry_next(struct nf_flow_rule *flow_rule)
|
|
|
|
{
|
|
|
|
int i = flow_rule->rule->action.num_entries++;
|
|
|
|
|
|
|
|
return &flow_rule->rule->action.entries[i];
|
|
|
|
}
|
|
|
|
|
2019-11-11 23:29:56 +00:00
|
|
|
static int flow_offload_eth_src(struct net *net,
|
|
|
|
const struct flow_offload *flow,
|
|
|
|
enum flow_offload_tuple_dir dir,
|
2019-11-13 13:08:00 +00:00
|
|
|
struct nf_flow_rule *flow_rule)
|
2019-11-11 23:29:56 +00:00
|
|
|
{
|
2019-11-13 13:08:00 +00:00
|
|
|
struct flow_action_entry *entry0 = flow_action_entry_next(flow_rule);
|
|
|
|
struct flow_action_entry *entry1 = flow_action_entry_next(flow_rule);
|
2021-03-24 01:30:46 +00:00
|
|
|
const struct flow_offload_tuple *other_tuple, *this_tuple;
|
|
|
|
struct net_device *dev = NULL;
|
|
|
|
const unsigned char *addr;
|
2019-11-11 23:29:56 +00:00
|
|
|
u32 mask, val;
|
|
|
|
u16 val16;
|
|
|
|
|
2021-03-24 01:30:46 +00:00
|
|
|
this_tuple = &flow->tuplehash[dir].tuple;
|
|
|
|
|
|
|
|
switch (this_tuple->xmit_type) {
|
|
|
|
case FLOW_OFFLOAD_XMIT_DIRECT:
|
|
|
|
addr = this_tuple->out.h_source;
|
|
|
|
break;
|
|
|
|
case FLOW_OFFLOAD_XMIT_NEIGH:
|
|
|
|
other_tuple = &flow->tuplehash[!dir].tuple;
|
|
|
|
dev = dev_get_by_index(net, other_tuple->iifidx);
|
|
|
|
if (!dev)
|
|
|
|
return -ENOENT;
|
|
|
|
|
|
|
|
addr = dev->dev_addr;
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
return -EOPNOTSUPP;
|
|
|
|
}
|
2019-11-11 23:29:56 +00:00
|
|
|
|
|
|
|
mask = ~0xffff0000;
|
2021-03-24 01:30:46 +00:00
|
|
|
memcpy(&val16, addr, 2);
|
2019-11-11 23:29:56 +00:00
|
|
|
val = val16 << 16;
|
|
|
|
flow_offload_mangle(entry0, FLOW_ACT_MANGLE_HDR_TYPE_ETH, 4,
|
2019-12-07 17:38:12 +00:00
|
|
|
&val, &mask);
|
2019-11-11 23:29:56 +00:00
|
|
|
|
|
|
|
mask = ~0xffffffff;
|
2021-03-24 01:30:46 +00:00
|
|
|
memcpy(&val, addr + 2, 4);
|
2019-11-11 23:29:56 +00:00
|
|
|
flow_offload_mangle(entry1, FLOW_ACT_MANGLE_HDR_TYPE_ETH, 8,
|
2019-12-07 17:38:12 +00:00
|
|
|
&val, &mask);
|
2021-03-24 01:30:46 +00:00
|
|
|
|
2021-08-05 11:55:27 +00:00
|
|
|
dev_put(dev);
|
2019-11-11 23:29:56 +00:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int flow_offload_eth_dst(struct net *net,
|
|
|
|
const struct flow_offload *flow,
|
|
|
|
enum flow_offload_tuple_dir dir,
|
2019-11-13 13:08:00 +00:00
|
|
|
struct nf_flow_rule *flow_rule)
|
2019-11-11 23:29:56 +00:00
|
|
|
{
|
2019-11-13 13:08:00 +00:00
|
|
|
struct flow_action_entry *entry0 = flow_action_entry_next(flow_rule);
|
|
|
|
struct flow_action_entry *entry1 = flow_action_entry_next(flow_rule);
|
2021-03-24 01:30:46 +00:00
|
|
|
const struct flow_offload_tuple *other_tuple, *this_tuple;
|
2019-12-20 04:14:36 +00:00
|
|
|
const struct dst_entry *dst_cache;
|
2019-12-20 04:14:37 +00:00
|
|
|
unsigned char ha[ETH_ALEN];
|
2019-11-11 23:29:56 +00:00
|
|
|
struct neighbour *n;
|
2021-03-24 01:30:46 +00:00
|
|
|
const void *daddr;
|
2019-11-11 23:29:56 +00:00
|
|
|
u32 mask, val;
|
2019-12-20 04:14:37 +00:00
|
|
|
u8 nud_state;
|
2019-11-11 23:29:56 +00:00
|
|
|
u16 val16;
|
|
|
|
|
2021-03-24 01:30:46 +00:00
|
|
|
this_tuple = &flow->tuplehash[dir].tuple;
|
2019-12-20 04:14:37 +00:00
|
|
|
|
2021-03-24 01:30:46 +00:00
|
|
|
switch (this_tuple->xmit_type) {
|
|
|
|
case FLOW_OFFLOAD_XMIT_DIRECT:
|
|
|
|
ether_addr_copy(ha, this_tuple->out.h_dest);
|
|
|
|
break;
|
|
|
|
case FLOW_OFFLOAD_XMIT_NEIGH:
|
|
|
|
other_tuple = &flow->tuplehash[!dir].tuple;
|
|
|
|
daddr = &other_tuple->src_v4;
|
|
|
|
dst_cache = this_tuple->dst_cache;
|
|
|
|
n = dst_neigh_lookup(dst_cache, daddr);
|
|
|
|
if (!n)
|
|
|
|
return -ENOENT;
|
|
|
|
|
|
|
|
read_lock_bh(&n->lock);
|
|
|
|
nud_state = n->nud_state;
|
|
|
|
ether_addr_copy(ha, n->ha);
|
|
|
|
read_unlock_bh(&n->lock);
|
2019-12-20 04:14:37 +00:00
|
|
|
neigh_release(n);
|
2021-03-24 01:30:46 +00:00
|
|
|
|
|
|
|
if (!(nud_state & NUD_VALID))
|
|
|
|
return -ENOENT;
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
return -EOPNOTSUPP;
|
2019-12-20 04:14:37 +00:00
|
|
|
}
|
|
|
|
|
2019-11-11 23:29:56 +00:00
|
|
|
mask = ~0xffffffff;
|
2019-12-20 04:14:37 +00:00
|
|
|
memcpy(&val, ha, 4);
|
2019-11-11 23:29:56 +00:00
|
|
|
flow_offload_mangle(entry0, FLOW_ACT_MANGLE_HDR_TYPE_ETH, 0,
|
2019-12-07 17:38:12 +00:00
|
|
|
&val, &mask);
|
2019-11-11 23:29:56 +00:00
|
|
|
|
|
|
|
mask = ~0x0000ffff;
|
2019-12-20 04:14:37 +00:00
|
|
|
memcpy(&val16, ha + 4, 2);
|
2019-11-11 23:29:56 +00:00
|
|
|
val = val16;
|
|
|
|
flow_offload_mangle(entry1, FLOW_ACT_MANGLE_HDR_TYPE_ETH, 4,
|
2019-12-07 17:38:12 +00:00
|
|
|
&val, &mask);
|
2019-11-11 23:29:56 +00:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void flow_offload_ipv4_snat(struct net *net,
|
|
|
|
const struct flow_offload *flow,
|
|
|
|
enum flow_offload_tuple_dir dir,
|
2019-11-13 13:08:00 +00:00
|
|
|
struct nf_flow_rule *flow_rule)
|
2019-11-11 23:29:56 +00:00
|
|
|
{
|
2019-11-13 13:08:00 +00:00
|
|
|
struct flow_action_entry *entry = flow_action_entry_next(flow_rule);
|
2019-11-11 23:29:56 +00:00
|
|
|
u32 mask = ~htonl(0xffffffff);
|
|
|
|
__be32 addr;
|
|
|
|
u32 offset;
|
|
|
|
|
|
|
|
switch (dir) {
|
|
|
|
case FLOW_OFFLOAD_DIR_ORIGINAL:
|
|
|
|
addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.dst_v4.s_addr;
|
|
|
|
offset = offsetof(struct iphdr, saddr);
|
|
|
|
break;
|
|
|
|
case FLOW_OFFLOAD_DIR_REPLY:
|
|
|
|
addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.src_v4.s_addr;
|
|
|
|
offset = offsetof(struct iphdr, daddr);
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
flow_offload_mangle(entry, FLOW_ACT_MANGLE_HDR_TYPE_IP4, offset,
|
2019-12-07 17:38:12 +00:00
|
|
|
&addr, &mask);
|
2019-11-11 23:29:56 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void flow_offload_ipv4_dnat(struct net *net,
|
|
|
|
const struct flow_offload *flow,
|
|
|
|
enum flow_offload_tuple_dir dir,
|
2019-11-13 13:08:00 +00:00
|
|
|
struct nf_flow_rule *flow_rule)
|
2019-11-11 23:29:56 +00:00
|
|
|
{
|
2019-11-13 13:08:00 +00:00
|
|
|
struct flow_action_entry *entry = flow_action_entry_next(flow_rule);
|
2019-11-11 23:29:56 +00:00
|
|
|
u32 mask = ~htonl(0xffffffff);
|
|
|
|
__be32 addr;
|
|
|
|
u32 offset;
|
|
|
|
|
|
|
|
switch (dir) {
|
|
|
|
case FLOW_OFFLOAD_DIR_ORIGINAL:
|
|
|
|
addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.src_v4.s_addr;
|
|
|
|
offset = offsetof(struct iphdr, daddr);
|
|
|
|
break;
|
|
|
|
case FLOW_OFFLOAD_DIR_REPLY:
|
|
|
|
addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.dst_v4.s_addr;
|
|
|
|
offset = offsetof(struct iphdr, saddr);
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
flow_offload_mangle(entry, FLOW_ACT_MANGLE_HDR_TYPE_IP4, offset,
|
2019-12-07 17:38:12 +00:00
|
|
|
&addr, &mask);
|
2019-11-11 23:29:56 +00:00
|
|
|
}
|
|
|
|
|
2019-11-13 13:08:01 +00:00
|
|
|
static void flow_offload_ipv6_mangle(struct nf_flow_rule *flow_rule,
|
|
|
|
unsigned int offset,
|
2019-12-07 17:38:12 +00:00
|
|
|
const __be32 *addr, const __be32 *mask)
|
2019-11-13 13:08:01 +00:00
|
|
|
{
|
|
|
|
struct flow_action_entry *entry;
|
2021-03-30 14:24:11 +00:00
|
|
|
int i, j;
|
2019-11-13 13:08:01 +00:00
|
|
|
|
2021-03-30 14:24:11 +00:00
|
|
|
for (i = 0, j = 0; i < sizeof(struct in6_addr) / sizeof(u32); i += sizeof(u32), j++) {
|
2019-11-13 13:08:01 +00:00
|
|
|
entry = flow_action_entry_next(flow_rule);
|
|
|
|
flow_offload_mangle(entry, FLOW_ACT_MANGLE_HDR_TYPE_IP6,
|
2021-03-30 14:24:11 +00:00
|
|
|
offset + i, &addr[j], mask);
|
2019-11-13 13:08:01 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static void flow_offload_ipv6_snat(struct net *net,
|
|
|
|
const struct flow_offload *flow,
|
|
|
|
enum flow_offload_tuple_dir dir,
|
|
|
|
struct nf_flow_rule *flow_rule)
|
|
|
|
{
|
|
|
|
u32 mask = ~htonl(0xffffffff);
|
2019-12-07 17:38:12 +00:00
|
|
|
const __be32 *addr;
|
2019-11-13 13:08:01 +00:00
|
|
|
u32 offset;
|
|
|
|
|
|
|
|
switch (dir) {
|
|
|
|
case FLOW_OFFLOAD_DIR_ORIGINAL:
|
2019-12-07 17:38:12 +00:00
|
|
|
addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.dst_v6.s6_addr32;
|
2019-11-13 13:08:01 +00:00
|
|
|
offset = offsetof(struct ipv6hdr, saddr);
|
|
|
|
break;
|
|
|
|
case FLOW_OFFLOAD_DIR_REPLY:
|
2019-12-07 17:38:12 +00:00
|
|
|
addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.src_v6.s6_addr32;
|
2019-11-13 13:08:01 +00:00
|
|
|
offset = offsetof(struct ipv6hdr, daddr);
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2019-12-07 17:38:12 +00:00
|
|
|
flow_offload_ipv6_mangle(flow_rule, offset, addr, &mask);
|
2019-11-13 13:08:01 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void flow_offload_ipv6_dnat(struct net *net,
|
|
|
|
const struct flow_offload *flow,
|
|
|
|
enum flow_offload_tuple_dir dir,
|
|
|
|
struct nf_flow_rule *flow_rule)
|
|
|
|
{
|
|
|
|
u32 mask = ~htonl(0xffffffff);
|
2019-12-07 17:38:12 +00:00
|
|
|
const __be32 *addr;
|
2019-11-13 13:08:01 +00:00
|
|
|
u32 offset;
|
|
|
|
|
|
|
|
switch (dir) {
|
|
|
|
case FLOW_OFFLOAD_DIR_ORIGINAL:
|
2019-12-07 17:38:12 +00:00
|
|
|
addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.src_v6.s6_addr32;
|
2019-11-13 13:08:01 +00:00
|
|
|
offset = offsetof(struct ipv6hdr, daddr);
|
|
|
|
break;
|
|
|
|
case FLOW_OFFLOAD_DIR_REPLY:
|
2019-12-07 17:38:12 +00:00
|
|
|
addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.dst_v6.s6_addr32;
|
2019-11-13 13:08:01 +00:00
|
|
|
offset = offsetof(struct ipv6hdr, saddr);
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2019-12-07 17:38:12 +00:00
|
|
|
flow_offload_ipv6_mangle(flow_rule, offset, addr, &mask);
|
2019-11-13 13:08:01 +00:00
|
|
|
}
|
|
|
|
|
2019-11-11 23:29:56 +00:00
|
|
|
static int flow_offload_l4proto(const struct flow_offload *flow)
|
|
|
|
{
|
|
|
|
u8 protonum = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.l4proto;
|
|
|
|
u8 type = 0;
|
|
|
|
|
|
|
|
switch (protonum) {
|
|
|
|
case IPPROTO_TCP:
|
|
|
|
type = FLOW_ACT_MANGLE_HDR_TYPE_TCP;
|
|
|
|
break;
|
|
|
|
case IPPROTO_UDP:
|
|
|
|
type = FLOW_ACT_MANGLE_HDR_TYPE_UDP;
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
return type;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void flow_offload_port_snat(struct net *net,
|
|
|
|
const struct flow_offload *flow,
|
|
|
|
enum flow_offload_tuple_dir dir,
|
2019-11-13 13:08:00 +00:00
|
|
|
struct nf_flow_rule *flow_rule)
|
2019-11-11 23:29:56 +00:00
|
|
|
{
|
2019-11-13 13:08:00 +00:00
|
|
|
struct flow_action_entry *entry = flow_action_entry_next(flow_rule);
|
2019-12-20 04:14:38 +00:00
|
|
|
u32 mask, port;
|
2019-11-11 23:29:56 +00:00
|
|
|
u32 offset;
|
|
|
|
|
|
|
|
switch (dir) {
|
|
|
|
case FLOW_OFFLOAD_DIR_ORIGINAL:
|
2019-12-07 17:38:12 +00:00
|
|
|
port = ntohs(flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.dst_port);
|
2019-11-11 23:29:56 +00:00
|
|
|
offset = 0; /* offsetof(struct tcphdr, source); */
|
2019-12-20 04:14:38 +00:00
|
|
|
port = htonl(port << 16);
|
|
|
|
mask = ~htonl(0xffff0000);
|
2019-11-11 23:29:56 +00:00
|
|
|
break;
|
|
|
|
case FLOW_OFFLOAD_DIR_REPLY:
|
2019-12-07 17:38:12 +00:00
|
|
|
port = ntohs(flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.src_port);
|
2019-11-11 23:29:56 +00:00
|
|
|
offset = 0; /* offsetof(struct tcphdr, dest); */
|
2019-12-20 04:14:38 +00:00
|
|
|
port = htonl(port);
|
|
|
|
mask = ~htonl(0xffff);
|
2019-11-11 23:29:56 +00:00
|
|
|
break;
|
|
|
|
default:
|
2019-11-26 20:12:26 +00:00
|
|
|
return;
|
2019-11-11 23:29:56 +00:00
|
|
|
}
|
2019-12-20 04:14:38 +00:00
|
|
|
|
2019-11-11 23:29:56 +00:00
|
|
|
flow_offload_mangle(entry, flow_offload_l4proto(flow), offset,
|
2019-12-07 17:38:12 +00:00
|
|
|
&port, &mask);
|
2019-11-11 23:29:56 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void flow_offload_port_dnat(struct net *net,
|
|
|
|
const struct flow_offload *flow,
|
|
|
|
enum flow_offload_tuple_dir dir,
|
2019-11-13 13:08:00 +00:00
|
|
|
struct nf_flow_rule *flow_rule)
|
2019-11-11 23:29:56 +00:00
|
|
|
{
|
2019-11-13 13:08:00 +00:00
|
|
|
struct flow_action_entry *entry = flow_action_entry_next(flow_rule);
|
2019-12-20 04:14:38 +00:00
|
|
|
u32 mask, port;
|
2019-11-11 23:29:56 +00:00
|
|
|
u32 offset;
|
|
|
|
|
|
|
|
switch (dir) {
|
|
|
|
case FLOW_OFFLOAD_DIR_ORIGINAL:
|
2019-12-20 04:14:38 +00:00
|
|
|
port = ntohs(flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.src_port);
|
|
|
|
offset = 0; /* offsetof(struct tcphdr, dest); */
|
|
|
|
port = htonl(port);
|
|
|
|
mask = ~htonl(0xffff);
|
2019-11-11 23:29:56 +00:00
|
|
|
break;
|
|
|
|
case FLOW_OFFLOAD_DIR_REPLY:
|
2019-12-20 04:14:38 +00:00
|
|
|
port = ntohs(flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.dst_port);
|
|
|
|
offset = 0; /* offsetof(struct tcphdr, source); */
|
|
|
|
port = htonl(port << 16);
|
|
|
|
mask = ~htonl(0xffff0000);
|
2019-11-11 23:29:56 +00:00
|
|
|
break;
|
|
|
|
default:
|
2019-11-26 20:12:26 +00:00
|
|
|
return;
|
2019-11-11 23:29:56 +00:00
|
|
|
}
|
2019-12-20 04:14:38 +00:00
|
|
|
|
2019-11-11 23:29:56 +00:00
|
|
|
flow_offload_mangle(entry, flow_offload_l4proto(flow), offset,
|
2019-12-07 17:38:12 +00:00
|
|
|
&port, &mask);
|
2019-11-11 23:29:56 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void flow_offload_ipv4_checksum(struct net *net,
|
|
|
|
const struct flow_offload *flow,
|
2019-11-13 13:08:00 +00:00
|
|
|
struct nf_flow_rule *flow_rule)
|
2019-11-11 23:29:56 +00:00
|
|
|
{
|
|
|
|
u8 protonum = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.l4proto;
|
2019-11-13 13:08:00 +00:00
|
|
|
struct flow_action_entry *entry = flow_action_entry_next(flow_rule);
|
2019-11-11 23:29:56 +00:00
|
|
|
|
|
|
|
entry->id = FLOW_ACTION_CSUM;
|
|
|
|
entry->csum_flags = TCA_CSUM_UPDATE_FLAG_IPV4HDR;
|
|
|
|
|
|
|
|
switch (protonum) {
|
|
|
|
case IPPROTO_TCP:
|
|
|
|
entry->csum_flags |= TCA_CSUM_UPDATE_FLAG_TCP;
|
|
|
|
break;
|
|
|
|
case IPPROTO_UDP:
|
|
|
|
entry->csum_flags |= TCA_CSUM_UPDATE_FLAG_UDP;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-03-24 01:30:46 +00:00
|
|
|
static void flow_offload_redirect(struct net *net,
|
|
|
|
const struct flow_offload *flow,
|
2019-11-11 23:29:56 +00:00
|
|
|
enum flow_offload_tuple_dir dir,
|
2019-11-13 13:08:00 +00:00
|
|
|
struct nf_flow_rule *flow_rule)
|
2019-11-11 23:29:56 +00:00
|
|
|
{
|
2021-03-24 01:30:46 +00:00
|
|
|
const struct flow_offload_tuple *this_tuple, *other_tuple;
|
|
|
|
struct flow_action_entry *entry;
|
|
|
|
struct net_device *dev;
|
|
|
|
int ifindex;
|
|
|
|
|
|
|
|
this_tuple = &flow->tuplehash[dir].tuple;
|
|
|
|
switch (this_tuple->xmit_type) {
|
|
|
|
case FLOW_OFFLOAD_XMIT_DIRECT:
|
|
|
|
this_tuple = &flow->tuplehash[dir].tuple;
|
2021-03-24 01:30:47 +00:00
|
|
|
ifindex = this_tuple->out.hw_ifidx;
|
2021-03-24 01:30:46 +00:00
|
|
|
break;
|
|
|
|
case FLOW_OFFLOAD_XMIT_NEIGH:
|
|
|
|
other_tuple = &flow->tuplehash[!dir].tuple;
|
|
|
|
ifindex = other_tuple->iifidx;
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
dev = dev_get_by_index(net, ifindex);
|
|
|
|
if (!dev)
|
|
|
|
return;
|
2019-11-11 23:29:56 +00:00
|
|
|
|
2021-03-24 01:30:46 +00:00
|
|
|
entry = flow_action_entry_next(flow_rule);
|
2019-11-11 23:29:56 +00:00
|
|
|
entry->id = FLOW_ACTION_REDIRECT;
|
2021-03-24 01:30:46 +00:00
|
|
|
entry->dev = dev;
|
2019-11-11 23:29:56 +00:00
|
|
|
}
|
|
|
|
|
2020-02-24 05:22:55 +00:00
|
|
|
static void flow_offload_encap_tunnel(const struct flow_offload *flow,
|
|
|
|
enum flow_offload_tuple_dir dir,
|
|
|
|
struct nf_flow_rule *flow_rule)
|
|
|
|
{
|
2021-03-24 01:30:46 +00:00
|
|
|
const struct flow_offload_tuple *this_tuple;
|
2020-02-24 05:22:55 +00:00
|
|
|
struct flow_action_entry *entry;
|
|
|
|
struct dst_entry *dst;
|
|
|
|
|
2021-03-24 01:30:46 +00:00
|
|
|
this_tuple = &flow->tuplehash[dir].tuple;
|
|
|
|
if (this_tuple->xmit_type == FLOW_OFFLOAD_XMIT_DIRECT)
|
|
|
|
return;
|
|
|
|
|
|
|
|
dst = this_tuple->dst_cache;
|
2020-03-19 04:52:45 +00:00
|
|
|
if (dst && dst->lwtstate) {
|
2020-02-24 05:22:55 +00:00
|
|
|
struct ip_tunnel_info *tun_info;
|
|
|
|
|
|
|
|
tun_info = lwt_tun_info(dst->lwtstate);
|
|
|
|
if (tun_info && (tun_info->mode & IP_TUNNEL_INFO_TX)) {
|
|
|
|
entry = flow_action_entry_next(flow_rule);
|
|
|
|
entry->id = FLOW_ACTION_TUNNEL_ENCAP;
|
|
|
|
entry->tunnel = tun_info;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static void flow_offload_decap_tunnel(const struct flow_offload *flow,
|
|
|
|
enum flow_offload_tuple_dir dir,
|
|
|
|
struct nf_flow_rule *flow_rule)
|
|
|
|
{
|
2021-03-24 01:30:46 +00:00
|
|
|
const struct flow_offload_tuple *other_tuple;
|
2020-02-24 05:22:55 +00:00
|
|
|
struct flow_action_entry *entry;
|
|
|
|
struct dst_entry *dst;
|
|
|
|
|
2021-03-24 01:30:46 +00:00
|
|
|
other_tuple = &flow->tuplehash[!dir].tuple;
|
|
|
|
if (other_tuple->xmit_type == FLOW_OFFLOAD_XMIT_DIRECT)
|
|
|
|
return;
|
|
|
|
|
|
|
|
dst = other_tuple->dst_cache;
|
2020-03-19 04:52:45 +00:00
|
|
|
if (dst && dst->lwtstate) {
|
2020-02-24 05:22:55 +00:00
|
|
|
struct ip_tunnel_info *tun_info;
|
|
|
|
|
|
|
|
tun_info = lwt_tun_info(dst->lwtstate);
|
|
|
|
if (tun_info && (tun_info->mode & IP_TUNNEL_INFO_TX)) {
|
|
|
|
entry = flow_action_entry_next(flow_rule);
|
|
|
|
entry->id = FLOW_ACTION_TUNNEL_DECAP;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-03-24 01:30:46 +00:00
|
|
|
static int
|
|
|
|
nf_flow_rule_route_common(struct net *net, const struct flow_offload *flow,
|
|
|
|
enum flow_offload_tuple_dir dir,
|
|
|
|
struct nf_flow_rule *flow_rule)
|
2019-11-11 23:29:56 +00:00
|
|
|
{
|
2021-03-24 01:30:46 +00:00
|
|
|
const struct flow_offload_tuple *other_tuple;
|
2021-04-03 13:59:43 +00:00
|
|
|
const struct flow_offload_tuple *tuple;
|
2021-03-24 01:30:46 +00:00
|
|
|
int i;
|
|
|
|
|
2020-02-24 05:22:55 +00:00
|
|
|
flow_offload_decap_tunnel(flow, dir, flow_rule);
|
|
|
|
flow_offload_encap_tunnel(flow, dir, flow_rule);
|
|
|
|
|
2019-11-13 13:08:00 +00:00
|
|
|
if (flow_offload_eth_src(net, flow, dir, flow_rule) < 0 ||
|
|
|
|
flow_offload_eth_dst(net, flow, dir, flow_rule) < 0)
|
2019-11-11 23:29:56 +00:00
|
|
|
return -1;
|
|
|
|
|
2021-04-03 13:59:43 +00:00
|
|
|
tuple = &flow->tuplehash[dir].tuple;
|
|
|
|
|
|
|
|
for (i = 0; i < tuple->encap_num; i++) {
|
|
|
|
struct flow_action_entry *entry;
|
|
|
|
|
|
|
|
if (tuple->in_vlan_ingress & BIT(i))
|
|
|
|
continue;
|
|
|
|
|
|
|
|
if (tuple->encap[i].proto == htons(ETH_P_8021Q)) {
|
|
|
|
entry = flow_action_entry_next(flow_rule);
|
|
|
|
entry->id = FLOW_ACTION_VLAN_POP;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-03-24 01:30:46 +00:00
|
|
|
other_tuple = &flow->tuplehash[!dir].tuple;
|
|
|
|
|
|
|
|
for (i = 0; i < other_tuple->encap_num; i++) {
|
2021-03-24 01:30:48 +00:00
|
|
|
struct flow_action_entry *entry;
|
2021-03-24 01:30:46 +00:00
|
|
|
|
2021-03-24 01:30:48 +00:00
|
|
|
if (other_tuple->in_vlan_ingress & BIT(i))
|
|
|
|
continue;
|
|
|
|
|
|
|
|
entry = flow_action_entry_next(flow_rule);
|
2021-03-24 01:30:50 +00:00
|
|
|
|
|
|
|
switch (other_tuple->encap[i].proto) {
|
|
|
|
case htons(ETH_P_PPP_SES):
|
|
|
|
entry->id = FLOW_ACTION_PPPOE_PUSH;
|
|
|
|
entry->pppoe.sid = other_tuple->encap[i].id;
|
|
|
|
break;
|
|
|
|
case htons(ETH_P_8021Q):
|
|
|
|
entry->id = FLOW_ACTION_VLAN_PUSH;
|
|
|
|
entry->vlan.vid = other_tuple->encap[i].id;
|
|
|
|
entry->vlan.proto = other_tuple->encap[i].proto;
|
|
|
|
break;
|
|
|
|
}
|
2021-03-24 01:30:46 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
int nf_flow_rule_route_ipv4(struct net *net, const struct flow_offload *flow,
|
|
|
|
enum flow_offload_tuple_dir dir,
|
|
|
|
struct nf_flow_rule *flow_rule)
|
|
|
|
{
|
|
|
|
if (nf_flow_rule_route_common(net, flow, dir, flow_rule) < 0)
|
|
|
|
return -1;
|
|
|
|
|
2020-01-05 19:41:15 +00:00
|
|
|
if (test_bit(NF_FLOW_SNAT, &flow->flags)) {
|
2019-11-13 13:08:00 +00:00
|
|
|
flow_offload_ipv4_snat(net, flow, dir, flow_rule);
|
|
|
|
flow_offload_port_snat(net, flow, dir, flow_rule);
|
2019-11-11 23:29:56 +00:00
|
|
|
}
|
2020-01-05 19:41:15 +00:00
|
|
|
if (test_bit(NF_FLOW_DNAT, &flow->flags)) {
|
2019-11-13 13:08:00 +00:00
|
|
|
flow_offload_ipv4_dnat(net, flow, dir, flow_rule);
|
|
|
|
flow_offload_port_dnat(net, flow, dir, flow_rule);
|
2019-11-11 23:29:56 +00:00
|
|
|
}
|
2020-01-05 19:41:15 +00:00
|
|
|
if (test_bit(NF_FLOW_SNAT, &flow->flags) ||
|
|
|
|
test_bit(NF_FLOW_DNAT, &flow->flags))
|
2019-11-13 13:08:00 +00:00
|
|
|
flow_offload_ipv4_checksum(net, flow, flow_rule);
|
2019-11-11 23:29:56 +00:00
|
|
|
|
2021-03-24 01:30:46 +00:00
|
|
|
flow_offload_redirect(net, flow, dir, flow_rule);
|
2019-11-11 23:29:56 +00:00
|
|
|
|
2019-11-13 13:08:00 +00:00
|
|
|
return 0;
|
2019-11-11 23:29:56 +00:00
|
|
|
}
|
2019-11-13 13:08:01 +00:00
|
|
|
EXPORT_SYMBOL_GPL(nf_flow_rule_route_ipv4);
|
|
|
|
|
|
|
|
int nf_flow_rule_route_ipv6(struct net *net, const struct flow_offload *flow,
|
|
|
|
enum flow_offload_tuple_dir dir,
|
|
|
|
struct nf_flow_rule *flow_rule)
|
|
|
|
{
|
2021-03-24 01:30:46 +00:00
|
|
|
if (nf_flow_rule_route_common(net, flow, dir, flow_rule) < 0)
|
2019-11-13 13:08:01 +00:00
|
|
|
return -1;
|
|
|
|
|
2020-01-05 19:41:15 +00:00
|
|
|
if (test_bit(NF_FLOW_SNAT, &flow->flags)) {
|
2019-11-13 13:08:01 +00:00
|
|
|
flow_offload_ipv6_snat(net, flow, dir, flow_rule);
|
|
|
|
flow_offload_port_snat(net, flow, dir, flow_rule);
|
|
|
|
}
|
2020-01-05 19:41:15 +00:00
|
|
|
if (test_bit(NF_FLOW_DNAT, &flow->flags)) {
|
2019-11-13 13:08:01 +00:00
|
|
|
flow_offload_ipv6_dnat(net, flow, dir, flow_rule);
|
|
|
|
flow_offload_port_dnat(net, flow, dir, flow_rule);
|
|
|
|
}
|
|
|
|
|
2021-03-24 01:30:46 +00:00
|
|
|
flow_offload_redirect(net, flow, dir, flow_rule);
|
2019-11-13 13:08:01 +00:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(nf_flow_rule_route_ipv6);
|
|
|
|
|
|
|
|
#define NF_FLOW_RULE_ACTION_MAX 16
|
2019-11-11 23:29:56 +00:00
|
|
|
|
|
|
|
static struct nf_flow_rule *
|
|
|
|
nf_flow_offload_rule_alloc(struct net *net,
|
|
|
|
const struct flow_offload_work *offload,
|
|
|
|
enum flow_offload_tuple_dir dir)
|
|
|
|
{
|
|
|
|
const struct nf_flowtable *flowtable = offload->flowtable;
|
2021-03-24 01:30:46 +00:00
|
|
|
const struct flow_offload_tuple *tuple, *other_tuple;
|
2019-11-11 23:29:56 +00:00
|
|
|
const struct flow_offload *flow = offload->flow;
|
2021-03-24 01:30:46 +00:00
|
|
|
struct dst_entry *other_dst = NULL;
|
2019-11-11 23:29:56 +00:00
|
|
|
struct nf_flow_rule *flow_rule;
|
2019-11-13 13:08:00 +00:00
|
|
|
int err = -ENOMEM;
|
2019-11-11 23:29:56 +00:00
|
|
|
|
|
|
|
flow_rule = kzalloc(sizeof(*flow_rule), GFP_KERNEL);
|
|
|
|
if (!flow_rule)
|
|
|
|
goto err_flow;
|
|
|
|
|
2019-11-13 13:08:01 +00:00
|
|
|
flow_rule->rule = flow_rule_alloc(NF_FLOW_RULE_ACTION_MAX);
|
2019-11-11 23:29:56 +00:00
|
|
|
if (!flow_rule->rule)
|
|
|
|
goto err_flow_rule;
|
|
|
|
|
|
|
|
flow_rule->rule->match.dissector = &flow_rule->match.dissector;
|
|
|
|
flow_rule->rule->match.mask = &flow_rule->match.mask;
|
|
|
|
flow_rule->rule->match.key = &flow_rule->match.key;
|
|
|
|
|
|
|
|
tuple = &flow->tuplehash[dir].tuple;
|
2021-03-24 01:30:46 +00:00
|
|
|
other_tuple = &flow->tuplehash[!dir].tuple;
|
|
|
|
if (other_tuple->xmit_type == FLOW_OFFLOAD_XMIT_NEIGH)
|
|
|
|
other_dst = other_tuple->dst_cache;
|
|
|
|
|
2020-02-24 04:22:54 +00:00
|
|
|
err = nf_flow_rule_match(&flow_rule->match, tuple, other_dst);
|
2019-11-11 23:29:56 +00:00
|
|
|
if (err < 0)
|
|
|
|
goto err_flow_match;
|
|
|
|
|
2019-11-13 13:08:00 +00:00
|
|
|
flow_rule->rule->action.num_entries = 0;
|
|
|
|
if (flowtable->type->action(net, flow, dir, flow_rule) < 0)
|
2019-11-11 23:29:56 +00:00
|
|
|
goto err_flow_match;
|
|
|
|
|
|
|
|
return flow_rule;
|
|
|
|
|
|
|
|
err_flow_match:
|
|
|
|
kfree(flow_rule->rule);
|
|
|
|
err_flow_rule:
|
|
|
|
kfree(flow_rule);
|
|
|
|
err_flow:
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void __nf_flow_offload_destroy(struct nf_flow_rule *flow_rule)
|
|
|
|
{
|
|
|
|
struct flow_action_entry *entry;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = 0; i < flow_rule->rule->action.num_entries; i++) {
|
|
|
|
entry = &flow_rule->rule->action.entries[i];
|
|
|
|
if (entry->id != FLOW_ACTION_REDIRECT)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
dev_put(entry->dev);
|
|
|
|
}
|
|
|
|
kfree(flow_rule->rule);
|
|
|
|
kfree(flow_rule);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void nf_flow_offload_destroy(struct nf_flow_rule *flow_rule[])
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = 0; i < FLOW_OFFLOAD_DIR_MAX; i++)
|
|
|
|
__nf_flow_offload_destroy(flow_rule[i]);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int nf_flow_offload_alloc(const struct flow_offload_work *offload,
|
|
|
|
struct nf_flow_rule *flow_rule[])
|
|
|
|
{
|
|
|
|
struct net *net = read_pnet(&offload->flowtable->net);
|
|
|
|
|
|
|
|
flow_rule[0] = nf_flow_offload_rule_alloc(net, offload,
|
|
|
|
FLOW_OFFLOAD_DIR_ORIGINAL);
|
|
|
|
if (!flow_rule[0])
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
flow_rule[1] = nf_flow_offload_rule_alloc(net, offload,
|
|
|
|
FLOW_OFFLOAD_DIR_REPLY);
|
|
|
|
if (!flow_rule[1]) {
|
|
|
|
__nf_flow_offload_destroy(flow_rule[0]);
|
|
|
|
return -ENOMEM;
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void nf_flow_offload_init(struct flow_cls_offload *cls_flow,
|
|
|
|
__be16 proto, int priority,
|
|
|
|
enum flow_cls_command cmd,
|
|
|
|
const struct flow_offload_tuple *tuple,
|
|
|
|
struct netlink_ext_ack *extack)
|
|
|
|
{
|
|
|
|
cls_flow->common.protocol = proto;
|
|
|
|
cls_flow->common.prio = priority;
|
|
|
|
cls_flow->common.extack = extack;
|
|
|
|
cls_flow->command = cmd;
|
|
|
|
cls_flow->cookie = (unsigned long)tuple;
|
|
|
|
}
|
|
|
|
|
2020-01-13 18:02:14 +00:00
|
|
|
static int nf_flow_offload_tuple(struct nf_flowtable *flowtable,
|
|
|
|
struct flow_offload *flow,
|
|
|
|
struct nf_flow_rule *flow_rule,
|
|
|
|
enum flow_offload_tuple_dir dir,
|
|
|
|
int priority, int cmd,
|
2020-01-30 16:15:18 +00:00
|
|
|
struct flow_stats *stats,
|
2020-01-13 18:02:14 +00:00
|
|
|
struct list_head *block_cb_list)
|
2019-11-11 23:29:56 +00:00
|
|
|
{
|
|
|
|
struct flow_cls_offload cls_flow = {};
|
|
|
|
struct flow_block_cb *block_cb;
|
|
|
|
struct netlink_ext_ack extack;
|
|
|
|
__be16 proto = ETH_P_ALL;
|
|
|
|
int err, i = 0;
|
|
|
|
|
2020-01-13 18:02:14 +00:00
|
|
|
nf_flow_offload_init(&cls_flow, proto, priority, cmd,
|
|
|
|
&flow->tuplehash[dir].tuple, &extack);
|
|
|
|
if (cmd == FLOW_CLS_REPLACE)
|
|
|
|
cls_flow.rule = flow_rule->rule;
|
2019-11-11 23:29:56 +00:00
|
|
|
|
2020-03-27 09:12:29 +00:00
|
|
|
down_read(&flowtable->flow_block_lock);
|
2020-01-13 18:02:14 +00:00
|
|
|
list_for_each_entry(block_cb, block_cb_list, list) {
|
2019-11-20 05:12:22 +00:00
|
|
|
err = block_cb->cb(TC_SETUP_CLSFLOWER, &cls_flow,
|
2019-11-11 23:29:56 +00:00
|
|
|
block_cb->cb_priv);
|
|
|
|
if (err < 0)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
i++;
|
|
|
|
}
|
2020-03-27 09:12:29 +00:00
|
|
|
up_read(&flowtable->flow_block_lock);
|
2019-11-11 23:29:56 +00:00
|
|
|
|
2020-01-30 16:15:18 +00:00
|
|
|
if (cmd == FLOW_CLS_STATS)
|
|
|
|
memcpy(stats, &cls_flow.stats, sizeof(*stats));
|
|
|
|
|
2019-11-11 23:29:56 +00:00
|
|
|
return i;
|
|
|
|
}
|
|
|
|
|
2020-01-13 18:02:14 +00:00
|
|
|
static int flow_offload_tuple_add(struct flow_offload_work *offload,
|
|
|
|
struct nf_flow_rule *flow_rule,
|
|
|
|
enum flow_offload_tuple_dir dir)
|
|
|
|
{
|
|
|
|
return nf_flow_offload_tuple(offload->flowtable, offload->flow,
|
2022-03-18 12:11:23 +00:00
|
|
|
flow_rule, dir,
|
|
|
|
offload->flowtable->priority,
|
2020-01-30 16:15:18 +00:00
|
|
|
FLOW_CLS_REPLACE, NULL,
|
2020-01-13 18:02:14 +00:00
|
|
|
&offload->flowtable->flow_block.cb_list);
|
|
|
|
}
|
|
|
|
|
2019-11-11 23:29:56 +00:00
|
|
|
static void flow_offload_tuple_del(struct flow_offload_work *offload,
|
|
|
|
enum flow_offload_tuple_dir dir)
|
|
|
|
{
|
2020-01-13 18:02:14 +00:00
|
|
|
nf_flow_offload_tuple(offload->flowtable, offload->flow, NULL, dir,
|
2022-03-18 12:11:23 +00:00
|
|
|
offload->flowtable->priority,
|
|
|
|
FLOW_CLS_DESTROY, NULL,
|
2020-01-13 18:02:14 +00:00
|
|
|
&offload->flowtable->flow_block.cb_list);
|
2019-11-11 23:29:56 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static int flow_offload_rule_add(struct flow_offload_work *offload,
|
|
|
|
struct nf_flow_rule *flow_rule[])
|
|
|
|
{
|
|
|
|
int ok_count = 0;
|
|
|
|
|
|
|
|
ok_count += flow_offload_tuple_add(offload, flow_rule[0],
|
|
|
|
FLOW_OFFLOAD_DIR_ORIGINAL);
|
|
|
|
ok_count += flow_offload_tuple_add(offload, flow_rule[1],
|
|
|
|
FLOW_OFFLOAD_DIR_REPLY);
|
|
|
|
if (ok_count == 0)
|
|
|
|
return -ENOENT;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2020-01-06 11:56:47 +00:00
|
|
|
static void flow_offload_work_add(struct flow_offload_work *offload)
|
2019-11-11 23:29:56 +00:00
|
|
|
{
|
|
|
|
struct nf_flow_rule *flow_rule[FLOW_OFFLOAD_DIR_MAX];
|
|
|
|
int err;
|
|
|
|
|
|
|
|
err = nf_flow_offload_alloc(offload, flow_rule);
|
|
|
|
if (err < 0)
|
2020-01-06 11:56:47 +00:00
|
|
|
return;
|
2019-11-11 23:29:56 +00:00
|
|
|
|
|
|
|
err = flow_offload_rule_add(offload, flow_rule);
|
2020-01-06 11:56:47 +00:00
|
|
|
if (err < 0)
|
2021-05-10 11:50:24 +00:00
|
|
|
goto out;
|
2019-11-11 23:29:56 +00:00
|
|
|
|
2021-05-10 11:50:24 +00:00
|
|
|
set_bit(IPS_HW_OFFLOAD_BIT, &offload->flow->ct->status);
|
|
|
|
|
|
|
|
out:
|
2019-11-11 23:29:56 +00:00
|
|
|
nf_flow_offload_destroy(flow_rule);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void flow_offload_work_del(struct flow_offload_work *offload)
|
|
|
|
{
|
2020-04-21 15:04:16 +00:00
|
|
|
clear_bit(IPS_HW_OFFLOAD_BIT, &offload->flow->ct->status);
|
2019-11-11 23:29:56 +00:00
|
|
|
flow_offload_tuple_del(offload, FLOW_OFFLOAD_DIR_ORIGINAL);
|
|
|
|
flow_offload_tuple_del(offload, FLOW_OFFLOAD_DIR_REPLY);
|
2020-01-30 16:04:37 +00:00
|
|
|
set_bit(NF_FLOW_HW_DEAD, &offload->flow->flags);
|
2019-11-11 23:29:56 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void flow_offload_tuple_stats(struct flow_offload_work *offload,
|
|
|
|
enum flow_offload_tuple_dir dir,
|
|
|
|
struct flow_stats *stats)
|
|
|
|
{
|
2020-01-30 16:15:18 +00:00
|
|
|
nf_flow_offload_tuple(offload->flowtable, offload->flow, NULL, dir,
|
2022-03-18 12:11:23 +00:00
|
|
|
offload->flowtable->priority,
|
|
|
|
FLOW_CLS_STATS, stats,
|
2020-01-30 16:15:18 +00:00
|
|
|
&offload->flowtable->flow_block.cb_list);
|
2019-11-11 23:29:56 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void flow_offload_work_stats(struct flow_offload_work *offload)
|
|
|
|
{
|
|
|
|
struct flow_stats stats[FLOW_OFFLOAD_DIR_MAX] = {};
|
|
|
|
u64 lastused;
|
|
|
|
|
|
|
|
flow_offload_tuple_stats(offload, FLOW_OFFLOAD_DIR_ORIGINAL, &stats[0]);
|
|
|
|
flow_offload_tuple_stats(offload, FLOW_OFFLOAD_DIR_REPLY, &stats[1]);
|
|
|
|
|
|
|
|
lastused = max_t(u64, stats[0].lastused, stats[1].lastused);
|
|
|
|
offload->flow->timeout = max_t(u64, offload->flow->timeout,
|
2021-06-03 12:12:35 +00:00
|
|
|
lastused + flow_offload_get_timeout(offload->flow));
|
2020-03-28 00:57:54 +00:00
|
|
|
|
|
|
|
if (offload->flowtable->flags & NF_FLOWTABLE_COUNTER) {
|
|
|
|
if (stats[0].pkts)
|
|
|
|
nf_ct_acct_add(offload->flow->ct,
|
|
|
|
FLOW_OFFLOAD_DIR_ORIGINAL,
|
|
|
|
stats[0].pkts, stats[0].bytes);
|
|
|
|
if (stats[1].pkts)
|
|
|
|
nf_ct_acct_add(offload->flow->ct,
|
|
|
|
FLOW_OFFLOAD_DIR_REPLY,
|
|
|
|
stats[1].pkts, stats[1].bytes);
|
|
|
|
}
|
2019-11-11 23:29:56 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void flow_offload_work_handler(struct work_struct *work)
|
|
|
|
{
|
2020-03-27 09:12:30 +00:00
|
|
|
struct flow_offload_work *offload;
|
2022-06-15 10:43:55 +00:00
|
|
|
struct net *net;
|
2019-11-11 23:29:56 +00:00
|
|
|
|
2020-03-27 09:12:30 +00:00
|
|
|
offload = container_of(work, struct flow_offload_work, work);
|
2022-06-15 10:43:55 +00:00
|
|
|
net = read_pnet(&offload->flowtable->net);
|
2020-03-27 09:12:30 +00:00
|
|
|
switch (offload->cmd) {
|
2019-11-11 23:29:56 +00:00
|
|
|
case FLOW_CLS_REPLACE:
|
2020-01-06 11:56:47 +00:00
|
|
|
flow_offload_work_add(offload);
|
2022-06-15 10:43:55 +00:00
|
|
|
NF_FLOW_TABLE_STAT_DEC_ATOMIC(net, count_wq_add);
|
2019-11-11 23:29:56 +00:00
|
|
|
break;
|
|
|
|
case FLOW_CLS_DESTROY:
|
|
|
|
flow_offload_work_del(offload);
|
2022-06-15 10:43:55 +00:00
|
|
|
NF_FLOW_TABLE_STAT_DEC_ATOMIC(net, count_wq_del);
|
2019-11-11 23:29:56 +00:00
|
|
|
break;
|
|
|
|
case FLOW_CLS_STATS:
|
|
|
|
flow_offload_work_stats(offload);
|
2022-06-15 10:43:55 +00:00
|
|
|
NF_FLOW_TABLE_STAT_DEC_ATOMIC(net, count_wq_stats);
|
2019-11-11 23:29:56 +00:00
|
|
|
break;
|
|
|
|
default:
|
|
|
|
WARN_ON_ONCE(1);
|
|
|
|
}
|
2020-03-27 09:12:30 +00:00
|
|
|
|
2020-05-06 11:24:39 +00:00
|
|
|
clear_bit(NF_FLOW_HW_PENDING, &offload->flow->flags);
|
2020-03-27 09:12:30 +00:00
|
|
|
kfree(offload);
|
2019-11-11 23:29:56 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void flow_offload_queue_work(struct flow_offload_work *offload)
|
|
|
|
{
|
2022-06-15 10:43:55 +00:00
|
|
|
struct net *net = read_pnet(&offload->flowtable->net);
|
|
|
|
|
|
|
|
if (offload->cmd == FLOW_CLS_REPLACE) {
|
2022-11-24 17:54:10 +00:00
|
|
|
NF_FLOW_TABLE_STAT_INC_ATOMIC(net, count_wq_add);
|
2021-03-03 12:59:53 +00:00
|
|
|
queue_work(nf_flow_offload_add_wq, &offload->work);
|
2022-06-15 10:43:55 +00:00
|
|
|
} else if (offload->cmd == FLOW_CLS_DESTROY) {
|
2022-11-24 17:54:10 +00:00
|
|
|
NF_FLOW_TABLE_STAT_INC_ATOMIC(net, count_wq_del);
|
2021-03-03 12:59:53 +00:00
|
|
|
queue_work(nf_flow_offload_del_wq, &offload->work);
|
2022-06-15 10:43:55 +00:00
|
|
|
} else {
|
2022-11-24 17:54:10 +00:00
|
|
|
NF_FLOW_TABLE_STAT_INC_ATOMIC(net, count_wq_stats);
|
2021-03-03 12:59:53 +00:00
|
|
|
queue_work(nf_flow_offload_stats_wq, &offload->work);
|
2022-06-15 10:43:55 +00:00
|
|
|
}
|
2019-11-11 23:29:56 +00:00
|
|
|
}
|
|
|
|
|
2020-01-05 19:48:51 +00:00
|
|
|
static struct flow_offload_work *
|
|
|
|
nf_flow_offload_work_alloc(struct nf_flowtable *flowtable,
|
|
|
|
struct flow_offload *flow, unsigned int cmd)
|
2019-11-11 23:29:56 +00:00
|
|
|
{
|
|
|
|
struct flow_offload_work *offload;
|
|
|
|
|
2020-05-06 11:24:39 +00:00
|
|
|
if (test_and_set_bit(NF_FLOW_HW_PENDING, &flow->flags))
|
|
|
|
return NULL;
|
|
|
|
|
2019-11-11 23:29:56 +00:00
|
|
|
offload = kmalloc(sizeof(struct flow_offload_work), GFP_ATOMIC);
|
2020-05-06 11:24:39 +00:00
|
|
|
if (!offload) {
|
|
|
|
clear_bit(NF_FLOW_HW_PENDING, &flow->flags);
|
2020-01-05 19:48:51 +00:00
|
|
|
return NULL;
|
2020-05-06 11:24:39 +00:00
|
|
|
}
|
2019-11-11 23:29:56 +00:00
|
|
|
|
2020-01-05 19:48:51 +00:00
|
|
|
offload->cmd = cmd;
|
2019-11-11 23:29:56 +00:00
|
|
|
offload->flow = flow;
|
|
|
|
offload->flowtable = flowtable;
|
2020-03-27 09:12:30 +00:00
|
|
|
INIT_WORK(&offload->work, flow_offload_work_handler);
|
2019-11-11 23:29:56 +00:00
|
|
|
|
2020-01-05 19:48:51 +00:00
|
|
|
return offload;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
void nf_flow_offload_add(struct nf_flowtable *flowtable,
|
|
|
|
struct flow_offload *flow)
|
|
|
|
{
|
|
|
|
struct flow_offload_work *offload;
|
|
|
|
|
|
|
|
offload = nf_flow_offload_work_alloc(flowtable, flow, FLOW_CLS_REPLACE);
|
|
|
|
if (!offload)
|
|
|
|
return;
|
|
|
|
|
2019-11-11 23:29:56 +00:00
|
|
|
flow_offload_queue_work(offload);
|
|
|
|
}
|
|
|
|
|
|
|
|
void nf_flow_offload_del(struct nf_flowtable *flowtable,
|
|
|
|
struct flow_offload *flow)
|
|
|
|
{
|
|
|
|
struct flow_offload_work *offload;
|
|
|
|
|
2020-01-05 19:48:51 +00:00
|
|
|
offload = nf_flow_offload_work_alloc(flowtable, flow, FLOW_CLS_DESTROY);
|
2019-11-11 23:29:56 +00:00
|
|
|
if (!offload)
|
|
|
|
return;
|
|
|
|
|
2020-01-05 19:41:15 +00:00
|
|
|
set_bit(NF_FLOW_HW_DYING, &flow->flags);
|
2019-11-11 23:29:56 +00:00
|
|
|
flow_offload_queue_work(offload);
|
|
|
|
}
|
|
|
|
|
|
|
|
void nf_flow_offload_stats(struct nf_flowtable *flowtable,
|
|
|
|
struct flow_offload *flow)
|
|
|
|
{
|
|
|
|
struct flow_offload_work *offload;
|
2020-01-03 17:10:04 +00:00
|
|
|
__s32 delta;
|
2019-11-11 23:29:56 +00:00
|
|
|
|
2020-01-03 17:10:04 +00:00
|
|
|
delta = nf_flow_timeout_delta(flow->timeout);
|
2021-06-03 12:12:35 +00:00
|
|
|
if ((delta >= (9 * flow_offload_get_timeout(flow)) / 10))
|
2019-11-11 23:29:56 +00:00
|
|
|
return;
|
|
|
|
|
2020-01-05 19:48:51 +00:00
|
|
|
offload = nf_flow_offload_work_alloc(flowtable, flow, FLOW_CLS_STATS);
|
2019-11-11 23:29:56 +00:00
|
|
|
if (!offload)
|
|
|
|
return;
|
|
|
|
|
|
|
|
flow_offload_queue_work(offload);
|
|
|
|
}
|
|
|
|
|
2021-11-18 21:24:15 +00:00
|
|
|
void nf_flow_table_offload_flush_cleanup(struct nf_flowtable *flowtable)
|
|
|
|
{
|
|
|
|
if (nf_flowtable_hw_offload(flowtable)) {
|
|
|
|
flush_workqueue(nf_flow_offload_del_wq);
|
|
|
|
nf_flow_table_gc_run(flowtable);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-11-11 23:29:56 +00:00
|
|
|
void nf_flow_table_offload_flush(struct nf_flowtable *flowtable)
|
|
|
|
{
|
2021-03-03 12:59:53 +00:00
|
|
|
if (nf_flowtable_hw_offload(flowtable)) {
|
|
|
|
flush_workqueue(nf_flow_offload_add_wq);
|
|
|
|
flush_workqueue(nf_flow_offload_del_wq);
|
|
|
|
flush_workqueue(nf_flow_offload_stats_wq);
|
|
|
|
}
|
2019-11-11 23:29:56 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static int nf_flow_table_block_setup(struct nf_flowtable *flowtable,
|
|
|
|
struct flow_block_offload *bo,
|
|
|
|
enum flow_block_command cmd)
|
|
|
|
{
|
|
|
|
struct flow_block_cb *block_cb, *next;
|
|
|
|
int err = 0;
|
|
|
|
|
2022-11-21 18:26:15 +00:00
|
|
|
down_write(&flowtable->flow_block_lock);
|
2019-11-11 23:29:56 +00:00
|
|
|
switch (cmd) {
|
|
|
|
case FLOW_BLOCK_BIND:
|
|
|
|
list_splice(&bo->cb_list, &flowtable->flow_block.cb_list);
|
|
|
|
break;
|
|
|
|
case FLOW_BLOCK_UNBIND:
|
|
|
|
list_for_each_entry_safe(block_cb, next, &bo->cb_list, list) {
|
|
|
|
list_del(&block_cb->list);
|
|
|
|
flow_block_cb_free(block_cb);
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
WARN_ON_ONCE(1);
|
|
|
|
err = -EOPNOTSUPP;
|
|
|
|
}
|
2022-11-21 18:26:15 +00:00
|
|
|
up_write(&flowtable->flow_block_lock);
|
2019-11-11 23:29:56 +00:00
|
|
|
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
2020-02-24 05:22:52 +00:00
|
|
|
static void nf_flow_table_block_offload_init(struct flow_block_offload *bo,
|
|
|
|
struct net *net,
|
|
|
|
enum flow_block_command cmd,
|
|
|
|
struct nf_flowtable *flowtable,
|
|
|
|
struct netlink_ext_ack *extack)
|
|
|
|
{
|
|
|
|
memset(bo, 0, sizeof(*bo));
|
|
|
|
bo->net = net;
|
|
|
|
bo->block = &flowtable->flow_block;
|
|
|
|
bo->command = cmd;
|
|
|
|
bo->binder_type = FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS;
|
|
|
|
bo->extack = extack;
|
2021-08-17 17:05:18 +00:00
|
|
|
bo->cb_list_head = &flowtable->flow_block.cb_list;
|
2020-02-24 05:22:52 +00:00
|
|
|
INIT_LIST_HEAD(&bo->cb_list);
|
|
|
|
}
|
|
|
|
|
2020-05-29 00:25:37 +00:00
|
|
|
static void nf_flow_table_indr_cleanup(struct flow_block_cb *block_cb)
|
|
|
|
{
|
|
|
|
struct nf_flowtable *flowtable = block_cb->indr.data;
|
|
|
|
struct net_device *dev = block_cb->indr.dev;
|
|
|
|
|
|
|
|
nf_flow_table_gc_cleanup(flowtable, dev);
|
|
|
|
down_write(&flowtable->flow_block_lock);
|
|
|
|
list_del(&block_cb->list);
|
2020-06-18 12:49:10 +00:00
|
|
|
list_del(&block_cb->driver_list);
|
2020-05-29 00:25:37 +00:00
|
|
|
flow_block_cb_free(block_cb);
|
|
|
|
up_write(&flowtable->flow_block_lock);
|
|
|
|
}
|
|
|
|
|
2020-02-24 05:22:53 +00:00
|
|
|
static int nf_flow_table_indr_offload_cmd(struct flow_block_offload *bo,
|
|
|
|
struct nf_flowtable *flowtable,
|
|
|
|
struct net_device *dev,
|
|
|
|
enum flow_block_command cmd,
|
|
|
|
struct netlink_ext_ack *extack)
|
|
|
|
{
|
|
|
|
nf_flow_table_block_offload_init(bo, dev_net(dev), cmd, flowtable,
|
|
|
|
extack);
|
|
|
|
|
2020-07-10 21:55:03 +00:00
|
|
|
return flow_indr_dev_setup_offload(dev, NULL, TC_SETUP_FT, flowtable, bo,
|
2020-05-29 00:25:37 +00:00
|
|
|
nf_flow_table_indr_cleanup);
|
2020-02-24 05:22:53 +00:00
|
|
|
}
|
|
|
|
|
2020-01-13 18:02:22 +00:00
|
|
|
static int nf_flow_table_offload_cmd(struct flow_block_offload *bo,
|
|
|
|
struct nf_flowtable *flowtable,
|
|
|
|
struct net_device *dev,
|
|
|
|
enum flow_block_command cmd,
|
|
|
|
struct netlink_ext_ack *extack)
|
2019-11-11 23:29:56 +00:00
|
|
|
{
|
|
|
|
int err;
|
|
|
|
|
2020-02-24 05:22:52 +00:00
|
|
|
nf_flow_table_block_offload_init(bo, dev_net(dev), cmd, flowtable,
|
|
|
|
extack);
|
2022-11-21 18:26:15 +00:00
|
|
|
down_write(&flowtable->flow_block_lock);
|
2020-01-13 18:02:22 +00:00
|
|
|
err = dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_FT, bo);
|
2022-11-21 18:26:15 +00:00
|
|
|
up_write(&flowtable->flow_block_lock);
|
2020-01-13 18:02:22 +00:00
|
|
|
if (err < 0)
|
|
|
|
return err;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
int nf_flow_table_offload_setup(struct nf_flowtable *flowtable,
|
|
|
|
struct net_device *dev,
|
|
|
|
enum flow_block_command cmd)
|
|
|
|
{
|
|
|
|
struct netlink_ext_ack extack = {};
|
|
|
|
struct flow_block_offload bo;
|
|
|
|
int err;
|
2019-11-11 23:29:56 +00:00
|
|
|
|
2020-02-03 12:06:18 +00:00
|
|
|
if (!nf_flowtable_hw_offload(flowtable))
|
|
|
|
return 0;
|
|
|
|
|
2020-02-24 05:22:53 +00:00
|
|
|
if (dev->netdev_ops->ndo_setup_tc)
|
|
|
|
err = nf_flow_table_offload_cmd(&bo, flowtable, dev, cmd,
|
|
|
|
&extack);
|
|
|
|
else
|
|
|
|
err = nf_flow_table_indr_offload_cmd(&bo, flowtable, dev, cmd,
|
|
|
|
&extack);
|
2019-11-11 23:29:56 +00:00
|
|
|
if (err < 0)
|
|
|
|
return err;
|
|
|
|
|
|
|
|
return nf_flow_table_block_setup(flowtable, &bo, cmd);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(nf_flow_table_offload_setup);
|
|
|
|
|
|
|
|
int nf_flow_table_offload_init(void)
|
|
|
|
{
|
2021-03-03 12:59:53 +00:00
|
|
|
nf_flow_offload_add_wq = alloc_workqueue("nf_ft_offload_add",
|
|
|
|
WQ_UNBOUND | WQ_SYSFS, 0);
|
|
|
|
if (!nf_flow_offload_add_wq)
|
2020-03-27 09:12:30 +00:00
|
|
|
return -ENOMEM;
|
2019-11-11 23:29:56 +00:00
|
|
|
|
2021-03-03 12:59:53 +00:00
|
|
|
nf_flow_offload_del_wq = alloc_workqueue("nf_ft_offload_del",
|
|
|
|
WQ_UNBOUND | WQ_SYSFS, 0);
|
|
|
|
if (!nf_flow_offload_del_wq)
|
|
|
|
goto err_del_wq;
|
|
|
|
|
|
|
|
nf_flow_offload_stats_wq = alloc_workqueue("nf_ft_offload_stats",
|
|
|
|
WQ_UNBOUND | WQ_SYSFS, 0);
|
|
|
|
if (!nf_flow_offload_stats_wq)
|
|
|
|
goto err_stats_wq;
|
|
|
|
|
2019-11-11 23:29:56 +00:00
|
|
|
return 0;
|
2021-03-03 12:59:53 +00:00
|
|
|
|
|
|
|
err_stats_wq:
|
|
|
|
destroy_workqueue(nf_flow_offload_del_wq);
|
|
|
|
err_del_wq:
|
|
|
|
destroy_workqueue(nf_flow_offload_add_wq);
|
|
|
|
return -ENOMEM;
|
2019-11-11 23:29:56 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void nf_flow_table_offload_exit(void)
|
|
|
|
{
|
2021-03-03 12:59:53 +00:00
|
|
|
destroy_workqueue(nf_flow_offload_add_wq);
|
|
|
|
destroy_workqueue(nf_flow_offload_del_wq);
|
|
|
|
destroy_workqueue(nf_flow_offload_stats_wq);
|
2019-11-11 23:29:56 +00:00
|
|
|
}
|