mirrors
/
linux-stable
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'net-sched-retpoline' 

Pedro Tammela says:

====================
net/sched: retpoline wrappers for tc

In tc all qdics, classifiers and actions can be compiled as modules.
This results today in indirect calls in all transitions in the tc hierarchy.
Due to CONFIG_RETPOLINE, CPUs with mitigations=on might pay an extra cost on
indirect calls. For newer Intel cpus with IBRS the extra cost is
nonexistent, but AMD Zen cpus and older x86 cpus still go through the
retpoline thunk.

Known built-in symbols can be optimized into direct calls, thus
avoiding the retpoline thunk. So far, tc has not been leveraging this
build information and leaving out a performance optimization for some
CPUs. In this series we wire up 'tcf_classify()' and 'tcf_action_exec()'
with direct calls when known modules are compiled as built-in as an
opt-in optimization.

We measured these changes in one AMD Zen 4 cpu (Retpoline), one AMD Zen 3 cpu (Retpoline),
one Intel 10th Gen CPU (IBRS), one Intel 3rd Gen cpu (Retpoline) and one
Intel Xeon CPU (IBRS) using pktgen with 64b udp packets. Our test setup is a
dummy device with clsact and matchall in a kernel compiled with every
tc module as built-in.  We observed a 3-8% speed up on the retpoline CPUs,
when going through 1 tc filter, and a 60-100% speed up when going through 100 filters.
For the IBRS cpus we observed a 1-2% degradation in both scenarios, we believe
the extra branches check introduced a small overhead therefore we added
a static key that bypasses the wrapper on kernels not using the retpoline mitigation,
but compiled with CONFIG_RETPOLINE.

1 filter:
CPU        | before (pps) | after (pps) | diff
R9 7950X   | 5914980      | 6380227     | +7.8%
R9 5950X   | 4237838      | 4412241     | +4.1%
R9 5950X   | 4265287      | 4413757     | +3.4%   [*]
i5-3337U   | 1580565      | 1682406     | +6.4%
i5-10210U  | 3006074      | 3006857     | +0.0%
i5-10210U  | 3160245      | 3179945     | +0.6%   [*]
Xeon 6230R | 3196906      | 3197059     | +0.0%
Xeon 6230R | 3190392      | 3196153     | +0.01%  [*]

100 filters:
CPU        | before (pps) | after (pps) | diff
R9 7950X   | 373598       | 820396      | +119.59%
R9 5950X   | 313469       | 633303      | +102.03%
R9 5950X   | 313797       | 633150      | +101.77% [*]
i5-3337U   | 127454       | 211210      | +65.71%
i5-10210U  | 389259       | 381765      | -1.9%
i5-10210U  | 408812       | 412730      | +0.9%    [*]
Xeon 6230R | 415420       | 406612      | -2.1%
Xeon 6230R | 416705       | 405869      | -2.6%    [*]

[*] In these tests we ran pktgen with clone set to 1000.

On the 7950x system we also tested the impact of filters if iteration order
placement varied, first by compiling a kernel with the filter under test being
the first one in the static iteration and then repeating it with being last (of 15 classifiers existing today).
We saw a difference of +0.5-1% in pps between being the first in the iteration vs being the last.
Therefore we order the classifiers and actions according to relevance per our current thinking.

v5->v6:
- Address Eric Dumazet suggestions

v4->v5:
- Rebase

v3->v4:
- Address Eric Dumazet suggestions

v2->v3:
- Address suggestions by Jakub, Paolo and Eric
- Dropped RFC tag (I forgot to add it on v2)

v1->v2:
- Fix build errors found by the bots
- Address Kuniyuki Iwashima suggestions

====================

Signed-off-by: David S. Miller <davem@davemloft.net>
This commit is contained in:
David S. Miller 2022-12-09 09:18:08 +00:00
parent 0bdff1152c 9f3101dca3
commit b602d00384
38 changed files with 391 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
include/net/act_api.h
View File

@ -101,11 +101,6 @@ static inline enum flow_action_hw_stats tc_act_hw_stats(u8 hw_stats)
return hw_stats;
}
#ifdef CONFIG_NET_CLS_ACT
#define ACT_P_CREATED 1
#define ACT_P_DELETED 1
typedef void (*tc_action_priv_destructor)(void *priv);
struct tc_action_ops {
@ -140,6 +135,11 @@ struct tc_action_ops {
struct netlink_ext_ack *extack);
};
#ifdef CONFIG_NET_CLS_ACT
#define ACT_P_CREATED 1
#define ACT_P_DELETED 1
struct tc_action_net {
struct tcf_idrinfo *idrinfo;
const struct tc_action_ops *ops;

251
include/net/tc_wrapper.h Normal file
View File

@ -0,0 +1,251 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __NET_TC_WRAPPER_H
#define __NET_TC_WRAPPER_H
#include <net/pkt_cls.h>
#if IS_ENABLED(CONFIG_RETPOLINE)
#include <linux/cpufeature.h>
#include <linux/static_key.h>
#include <linux/indirect_call_wrapper.h>
#define TC_INDIRECT_SCOPE
extern struct static_key_false tc_skip_wrapper;
/* TC Actions */
#ifdef CONFIG_NET_CLS_ACT
#define TC_INDIRECT_ACTION_DECLARE(fname) \
INDIRECT_CALLABLE_DECLARE(int fname(struct sk_buff *skb, \
const struct tc_action *a, \
struct tcf_result *res))
TC_INDIRECT_ACTION_DECLARE(tcf_bpf_act);
TC_INDIRECT_ACTION_DECLARE(tcf_connmark_act);
TC_INDIRECT_ACTION_DECLARE(tcf_csum_act);
TC_INDIRECT_ACTION_DECLARE(tcf_ct_act);
TC_INDIRECT_ACTION_DECLARE(tcf_ctinfo_act);
TC_INDIRECT_ACTION_DECLARE(tcf_gact_act);
TC_INDIRECT_ACTION_DECLARE(tcf_gate_act);
TC_INDIRECT_ACTION_DECLARE(tcf_ife_act);
TC_INDIRECT_ACTION_DECLARE(tcf_ipt_act);
TC_INDIRECT_ACTION_DECLARE(tcf_mirred_act);
TC_INDIRECT_ACTION_DECLARE(tcf_mpls_act);
TC_INDIRECT_ACTION_DECLARE(tcf_nat_act);
TC_INDIRECT_ACTION_DECLARE(tcf_pedit_act);
TC_INDIRECT_ACTION_DECLARE(tcf_police_act);
TC_INDIRECT_ACTION_DECLARE(tcf_sample_act);
TC_INDIRECT_ACTION_DECLARE(tcf_simp_act);
TC_INDIRECT_ACTION_DECLARE(tcf_skbedit_act);
TC_INDIRECT_ACTION_DECLARE(tcf_skbmod_act);
TC_INDIRECT_ACTION_DECLARE(tcf_vlan_act);
TC_INDIRECT_ACTION_DECLARE(tunnel_key_act);
static inline int tc_act(struct sk_buff *skb, const struct tc_action *a,
struct tcf_result *res)
{
if (static_branch_likely(&tc_skip_wrapper))
goto skip;
#if IS_BUILTIN(CONFIG_NET_ACT_GACT)
if (a->ops->act == tcf_gact_act)
return tcf_gact_act(skb, a, res);
#endif
#if IS_BUILTIN(CONFIG_NET_ACT_MIRRED)
if (a->ops->act == tcf_mirred_act)
return tcf_mirred_act(skb, a, res);
#endif
#if IS_BUILTIN(CONFIG_NET_ACT_PEDIT)
if (a->ops->act == tcf_pedit_act)
return tcf_pedit_act(skb, a, res);
#endif
#if IS_BUILTIN(CONFIG_NET_ACT_SKBEDIT)
if (a->ops->act == tcf_skbedit_act)
return tcf_skbedit_act(skb, a, res);
#endif
#if IS_BUILTIN(CONFIG_NET_ACT_SKBMOD)
if (a->ops->act == tcf_skbmod_act)
return tcf_skbmod_act(skb, a, res);
#endif
#if IS_BUILTIN(CONFIG_NET_ACT_POLICE)
if (a->ops->act == tcf_police_act)
return tcf_police_act(skb, a, res);
#endif
#if IS_BUILTIN(CONFIG_NET_ACT_BPF)
if (a->ops->act == tcf_bpf_act)
return tcf_bpf_act(skb, a, res);
#endif
#if IS_BUILTIN(CONFIG_NET_ACT_CONNMARK)
if (a->ops->act == tcf_connmark_act)
return tcf_connmark_act(skb, a, res);
#endif
#if IS_BUILTIN(CONFIG_NET_ACT_CSUM)
if (a->ops->act == tcf_csum_act)
return tcf_csum_act(skb, a, res);
#endif
#if IS_BUILTIN(CONFIG_NET_ACT_CT)
if (a->ops->act == tcf_ct_act)
return tcf_ct_act(skb, a, res);
#endif
#if IS_BUILTIN(CONFIG_NET_ACT_CTINFO)
if (a->ops->act == tcf_ctinfo_act)
return tcf_ctinfo_act(skb, a, res);
#endif
#if IS_BUILTIN(CONFIG_NET_ACT_GATE)
if (a->ops->act == tcf_gate_act)
return tcf_gate_act(skb, a, res);
#endif
#if IS_BUILTIN(CONFIG_NET_ACT_MPLS)
if (a->ops->act == tcf_mpls_act)
return tcf_mpls_act(skb, a, res);
#endif
#if IS_BUILTIN(CONFIG_NET_ACT_NAT)
if (a->ops->act == tcf_nat_act)
return tcf_nat_act(skb, a, res);
#endif
#if IS_BUILTIN(CONFIG_NET_ACT_TUNNEL_KEY)
if (a->ops->act == tunnel_key_act)
return tunnel_key_act(skb, a, res);
#endif
#if IS_BUILTIN(CONFIG_NET_ACT_VLAN)
if (a->ops->act == tcf_vlan_act)
return tcf_vlan_act(skb, a, res);
#endif
#if IS_BUILTIN(CONFIG_NET_ACT_IFE)
if (a->ops->act == tcf_ife_act)
return tcf_ife_act(skb, a, res);
#endif
#if IS_BUILTIN(CONFIG_NET_ACT_IPT)
if (a->ops->act == tcf_ipt_act)
return tcf_ipt_act(skb, a, res);
#endif
#if IS_BUILTIN(CONFIG_NET_ACT_SIMP)
if (a->ops->act == tcf_simp_act)
return tcf_simp_act(skb, a, res);
#endif
#if IS_BUILTIN(CONFIG_NET_ACT_SAMPLE)
if (a->ops->act == tcf_sample_act)
return tcf_sample_act(skb, a, res);
#endif
skip:
return a->ops->act(skb, a, res);
}
#endif /* CONFIG_NET_CLS_ACT */
/* TC Filters */
#ifdef CONFIG_NET_CLS
#define TC_INDIRECT_FILTER_DECLARE(fname) \
INDIRECT_CALLABLE_DECLARE(int fname(struct sk_buff *skb, \
const struct tcf_proto *tp, \
struct tcf_result *res))
TC_INDIRECT_FILTER_DECLARE(basic_classify);
TC_INDIRECT_FILTER_DECLARE(cls_bpf_classify);
TC_INDIRECT_FILTER_DECLARE(cls_cgroup_classify);
TC_INDIRECT_FILTER_DECLARE(fl_classify);
TC_INDIRECT_FILTER_DECLARE(flow_classify);
TC_INDIRECT_FILTER_DECLARE(fw_classify);
TC_INDIRECT_FILTER_DECLARE(mall_classify);
TC_INDIRECT_FILTER_DECLARE(route4_classify);
TC_INDIRECT_FILTER_DECLARE(rsvp_classify);
TC_INDIRECT_FILTER_DECLARE(rsvp6_classify);
TC_INDIRECT_FILTER_DECLARE(tcindex_classify);
TC_INDIRECT_FILTER_DECLARE(u32_classify);
static inline int tc_classify(struct sk_buff *skb, const struct tcf_proto *tp,
struct tcf_result *res)
{
if (static_branch_likely(&tc_skip_wrapper))
goto skip;
#if IS_BUILTIN(CONFIG_NET_CLS_BPF)
if (tp->classify == cls_bpf_classify)
return cls_bpf_classify(skb, tp, res);
#endif
#if IS_BUILTIN(CONFIG_NET_CLS_U32)
if (tp->classify == u32_classify)
return u32_classify(skb, tp, res);
#endif
#if IS_BUILTIN(CONFIG_NET_CLS_FLOWER)
if (tp->classify == fl_classify)
return fl_classify(skb, tp, res);
#endif
#if IS_BUILTIN(CONFIG_NET_CLS_FW)
if (tp->classify == fw_classify)
return fw_classify(skb, tp, res);
#endif
#if IS_BUILTIN(CONFIG_NET_CLS_MATCHALL)
if (tp->classify == mall_classify)
return mall_classify(skb, tp, res);
#endif
#if IS_BUILTIN(CONFIG_NET_CLS_BASIC)
if (tp->classify == basic_classify)
return basic_classify(skb, tp, res);
#endif
#if IS_BUILTIN(CONFIG_NET_CLS_CGROUP)
if (tp->classify == cls_cgroup_classify)
return cls_cgroup_classify(skb, tp, res);
#endif
#if IS_BUILTIN(CONFIG_NET_CLS_FLOW)
if (tp->classify == flow_classify)
return flow_classify(skb, tp, res);
#endif
#if IS_BUILTIN(CONFIG_NET_CLS_ROUTE4)
if (tp->classify == route4_classify)
return route4_classify(skb, tp, res);
#endif
#if IS_BUILTIN(CONFIG_NET_CLS_RSVP)
if (tp->classify == rsvp_classify)
return rsvp_classify(skb, tp, res);
#endif
#if IS_BUILTIN(CONFIG_NET_CLS_RSVP6)
if (tp->classify == rsvp6_classify)
return rsvp6_classify(skb, tp, res);
#endif
#if IS_BUILTIN(CONFIG_NET_CLS_TCINDEX)
if (tp->classify == tcindex_classify)
return tcindex_classify(skb, tp, res);
#endif
skip:
return tp->classify(skb, tp, res);
}
static inline void tc_wrapper_init(void)
{
#ifdef CONFIG_X86
if (!cpu_feature_enabled(X86_FEATURE_RETPOLINE))
static_branch_enable(&tc_skip_wrapper);
#endif
}
#endif /* CONFIG_NET_CLS */
#else
#define TC_INDIRECT_SCOPE static
static inline int tc_act(struct sk_buff *skb, const struct tc_action *a,
struct tcf_result *res)
{
return a->ops->act(skb, a, res);
}
static inline int tc_classify(struct sk_buff *skb, const struct tcf_proto *tp,
struct tcf_result *res)
{
return tp->classify(skb, tp, res);
}
static inline void tc_wrapper_init(void)
{
}
#endif
#endif /* __NET_TC_WRAPPER_H */

3
net/sched/act_api.c
View File

@ -23,6 +23,7 @@
#include <net/act_api.h>
#include <net/netlink.h>
#include <net/flow_offload.h>
#include <net/tc_wrapper.h>
#ifdef CONFIG_INET
DEFINE_STATIC_KEY_FALSE(tcf_frag_xmit_count);
@ -1080,7 +1081,7 @@ restart_act_graph:
repeat_ttl = 32;
repeat:
ret = a->ops->act(skb, a, res);
ret = tc_act(skb, a, res);
if (unlikely(ret == TC_ACT_REPEAT)) {
if (--repeat_ttl != 0)
goto repeat;

6
net/sched/act_bpf.c
View File

@ -18,6 +18,7 @@
#include <linux/tc_act/tc_bpf.h>
#include <net/tc_act/tc_bpf.h>
#include <net/tc_wrapper.h>
#define ACT_BPF_NAME_LEN 256
@ -31,8 +32,9 @@ struct tcf_bpf_cfg {
static struct tc_action_ops act_bpf_ops;
static int tcf_bpf_act(struct sk_buff *skb, const struct tc_action *act,
struct tcf_result *res)
TC_INDIRECT_SCOPE int tcf_bpf_act(struct sk_buff *skb,
const struct tc_action *act,
struct tcf_result *res)
{
bool at_ingress = skb_at_tc_ingress(skb);
struct tcf_bpf *prog = to_bpf(act);

6
net/sched/act_connmark.c
View File

@ -20,6 +20,7 @@
#include <net/pkt_cls.h>
#include <uapi/linux/tc_act/tc_connmark.h>
#include <net/tc_act/tc_connmark.h>
#include <net/tc_wrapper.h>
#include <net/netfilter/nf_conntrack.h>
#include <net/netfilter/nf_conntrack_core.h>
@ -27,8 +28,9 @@
static struct tc_action_ops act_connmark_ops;
static int tcf_connmark_act(struct sk_buff *skb, const struct tc_action *a,
struct tcf_result *res)
TC_INDIRECT_SCOPE int tcf_connmark_act(struct sk_buff *skb,
const struct tc_action *a,
struct tcf_result *res)
{
const struct nf_conntrack_tuple_hash *thash;
struct nf_conntrack_tuple tuple;

6
net/sched/act_csum.c
View File

@ -32,6 +32,7 @@
#include <linux/tc_act/tc_csum.h>
#include <net/tc_act/tc_csum.h>
#include <net/tc_wrapper.h>
static const struct nla_policy csum_policy[TCA_CSUM_MAX + 1] = {
[TCA_CSUM_PARMS] = { .len = sizeof(struct tc_csum), },
@ -563,8 +564,9 @@ fail:
return 0;
}
static int tcf_csum_act(struct sk_buff *skb, const struct tc_action *a,
struct tcf_result *res)
TC_INDIRECT_SCOPE int tcf_csum_act(struct sk_buff *skb,
const struct tc_action *a,
struct tcf_result *res)
{
struct tcf_csum *p = to_tcf_csum(a);
bool orig_vlan_tag_present = false;

5
net/sched/act_ct.c
View File

@ -24,6 +24,7 @@
#include <net/ipv6_frag.h>
#include <uapi/linux/tc_act/tc_ct.h>
#include <net/tc_act/tc_ct.h>
#include <net/tc_wrapper.h>
#include <net/netfilter/nf_flow_table.h>
#include <net/netfilter/nf_conntrack.h>
@ -1038,8 +1039,8 @@ static int tcf_ct_act_nat(struct sk_buff *skb,
#endif
}
static int tcf_ct_act(struct sk_buff *skb, const struct tc_action *a,
struct tcf_result *res)
TC_INDIRECT_SCOPE int tcf_ct_act(struct sk_buff *skb, const struct tc_action *a,
struct tcf_result *res)
{
struct net *net = dev_net(skb->dev);
enum ip_conntrack_info ctinfo;

6
net/sched/act_ctinfo.c
View File

@ -18,6 +18,7 @@
#include <net/pkt_cls.h>
#include <uapi/linux/tc_act/tc_ctinfo.h>
#include <net/tc_act/tc_ctinfo.h>
#include <net/tc_wrapper.h>
#include <net/netfilter/nf_conntrack.h>
#include <net/netfilter/nf_conntrack_core.h>
@ -75,8 +76,9 @@ static void tcf_ctinfo_cpmark_set(struct nf_conn *ct, struct tcf_ctinfo *ca,
skb->mark = READ_ONCE(ct->mark) & cp->cpmarkmask;
}
static int tcf_ctinfo_act(struct sk_buff *skb, const struct tc_action *a,
struct tcf_result *res)
TC_INDIRECT_SCOPE int tcf_ctinfo_act(struct sk_buff *skb,
const struct tc_action *a,
struct tcf_result *res)
{
const struct nf_conntrack_tuple_hash *thash = NULL;
struct tcf_ctinfo *ca = to_ctinfo(a);

6
net/sched/act_gact.c
View File

@ -18,6 +18,7 @@
#include <net/pkt_cls.h>
#include <linux/tc_act/tc_gact.h>
#include <net/tc_act/tc_gact.h>
#include <net/tc_wrapper.h>
static struct tc_action_ops act_gact_ops;
@ -145,8 +146,9 @@ release_idr:
return err;
}
static int tcf_gact_act(struct sk_buff *skb, const struct tc_action *a,
struct tcf_result *res)
TC_INDIRECT_SCOPE int tcf_gact_act(struct sk_buff *skb,
const struct tc_action *a,
struct tcf_result *res)
{
struct tcf_gact *gact = to_gact(a);
int action = READ_ONCE(gact->tcf_action);

6
net/sched/act_gate.c
View File

@ -14,6 +14,7 @@
#include <net/netlink.h>
#include <net/pkt_cls.h>
#include <net/tc_act/tc_gate.h>
#include <net/tc_wrapper.h>
static struct tc_action_ops act_gate_ops;
@ -113,8 +114,9 @@ static enum hrtimer_restart gate_timer_func(struct hrtimer *timer)
return HRTIMER_RESTART;
}
static int tcf_gate_act(struct sk_buff *skb, const struct tc_action *a,
struct tcf_result *res)
TC_INDIRECT_SCOPE int tcf_gate_act(struct sk_buff *skb,
const struct tc_action *a,
struct tcf_result *res)
{
struct tcf_gate *gact = to_gate(a);

6
net/sched/act_ife.c
View File

@ -29,6 +29,7 @@
#include <net/tc_act/tc_ife.h>
#include <linux/etherdevice.h>
#include <net/ife.h>
#include <net/tc_wrapper.h>
static int max_metacnt = IFE_META_MAX + 1;
static struct tc_action_ops act_ife_ops;
@ -861,8 +862,9 @@ static int tcf_ife_encode(struct sk_buff *skb, const struct tc_action *a,
return action;
}
static int tcf_ife_act(struct sk_buff *skb, const struct tc_action *a,
struct tcf_result *res)
TC_INDIRECT_SCOPE int tcf_ife_act(struct sk_buff *skb,
const struct tc_action *a,
struct tcf_result *res)
{
struct tcf_ife_info *ife = to_ife(a);
struct tcf_ife_params *p;

6
net/sched/act_ipt.c
View File

@ -20,6 +20,7 @@
#include <net/pkt_sched.h>
#include <linux/tc_act/tc_ipt.h>
#include <net/tc_act/tc_ipt.h>
#include <net/tc_wrapper.h>
#include <linux/netfilter_ipv4/ip_tables.h>
@ -216,8 +217,9 @@ static int tcf_xt_init(struct net *net, struct nlattr *nla,
a, &act_xt_ops, tp, flags);
}
static int tcf_ipt_act(struct sk_buff *skb, const struct tc_action *a,
struct tcf_result *res)
TC_INDIRECT_SCOPE int tcf_ipt_act(struct sk_buff *skb,
const struct tc_action *a,
struct tcf_result *res)
{
int ret = 0, result = 0;
struct tcf_ipt *ipt = to_ipt(a);

6
net/sched/act_mirred.c
View File

@ -24,6 +24,7 @@
#include <net/pkt_cls.h>
#include <linux/tc_act/tc_mirred.h>
#include <net/tc_act/tc_mirred.h>
#include <net/tc_wrapper.h>
static LIST_HEAD(mirred_list);
static DEFINE_SPINLOCK(mirred_list_lock);
@ -217,8 +218,9 @@ static int tcf_mirred_forward(bool want_ingress, struct sk_buff *skb)
return err;
}
static int tcf_mirred_act(struct sk_buff *skb, const struct tc_action *a,
struct tcf_result *res)
TC_INDIRECT_SCOPE int tcf_mirred_act(struct sk_buff *skb,
const struct tc_action *a,
struct tcf_result *res)
{
struct tcf_mirred *m = to_mirred(a);
struct sk_buff *skb2 = skb;

6
net/sched/act_mpls.c
View File

@ -14,6 +14,7 @@
#include <net/pkt_sched.h>
#include <net/pkt_cls.h>
#include <net/tc_act/tc_mpls.h>
#include <net/tc_wrapper.h>
static struct tc_action_ops act_mpls_ops;
@ -49,8 +50,9 @@ static __be32 tcf_mpls_get_lse(struct mpls_shim_hdr *lse,
return cpu_to_be32(new_lse);
}
static int tcf_mpls_act(struct sk_buff *skb, const struct tc_action *a,
struct tcf_result *res)
TC_INDIRECT_SCOPE int tcf_mpls_act(struct sk_buff *skb,
const struct tc_action *a,
struct tcf_result *res)
{
struct tcf_mpls *m = to_mpls(a);
struct tcf_mpls_params *p;

7
net/sched/act_nat.c
View File

@ -24,7 +24,7 @@
#include <net/tc_act/tc_nat.h>
#include <net/tcp.h>
#include <net/udp.h>
#include <net/tc_wrapper.h>
static struct tc_action_ops act_nat_ops;
@ -98,8 +98,9 @@ release_idr:
return err;
}
static int tcf_nat_act(struct sk_buff *skb, const struct tc_action *a,
struct tcf_result *res)
TC_INDIRECT_SCOPE int tcf_nat_act(struct sk_buff *skb,
const struct tc_action *a,
struct tcf_result *res)
{
struct tcf_nat *p = to_tcf_nat(a);
struct iphdr *iph;

6
net/sched/act_pedit.c
View File

@ -20,6 +20,7 @@
#include <net/tc_act/tc_pedit.h>
#include <uapi/linux/tc_act/tc_pedit.h>
#include <net/pkt_cls.h>
#include <net/tc_wrapper.h>
static struct tc_action_ops act_pedit_ops;
@ -319,8 +320,9 @@ static int pedit_skb_hdr_offset(struct sk_buff *skb,
return ret;
}
static int tcf_pedit_act(struct sk_buff *skb, const struct tc_action *a,
struct tcf_result *res)
TC_INDIRECT_SCOPE int tcf_pedit_act(struct sk_buff *skb,
const struct tc_action *a,
struct tcf_result *res)
{
struct tcf_pedit *p = to_pedit(a);
u32 max_offset;

6
net/sched/act_police.c
View File

@ -19,6 +19,7 @@
#include <net/netlink.h>
#include <net/pkt_cls.h>
#include <net/tc_act/tc_police.h>
#include <net/tc_wrapper.h>
/* Each policer is serialized by its individual spinlock */
@ -242,8 +243,9 @@ static bool tcf_police_mtu_check(struct sk_buff *skb, u32 limit)
return len <= limit;
}
static int tcf_police_act(struct sk_buff *skb, const struct tc_action *a,
struct tcf_result *res)
TC_INDIRECT_SCOPE int tcf_police_act(struct sk_buff *skb,
const struct tc_action *a,
struct tcf_result *res)
{
struct tcf_police *police = to_police(a);
s64 now, toks, ppstoks = 0, ptoks = 0;

6
net/sched/act_sample.c
View File

@ -20,6 +20,7 @@
#include <net/tc_act/tc_sample.h>
#include <net/psample.h>
#include <net/pkt_cls.h>
#include <net/tc_wrapper.h>
#include <linux/if_arp.h>
@ -153,8 +154,9 @@ static bool tcf_sample_dev_ok_push(struct net_device *dev)
}
}
static int tcf_sample_act(struct sk_buff *skb, const struct tc_action *a,
struct tcf_result *res)
TC_INDIRECT_SCOPE int tcf_sample_act(struct sk_buff *skb,
const struct tc_action *a,
struct tcf_result *res)
{
struct tcf_sample *s = to_sample(a);
struct psample_group *psample_group;

6
net/sched/act_simple.c
View File

@ -14,6 +14,7 @@
#include <net/netlink.h>
#include <net/pkt_sched.h>
#include <net/pkt_cls.h>
#include <net/tc_wrapper.h>
#include <linux/tc_act/tc_defact.h>
#include <net/tc_act/tc_defact.h>
@ -21,8 +22,9 @@
static struct tc_action_ops act_simp_ops;
#define SIMP_MAX_DATA 32
static int tcf_simp_act(struct sk_buff *skb, const struct tc_action *a,
struct tcf_result *res)
TC_INDIRECT_SCOPE int tcf_simp_act(struct sk_buff *skb,
const struct tc_action *a,
struct tcf_result *res)
{
struct tcf_defact *d = to_defact(a);

6
net/sched/act_skbedit.c
View File

@ -16,6 +16,7 @@
#include <net/ipv6.h>
#include <net/dsfield.h>
#include <net/pkt_cls.h>
#include <net/tc_wrapper.h>
#include <linux/tc_act/tc_skbedit.h>
#include <net/tc_act/tc_skbedit.h>
@ -36,8 +37,9 @@ static u16 tcf_skbedit_hash(struct tcf_skbedit_params *params,
return netdev_cap_txqueue(skb->dev, queue_mapping);
}
static int tcf_skbedit_act(struct sk_buff *skb, const struct tc_action *a,
struct tcf_result *res)
TC_INDIRECT_SCOPE int tcf_skbedit_act(struct sk_buff *skb,
const struct tc_action *a,
struct tcf_result *res)
{
struct tcf_skbedit *d = to_skbedit(a);
struct tcf_skbedit_params *params;

6
net/sched/act_skbmod.c
View File

@ -15,14 +15,16 @@
#include <net/netlink.h>
#include <net/pkt_sched.h>
#include <net/pkt_cls.h>
#include <net/tc_wrapper.h>
#include <linux/tc_act/tc_skbmod.h>
#include <net/tc_act/tc_skbmod.h>
static struct tc_action_ops act_skbmod_ops;
static int tcf_skbmod_act(struct sk_buff *skb, const struct tc_action *a,
struct tcf_result *res)
TC_INDIRECT_SCOPE int tcf_skbmod_act(struct sk_buff *skb,
const struct tc_action *a,
struct tcf_result *res)
{
struct tcf_skbmod *d = to_skbmod(a);
int action, max_edit_len, err;

6
net/sched/act_tunnel_key.c
View File

@ -16,14 +16,16 @@
#include <net/pkt_sched.h>
#include <net/dst.h>
#include <net/pkt_cls.h>
#include <net/tc_wrapper.h>
#include <linux/tc_act/tc_tunnel_key.h>
#include <net/tc_act/tc_tunnel_key.h>
static struct tc_action_ops act_tunnel_key_ops;
static int tunnel_key_act(struct sk_buff *skb, const struct tc_action *a,
struct tcf_result *res)
TC_INDIRECT_SCOPE int tunnel_key_act(struct sk_buff *skb,
const struct tc_action *a,
struct tcf_result *res)
{
struct tcf_tunnel_key *t = to_tunnel_key(a);
struct tcf_tunnel_key_params *params;

6
net/sched/act_vlan.c
View File

@ -12,14 +12,16 @@
#include <net/netlink.h>
#include <net/pkt_sched.h>
#include <net/pkt_cls.h>
#include <net/tc_wrapper.h>
#include <linux/tc_act/tc_vlan.h>
#include <net/tc_act/tc_vlan.h>
static struct tc_action_ops act_vlan_ops;
static int tcf_vlan_act(struct sk_buff *skb, const struct tc_action *a,
struct tcf_result *res)
TC_INDIRECT_SCOPE int tcf_vlan_act(struct sk_buff *skb,
const struct tc_action *a,
struct tcf_result *res)
{
struct tcf_vlan *v = to_vlan(a);
struct tcf_vlan_params *p;

3
net/sched/cls_api.c
View File

@ -40,6 +40,7 @@
#include <net/tc_act/tc_mpls.h>
#include <net/tc_act/tc_gate.h>
#include <net/flow_offload.h>
#include <net/tc_wrapper.h>
extern const struct nla_policy rtm_tca_policy[TCA_MAX + 1];
@ -1564,7 +1565,7 @@ reclassify:
tp->protocol != htons(ETH_P_ALL))
continue;
err = tp->classify(skb, tp, res);
err = tc_classify(skb, tp, res);
#ifdef CONFIG_NET_CLS_ACT
if (unlikely(err == TC_ACT_RECLASSIFY && !compat_mode)) {
first_tp = orig_tp;

6
net/sched/cls_basic.c
View File

@ -18,6 +18,7 @@
#include <net/netlink.h>
#include <net/act_api.h>
#include <net/pkt_cls.h>
#include <net/tc_wrapper.h>
struct basic_head {
struct list_head flist;
@ -36,8 +37,9 @@ struct basic_filter {
struct rcu_work rwork;
};
static int basic_classify(struct sk_buff *skb, const struct tcf_proto *tp,
struct tcf_result *res)
TC_INDIRECT_SCOPE int basic_classify(struct sk_buff *skb,
const struct tcf_proto *tp,
struct tcf_result *res)
{
int r;
struct basic_head *head = rcu_dereference_bh(tp->root);

6
net/sched/cls_bpf.c
View File

@ -19,6 +19,7 @@
#include <net/rtnetlink.h>
#include <net/pkt_cls.h>
#include <net/sock.h>
#include <net/tc_wrapper.h>
MODULE_LICENSE("GPL");
MODULE_AUTHOR("Daniel Borkmann <dborkman@redhat.com>");
@ -77,8 +78,9 @@ static int cls_bpf_exec_opcode(int code)
}
}
static int cls_bpf_classify(struct sk_buff *skb, const struct tcf_proto *tp,
struct tcf_result *res)
TC_INDIRECT_SCOPE int cls_bpf_classify(struct sk_buff *skb,
const struct tcf_proto *tp,
struct tcf_result *res)
{
struct cls_bpf_head *head = rcu_dereference_bh(tp->root);
bool at_ingress = skb_at_tc_ingress(skb);

6
net/sched/cls_cgroup.c
View File

@ -13,6 +13,7 @@
#include <net/pkt_cls.h>
#include <net/sock.h>
#include <net/cls_cgroup.h>
#include <net/tc_wrapper.h>
struct cls_cgroup_head {
u32 handle;
@ -22,8 +23,9 @@ struct cls_cgroup_head {
struct rcu_work rwork;
};
static int cls_cgroup_classify(struct sk_buff *skb, const struct tcf_proto *tp,
struct tcf_result *res)
TC_INDIRECT_SCOPE int cls_cgroup_classify(struct sk_buff *skb,
const struct tcf_proto *tp,
struct tcf_result *res)
{
struct cls_cgroup_head *head = rcu_dereference_bh(tp->root);
u32 classid = task_get_classid(skb);

6
net/sched/cls_flow.c
View File

@ -24,6 +24,7 @@
#include <net/ip.h>
#include <net/route.h>
#include <net/flow_dissector.h>
#include <net/tc_wrapper.h>
#if IS_ENABLED(CONFIG_NF_CONNTRACK)
#include <net/netfilter/nf_conntrack.h>
@ -292,8 +293,9 @@ static u32 flow_key_get(struct sk_buff *skb, int key, struct flow_keys *flow)
(1 << FLOW_KEY_NFCT_PROTO_SRC) | \
(1 << FLOW_KEY_NFCT_PROTO_DST))
static int flow_classify(struct sk_buff *skb, const struct tcf_proto *tp,
struct tcf_result *res)
TC_INDIRECT_SCOPE int flow_classify(struct sk_buff *skb,
const struct tcf_proto *tp,
struct tcf_result *res)
{
struct flow_head *head = rcu_dereference_bh(tp->root);
struct flow_filter *f;

6
net/sched/cls_flower.c
View File

@ -27,6 +27,7 @@
#include <net/vxlan.h>
#include <net/erspan.h>
#include <net/gtp.h>
#include <net/tc_wrapper.h>
#include <net/dst.h>
#include <net/dst_metadata.h>
@ -305,8 +306,9 @@ static u16 fl_ct_info_to_flower_map[] = {
TCA_FLOWER_KEY_CT_FLAGS_NEW,
};
static int fl_classify(struct sk_buff *skb, const struct tcf_proto *tp,
struct tcf_result *res)
TC_INDIRECT_SCOPE int fl_classify(struct sk_buff *skb,
const struct tcf_proto *tp,
struct tcf_result *res)
{
struct cls_fl_head *head = rcu_dereference_bh(tp->root);
bool post_ct = tc_skb_cb(skb)->post_ct;

6
net/sched/cls_fw.c
View File

@ -21,6 +21,7 @@
#include <net/act_api.h>
#include <net/pkt_cls.h>
#include <net/sch_generic.h>
#include <net/tc_wrapper.h>
#define HTSIZE 256
@ -47,8 +48,9 @@ static u32 fw_hash(u32 handle)
return handle % HTSIZE;
}
static int fw_classify(struct sk_buff *skb, const struct tcf_proto *tp,
struct tcf_result *res)
TC_INDIRECT_SCOPE int fw_classify(struct sk_buff *skb,
const struct tcf_proto *tp,
struct tcf_result *res)
{
struct fw_head *head = rcu_dereference_bh(tp->root);
struct fw_filter *f;

6
net/sched/cls_matchall.c
View File

@ -12,6 +12,7 @@
#include <net/sch_generic.h>
#include <net/pkt_cls.h>
#include <net/tc_wrapper.h>
struct cls_mall_head {
struct tcf_exts exts;
@ -24,8 +25,9 @@ struct cls_mall_head {
bool deleting;
};
static int mall_classify(struct sk_buff *skb, const struct tcf_proto *tp,
struct tcf_result *res)
TC_INDIRECT_SCOPE int mall_classify(struct sk_buff *skb,
const struct tcf_proto *tp,
struct tcf_result *res)
{
struct cls_mall_head *head = rcu_dereference_bh(tp->root);

6
net/sched/cls_route.c
View File

@ -17,6 +17,7 @@
#include <net/netlink.h>
#include <net/act_api.h>
#include <net/pkt_cls.h>
#include <net/tc_wrapper.h>
/*
* 1. For now we assume that route tags < 256.
@ -121,8 +122,9 @@ static inline int route4_hash_wild(void)
return 0; \
}
static int route4_classify(struct sk_buff *skb, const struct tcf_proto *tp,
struct tcf_result *res)
TC_INDIRECT_SCOPE int route4_classify(struct sk_buff *skb,
const struct tcf_proto *tp,
struct tcf_result *res)
{
struct route4_head *head = rcu_dereference_bh(tp->root);
struct dst_entry *dst;

2
net/sched/cls_rsvp.c
View File

@ -15,10 +15,12 @@
#include <net/netlink.h>
#include <net/act_api.h>
#include <net/pkt_cls.h>
#include <net/tc_wrapper.h>
#define RSVP_DST_LEN 1
#define RSVP_ID "rsvp"
#define RSVP_OPS cls_rsvp_ops
#define RSVP_CLS rsvp_classify
#include "cls_rsvp.h"
MODULE_LICENSE("GPL");

6
net/sched/cls_rsvp.h
View File

@ -124,8 +124,8 @@ static inline unsigned int hash_src(__be32 *src)
return r; \
}
static int rsvp_classify(struct sk_buff *skb, const struct tcf_proto *tp,
struct tcf_result *res)
TC_INDIRECT_SCOPE int RSVP_CLS(struct sk_buff *skb, const struct tcf_proto *tp,
struct tcf_result *res)
{
struct rsvp_head *head = rcu_dereference_bh(tp->root);
struct rsvp_session *s;
@ -738,7 +738,7 @@ static void rsvp_bind_class(void *fh, u32 classid, unsigned long cl, void *q,
static struct tcf_proto_ops RSVP_OPS __read_mostly = {
.kind = RSVP_ID,
.classify = rsvp_classify,
.classify = RSVP_CLS,
.init = rsvp_init,
.destroy = rsvp_destroy,
.get = rsvp_get,

2
net/sched/cls_rsvp6.c
View File

@ -15,10 +15,12 @@
#include <net/act_api.h>
#include <net/pkt_cls.h>
#include <net/netlink.h>
#include <net/tc_wrapper.h>
#define RSVP_DST_LEN 4
#define RSVP_ID "rsvp6"
#define RSVP_OPS cls_rsvp6_ops
#define RSVP_CLS rsvp6_classify
#include "cls_rsvp.h"
MODULE_LICENSE("GPL");

7
net/sched/cls_tcindex.c
View File

@ -16,6 +16,7 @@
#include <net/netlink.h>
#include <net/pkt_cls.h>
#include <net/sch_generic.h>
#include <net/tc_wrapper.h>
/*
* Passing parameters to the root seems to be done more awkwardly than really
@ -98,9 +99,9 @@ static struct tcindex_filter_result *tcindex_lookup(struct tcindex_data *p,
return NULL;
}
static int tcindex_classify(struct sk_buff *skb, const struct tcf_proto *tp,
struct tcf_result *res)
TC_INDIRECT_SCOPE int tcindex_classify(struct sk_buff *skb,
const struct tcf_proto *tp,
struct tcf_result *res)
{
struct tcindex_data *p = rcu_dereference_bh(tp->root);
struct tcindex_filter_result *f;

6
net/sched/cls_u32.c
View File

@ -39,6 +39,7 @@
#include <net/act_api.h>
#include <net/pkt_cls.h>
#include <linux/idr.h>
#include <net/tc_wrapper.h>
struct tc_u_knode {
struct tc_u_knode __rcu *next;
@ -100,8 +101,9 @@ static inline unsigned int u32_hash_fold(__be32 key,
return h;
}
static int u32_classify(struct sk_buff *skb, const struct tcf_proto *tp,
struct tcf_result *res)
TC_INDIRECT_SCOPE int u32_classify(struct sk_buff *skb,
const struct tcf_proto *tp,
struct tcf_result *res)
{
struct {
struct tc_u_knode *knode;

5
net/sched/sch_api.c
View File

@ -31,6 +31,7 @@
#include <net/netlink.h>
#include <net/pkt_sched.h>
#include <net/pkt_cls.h>
#include <net/tc_wrapper.h>
#include <trace/events/qdisc.h>
@ -2273,6 +2274,8 @@ static struct pernet_operations psched_net_ops = {
.exit = psched_net_exit,
};
DEFINE_STATIC_KEY_FALSE(tc_skip_wrapper);
static int __init pktsched_init(void)
{
int err;
@ -2300,6 +2303,8 @@ static int __init pktsched_init(void)
rtnl_register(PF_UNSPEC, RTM_GETTCLASS, tc_ctl_tclass, tc_dump_tclass,
0);
tc_wrapper_init();
return 0;
}