2019-05-19 12:08:20 +00:00
|
|
|
// SPDX-License-Identifier: GPL-2.0-only
|
2007-09-25 23:14:46 +00:00
|
|
|
/*
|
|
|
|
* drivers/net/veth.c
|
|
|
|
*
|
|
|
|
* Copyright (C) 2007 OpenVZ http://openvz.org, SWsoft Inc
|
|
|
|
*
|
|
|
|
* Author: Pavel Emelianov <xemul@openvz.org>
|
|
|
|
* Ethtool interface from: Eric W. Biederman <ebiederm@xmission.com>
|
|
|
|
*
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include <linux/netdevice.h>
|
include cleanup: Update gfp.h and slab.h includes to prepare for breaking implicit slab.h inclusion from percpu.h
percpu.h is included by sched.h and module.h and thus ends up being
included when building most .c files. percpu.h includes slab.h which
in turn includes gfp.h making everything defined by the two files
universally available and complicating inclusion dependencies.
percpu.h -> slab.h dependency is about to be removed. Prepare for
this change by updating users of gfp and slab facilities include those
headers directly instead of assuming availability. As this conversion
needs to touch large number of source files, the following script is
used as the basis of conversion.
http://userweb.kernel.org/~tj/misc/slabh-sweep.py
The script does the followings.
* Scan files for gfp and slab usages and update includes such that
only the necessary includes are there. ie. if only gfp is used,
gfp.h, if slab is used, slab.h.
* When the script inserts a new include, it looks at the include
blocks and try to put the new include such that its order conforms
to its surrounding. It's put in the include block which contains
core kernel includes, in the same order that the rest are ordered -
alphabetical, Christmas tree, rev-Xmas-tree or at the end if there
doesn't seem to be any matching order.
* If the script can't find a place to put a new include (mostly
because the file doesn't have fitting include block), it prints out
an error message indicating which .h file needs to be added to the
file.
The conversion was done in the following steps.
1. The initial automatic conversion of all .c files updated slightly
over 4000 files, deleting around 700 includes and adding ~480 gfp.h
and ~3000 slab.h inclusions. The script emitted errors for ~400
files.
2. Each error was manually checked. Some didn't need the inclusion,
some needed manual addition while adding it to implementation .h or
embedding .c file was more appropriate for others. This step added
inclusions to around 150 files.
3. The script was run again and the output was compared to the edits
from #2 to make sure no file was left behind.
4. Several build tests were done and a couple of problems were fixed.
e.g. lib/decompress_*.c used malloc/free() wrappers around slab
APIs requiring slab.h to be added manually.
5. The script was run on all .h files but without automatically
editing them as sprinkling gfp.h and slab.h inclusions around .h
files could easily lead to inclusion dependency hell. Most gfp.h
inclusion directives were ignored as stuff from gfp.h was usually
wildly available and often used in preprocessor macros. Each
slab.h inclusion directive was examined and added manually as
necessary.
6. percpu.h was updated not to include slab.h.
7. Build test were done on the following configurations and failures
were fixed. CONFIG_GCOV_KERNEL was turned off for all tests (as my
distributed build env didn't work with gcov compiles) and a few
more options had to be turned off depending on archs to make things
build (like ipr on powerpc/64 which failed due to missing writeq).
* x86 and x86_64 UP and SMP allmodconfig and a custom test config.
* powerpc and powerpc64 SMP allmodconfig
* sparc and sparc64 SMP allmodconfig
* ia64 SMP allmodconfig
* s390 SMP allmodconfig
* alpha SMP allmodconfig
* um on x86_64 SMP allmodconfig
8. percpu.h modifications were reverted so that it could be applied as
a separate patch and serve as bisection point.
Given the fact that I had only a couple of failures from tests on step
6, I'm fairly confident about the coverage of this conversion patch.
If there is a breakage, it's likely to be something in one of the arch
headers which should be easily discoverable easily on most builds of
the specific arch.
Signed-off-by: Tejun Heo <tj@kernel.org>
Guess-its-ok-by: Christoph Lameter <cl@linux-foundation.org>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
2010-03-24 08:04:11 +00:00
|
|
|
#include <linux/slab.h>
|
2007-09-25 23:14:46 +00:00
|
|
|
#include <linux/ethtool.h>
|
|
|
|
#include <linux/etherdevice.h>
|
2011-06-20 05:48:34 +00:00
|
|
|
#include <linux/u64_stats_sync.h>
|
2007-09-25 23:14:46 +00:00
|
|
|
|
2014-02-18 19:53:18 +00:00
|
|
|
#include <net/rtnetlink.h>
|
2007-09-25 23:14:46 +00:00
|
|
|
#include <net/dst.h>
|
|
|
|
#include <net/xfrm.h>
|
2018-08-03 07:58:14 +00:00
|
|
|
#include <net/xdp.h>
|
2007-12-26 01:23:59 +00:00
|
|
|
#include <linux/veth.h>
|
2011-07-03 19:21:01 +00:00
|
|
|
#include <linux/module.h>
|
2018-08-03 07:58:10 +00:00
|
|
|
#include <linux/bpf.h>
|
|
|
|
#include <linux/filter.h>
|
|
|
|
#include <linux/ptr_ring.h>
|
|
|
|
#include <linux/bpf_trace.h>
|
2018-08-29 15:24:11 +00:00
|
|
|
#include <linux/net_tstamp.h>
|
2007-09-25 23:14:46 +00:00
|
|
|
|
|
|
|
#define DRV_NAME "veth"
|
|
|
|
#define DRV_VERSION "1.0"
|
|
|
|
|
2018-08-03 07:58:13 +00:00
|
|
|
#define VETH_XDP_FLAG BIT(0)
|
2018-08-03 07:58:10 +00:00
|
|
|
#define VETH_RING_SIZE 256
|
|
|
|
#define VETH_XDP_HEADROOM (XDP_PACKET_HEADROOM + NET_IP_ALIGN)
|
|
|
|
|
2019-06-13 09:39:59 +00:00
|
|
|
#define VETH_XDP_TX_BULK_SIZE 16
|
2021-01-29 22:04:08 +00:00
|
|
|
#define VETH_XDP_BATCH 16
|
2019-06-13 09:39:59 +00:00
|
|
|
|
2020-03-19 16:41:25 +00:00
|
|
|
struct veth_stats {
|
2020-03-19 16:41:26 +00:00
|
|
|
u64 rx_drops;
|
|
|
|
/* xdp */
|
2020-03-19 16:41:25 +00:00
|
|
|
u64 xdp_packets;
|
|
|
|
u64 xdp_bytes;
|
2020-03-19 16:41:26 +00:00
|
|
|
u64 xdp_redirect;
|
2020-03-19 16:41:25 +00:00
|
|
|
u64 xdp_drops;
|
2020-03-19 16:41:26 +00:00
|
|
|
u64 xdp_tx;
|
2020-03-19 16:41:28 +00:00
|
|
|
u64 xdp_tx_err;
|
2020-03-26 22:10:20 +00:00
|
|
|
u64 peer_tq_xdp_xmit;
|
|
|
|
u64 peer_tq_xdp_xmit_err;
|
2020-03-19 16:41:25 +00:00
|
|
|
};
|
|
|
|
|
2018-10-11 09:36:49 +00:00
|
|
|
struct veth_rq_stats {
|
2020-03-19 16:41:25 +00:00
|
|
|
struct veth_stats vs;
|
2018-10-11 09:36:49 +00:00
|
|
|
struct u64_stats_sync syncp;
|
|
|
|
};
|
|
|
|
|
2018-08-03 07:58:18 +00:00
|
|
|
struct veth_rq {
|
2018-08-03 07:58:10 +00:00
|
|
|
struct napi_struct xdp_napi;
|
2021-04-09 11:04:38 +00:00
|
|
|
struct napi_struct __rcu *napi; /* points to xdp_napi when the latter is initialized */
|
2018-08-03 07:58:10 +00:00
|
|
|
struct net_device *dev;
|
|
|
|
struct bpf_prog __rcu *xdp_prog;
|
2018-08-03 07:58:17 +00:00
|
|
|
struct xdp_mem_info xdp_mem;
|
2018-10-11 09:36:49 +00:00
|
|
|
struct veth_rq_stats stats;
|
2018-08-03 07:58:10 +00:00
|
|
|
bool rx_notify_masked;
|
|
|
|
struct ptr_ring xdp_ring;
|
|
|
|
struct xdp_rxq_info xdp_rxq;
|
2007-09-25 23:14:46 +00:00
|
|
|
};
|
|
|
|
|
2018-08-03 07:58:18 +00:00
|
|
|
struct veth_priv {
|
|
|
|
struct net_device __rcu *peer;
|
|
|
|
atomic64_t dropped;
|
|
|
|
struct bpf_prog *_xdp_prog;
|
|
|
|
struct veth_rq *rq;
|
|
|
|
unsigned int requested_headroom;
|
|
|
|
};
|
|
|
|
|
2019-06-13 09:39:59 +00:00
|
|
|
struct veth_xdp_tx_bq {
|
|
|
|
struct xdp_frame *q[VETH_XDP_TX_BULK_SIZE];
|
|
|
|
unsigned int count;
|
|
|
|
};
|
|
|
|
|
2007-09-25 23:14:46 +00:00
|
|
|
/*
|
|
|
|
* ethtool interface
|
|
|
|
*/
|
|
|
|
|
2018-10-11 09:36:50 +00:00
|
|
|
struct veth_q_stat_desc {
|
|
|
|
char desc[ETH_GSTRING_LEN];
|
|
|
|
size_t offset;
|
|
|
|
};
|
|
|
|
|
2020-03-19 16:41:25 +00:00
|
|
|
#define VETH_RQ_STAT(m) offsetof(struct veth_stats, m)
|
2018-10-11 09:36:50 +00:00
|
|
|
|
|
|
|
static const struct veth_q_stat_desc veth_rq_stats_desc[] = {
|
|
|
|
{ "xdp_packets", VETH_RQ_STAT(xdp_packets) },
|
|
|
|
{ "xdp_bytes", VETH_RQ_STAT(xdp_bytes) },
|
2020-03-26 22:10:20 +00:00
|
|
|
{ "drops", VETH_RQ_STAT(rx_drops) },
|
|
|
|
{ "xdp_redirect", VETH_RQ_STAT(xdp_redirect) },
|
|
|
|
{ "xdp_drops", VETH_RQ_STAT(xdp_drops) },
|
|
|
|
{ "xdp_tx", VETH_RQ_STAT(xdp_tx) },
|
|
|
|
{ "xdp_tx_errors", VETH_RQ_STAT(xdp_tx_err) },
|
2018-10-11 09:36:50 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
#define VETH_RQ_STATS_LEN ARRAY_SIZE(veth_rq_stats_desc)
|
|
|
|
|
2020-03-26 22:10:20 +00:00
|
|
|
static const struct veth_q_stat_desc veth_tq_stats_desc[] = {
|
|
|
|
{ "xdp_xmit", VETH_RQ_STAT(peer_tq_xdp_xmit) },
|
|
|
|
{ "xdp_xmit_errors", VETH_RQ_STAT(peer_tq_xdp_xmit_err) },
|
|
|
|
};
|
|
|
|
|
|
|
|
#define VETH_TQ_STATS_LEN ARRAY_SIZE(veth_tq_stats_desc)
|
|
|
|
|
2007-09-25 23:14:46 +00:00
|
|
|
static struct {
|
|
|
|
const char string[ETH_GSTRING_LEN];
|
|
|
|
} ethtool_stats_keys[] = {
|
|
|
|
{ "peer_ifindex" },
|
|
|
|
};
|
|
|
|
|
2017-03-29 06:24:21 +00:00
|
|
|
static int veth_get_link_ksettings(struct net_device *dev,
|
|
|
|
struct ethtool_link_ksettings *cmd)
|
2007-09-25 23:14:46 +00:00
|
|
|
{
|
2017-03-29 06:24:21 +00:00
|
|
|
cmd->base.speed = SPEED_10000;
|
|
|
|
cmd->base.duplex = DUPLEX_FULL;
|
|
|
|
cmd->base.port = PORT_TP;
|
|
|
|
cmd->base.autoneg = AUTONEG_DISABLE;
|
2007-09-25 23:14:46 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void veth_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
|
|
|
|
{
|
2011-11-15 14:59:53 +00:00
|
|
|
strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
|
|
|
|
strlcpy(info->version, DRV_VERSION, sizeof(info->version));
|
2007-09-25 23:14:46 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void veth_get_strings(struct net_device *dev, u32 stringset, u8 *buf)
|
|
|
|
{
|
2021-11-25 02:54:44 +00:00
|
|
|
u8 *p = buf;
|
2018-10-11 09:36:50 +00:00
|
|
|
int i, j;
|
|
|
|
|
2007-09-25 23:14:46 +00:00
|
|
|
switch(stringset) {
|
|
|
|
case ETH_SS_STATS:
|
2018-10-11 09:36:50 +00:00
|
|
|
memcpy(p, ðtool_stats_keys, sizeof(ethtool_stats_keys));
|
|
|
|
p += sizeof(ethtool_stats_keys);
|
2021-11-25 02:54:44 +00:00
|
|
|
for (i = 0; i < dev->real_num_rx_queues; i++)
|
|
|
|
for (j = 0; j < VETH_RQ_STATS_LEN; j++)
|
|
|
|
ethtool_sprintf(&p, "rx_queue_%u_%.18s",
|
|
|
|
i, veth_rq_stats_desc[j].desc);
|
|
|
|
|
|
|
|
for (i = 0; i < dev->real_num_tx_queues; i++)
|
|
|
|
for (j = 0; j < VETH_TQ_STATS_LEN; j++)
|
|
|
|
ethtool_sprintf(&p, "tx_queue_%u_%.18s",
|
|
|
|
i, veth_tq_stats_desc[j].desc);
|
2007-09-25 23:14:46 +00:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2007-10-04 01:07:32 +00:00
|
|
|
static int veth_get_sset_count(struct net_device *dev, int sset)
|
2007-09-25 23:14:46 +00:00
|
|
|
{
|
2007-10-04 01:07:32 +00:00
|
|
|
switch (sset) {
|
|
|
|
case ETH_SS_STATS:
|
2018-10-11 09:36:50 +00:00
|
|
|
return ARRAY_SIZE(ethtool_stats_keys) +
|
2020-03-26 22:10:20 +00:00
|
|
|
VETH_RQ_STATS_LEN * dev->real_num_rx_queues +
|
|
|
|
VETH_TQ_STATS_LEN * dev->real_num_tx_queues;
|
2007-10-04 01:07:32 +00:00
|
|
|
default:
|
|
|
|
return -EOPNOTSUPP;
|
|
|
|
}
|
2007-09-25 23:14:46 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void veth_get_ethtool_stats(struct net_device *dev,
|
|
|
|
struct ethtool_stats *stats, u64 *data)
|
|
|
|
{
|
2020-03-26 22:10:20 +00:00
|
|
|
struct veth_priv *rcv_priv, *priv = netdev_priv(dev);
|
2013-01-04 15:42:40 +00:00
|
|
|
struct net_device *peer = rtnl_dereference(priv->peer);
|
2018-10-11 09:36:50 +00:00
|
|
|
int i, j, idx;
|
2007-09-25 23:14:46 +00:00
|
|
|
|
2013-01-04 15:42:40 +00:00
|
|
|
data[0] = peer ? peer->ifindex : 0;
|
2018-10-11 09:36:50 +00:00
|
|
|
idx = 1;
|
|
|
|
for (i = 0; i < dev->real_num_rx_queues; i++) {
|
|
|
|
const struct veth_rq_stats *rq_stats = &priv->rq[i].stats;
|
2020-03-19 16:41:25 +00:00
|
|
|
const void *stats_base = (void *)&rq_stats->vs;
|
2018-10-11 09:36:50 +00:00
|
|
|
unsigned int start;
|
|
|
|
size_t offset;
|
|
|
|
|
|
|
|
do {
|
|
|
|
start = u64_stats_fetch_begin_irq(&rq_stats->syncp);
|
|
|
|
for (j = 0; j < VETH_RQ_STATS_LEN; j++) {
|
|
|
|
offset = veth_rq_stats_desc[j].offset;
|
|
|
|
data[idx + j] = *(u64 *)(stats_base + offset);
|
|
|
|
}
|
|
|
|
} while (u64_stats_fetch_retry_irq(&rq_stats->syncp, start));
|
|
|
|
idx += VETH_RQ_STATS_LEN;
|
|
|
|
}
|
2020-03-26 22:10:20 +00:00
|
|
|
|
|
|
|
if (!peer)
|
|
|
|
return;
|
|
|
|
|
|
|
|
rcv_priv = netdev_priv(peer);
|
|
|
|
for (i = 0; i < peer->real_num_rx_queues; i++) {
|
|
|
|
const struct veth_rq_stats *rq_stats = &rcv_priv->rq[i].stats;
|
|
|
|
const void *base = (void *)&rq_stats->vs;
|
|
|
|
unsigned int start, tx_idx = idx;
|
|
|
|
size_t offset;
|
|
|
|
|
|
|
|
tx_idx += (i % dev->real_num_tx_queues) * VETH_TQ_STATS_LEN;
|
|
|
|
do {
|
|
|
|
start = u64_stats_fetch_begin_irq(&rq_stats->syncp);
|
|
|
|
for (j = 0; j < VETH_TQ_STATS_LEN; j++) {
|
|
|
|
offset = veth_tq_stats_desc[j].offset;
|
|
|
|
data[tx_idx + j] += *(u64 *)(base + offset);
|
|
|
|
}
|
|
|
|
} while (u64_stats_fetch_retry_irq(&rq_stats->syncp, start));
|
|
|
|
}
|
2007-09-25 23:14:46 +00:00
|
|
|
}
|
|
|
|
|
2021-03-29 22:43:12 +00:00
|
|
|
static void veth_get_channels(struct net_device *dev,
|
|
|
|
struct ethtool_channels *channels)
|
|
|
|
{
|
|
|
|
channels->tx_count = dev->real_num_tx_queues;
|
|
|
|
channels->rx_count = dev->real_num_rx_queues;
|
2021-07-20 08:41:50 +00:00
|
|
|
channels->max_tx = dev->num_tx_queues;
|
|
|
|
channels->max_rx = dev->num_rx_queues;
|
2021-03-29 22:43:12 +00:00
|
|
|
}
|
|
|
|
|
2021-07-20 08:41:50 +00:00
|
|
|
static int veth_set_channels(struct net_device *dev,
|
|
|
|
struct ethtool_channels *ch);
|
|
|
|
|
2009-09-02 08:03:33 +00:00
|
|
|
static const struct ethtool_ops veth_ethtool_ops = {
|
2007-09-25 23:14:46 +00:00
|
|
|
.get_drvinfo = veth_get_drvinfo,
|
|
|
|
.get_link = ethtool_op_get_link,
|
|
|
|
.get_strings = veth_get_strings,
|
2007-10-04 01:07:32 +00:00
|
|
|
.get_sset_count = veth_get_sset_count,
|
2007-09-25 23:14:46 +00:00
|
|
|
.get_ethtool_stats = veth_get_ethtool_stats,
|
2017-03-29 06:24:21 +00:00
|
|
|
.get_link_ksettings = veth_get_link_ksettings,
|
2019-04-12 11:06:15 +00:00
|
|
|
.get_ts_info = ethtool_op_get_ts_info,
|
2021-03-29 22:43:12 +00:00
|
|
|
.get_channels = veth_get_channels,
|
2021-07-20 08:41:50 +00:00
|
|
|
.set_channels = veth_set_channels,
|
2007-09-25 23:14:46 +00:00
|
|
|
};
|
|
|
|
|
2018-08-03 07:58:10 +00:00
|
|
|
/* general routines */
|
|
|
|
|
2018-08-03 07:58:13 +00:00
|
|
|
static bool veth_is_xdp_frame(void *ptr)
|
|
|
|
{
|
|
|
|
return (unsigned long)ptr & VETH_XDP_FLAG;
|
|
|
|
}
|
|
|
|
|
2020-08-19 02:00:27 +00:00
|
|
|
static struct xdp_frame *veth_ptr_to_xdp(void *ptr)
|
2018-08-03 07:58:13 +00:00
|
|
|
{
|
|
|
|
return (void *)((unsigned long)ptr & ~VETH_XDP_FLAG);
|
|
|
|
}
|
|
|
|
|
2020-08-19 02:00:27 +00:00
|
|
|
static void *veth_xdp_to_ptr(struct xdp_frame *xdp)
|
2018-08-03 07:58:14 +00:00
|
|
|
{
|
2020-08-19 02:00:27 +00:00
|
|
|
return (void *)((unsigned long)xdp | VETH_XDP_FLAG);
|
2018-08-03 07:58:14 +00:00
|
|
|
}
|
|
|
|
|
2018-08-03 07:58:13 +00:00
|
|
|
static void veth_ptr_free(void *ptr)
|
|
|
|
{
|
|
|
|
if (veth_is_xdp_frame(ptr))
|
|
|
|
xdp_return_frame(veth_ptr_to_xdp(ptr));
|
|
|
|
else
|
|
|
|
kfree_skb(ptr);
|
|
|
|
}
|
|
|
|
|
2018-08-03 07:58:18 +00:00
|
|
|
static void __veth_xdp_flush(struct veth_rq *rq)
|
2018-08-03 07:58:10 +00:00
|
|
|
{
|
|
|
|
/* Write ptr_ring before reading rx_notify_masked */
|
|
|
|
smp_mb();
|
2022-02-08 23:28:22 +00:00
|
|
|
if (!READ_ONCE(rq->rx_notify_masked) &&
|
|
|
|
napi_schedule_prep(&rq->xdp_napi)) {
|
|
|
|
WRITE_ONCE(rq->rx_notify_masked, true);
|
|
|
|
__napi_schedule(&rq->xdp_napi);
|
2018-08-03 07:58:10 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-08-03 07:58:18 +00:00
|
|
|
static int veth_xdp_rx(struct veth_rq *rq, struct sk_buff *skb)
|
2018-08-03 07:58:10 +00:00
|
|
|
{
|
2018-08-03 07:58:18 +00:00
|
|
|
if (unlikely(ptr_ring_produce(&rq->xdp_ring, skb))) {
|
2018-08-03 07:58:10 +00:00
|
|
|
dev_kfree_skb_any(skb);
|
|
|
|
return NET_RX_DROP;
|
|
|
|
}
|
|
|
|
|
|
|
|
return NET_RX_SUCCESS;
|
|
|
|
}
|
|
|
|
|
2018-08-03 07:58:18 +00:00
|
|
|
static int veth_forward_skb(struct net_device *dev, struct sk_buff *skb,
|
|
|
|
struct veth_rq *rq, bool xdp)
|
2007-09-25 23:14:46 +00:00
|
|
|
{
|
2018-08-03 07:58:10 +00:00
|
|
|
return __dev_forward_skb(dev, skb) ?: xdp ?
|
2018-08-03 07:58:18 +00:00
|
|
|
veth_xdp_rx(rq, skb) :
|
2018-08-03 07:58:10 +00:00
|
|
|
netif_rx(skb);
|
|
|
|
}
|
|
|
|
|
2021-04-09 11:04:39 +00:00
|
|
|
/* return true if the specified skb has chances of GRO aggregation
|
|
|
|
* Don't strive for accuracy, but try to avoid GRO overhead in the most
|
|
|
|
* common scenarios.
|
|
|
|
* When XDP is enabled, all traffic is considered eligible, as the xmit
|
|
|
|
* device has TSO off.
|
|
|
|
* When TSO is enabled on the xmit device, we are likely interested only
|
|
|
|
* in UDP aggregation, explicitly check for that if the skb is suspected
|
|
|
|
* - the sock_wfree destructor is used by UDP, ICMP and XDP sockets -
|
|
|
|
* to belong to locally generated UDP traffic.
|
|
|
|
*/
|
|
|
|
static bool veth_skb_is_eligible_for_gro(const struct net_device *dev,
|
|
|
|
const struct net_device *rcv,
|
|
|
|
const struct sk_buff *skb)
|
|
|
|
{
|
|
|
|
return !(dev->features & NETIF_F_ALL_TSO) ||
|
|
|
|
(skb->destructor == sock_wfree &&
|
|
|
|
rcv->features & (NETIF_F_GRO_FRAGLIST | NETIF_F_GRO_UDP_FWD));
|
|
|
|
}
|
|
|
|
|
2018-08-03 07:58:10 +00:00
|
|
|
static netdev_tx_t veth_xmit(struct sk_buff *skb, struct net_device *dev)
|
|
|
|
{
|
|
|
|
struct veth_priv *rcv_priv, *priv = netdev_priv(dev);
|
2018-08-03 07:58:18 +00:00
|
|
|
struct veth_rq *rq = NULL;
|
2013-01-04 15:42:40 +00:00
|
|
|
struct net_device *rcv;
|
2012-12-29 16:02:43 +00:00
|
|
|
int length = skb->len;
|
2021-04-09 11:04:38 +00:00
|
|
|
bool use_napi = false;
|
2018-08-03 07:58:18 +00:00
|
|
|
int rxq;
|
2007-09-25 23:14:46 +00:00
|
|
|
|
2013-01-04 15:42:40 +00:00
|
|
|
rcu_read_lock();
|
|
|
|
rcv = rcu_dereference(priv->peer);
|
2022-04-06 14:18:54 +00:00
|
|
|
if (unlikely(!rcv) || !pskb_may_pull(skb, ETH_HLEN)) {
|
2013-01-04 15:42:40 +00:00
|
|
|
kfree_skb(skb);
|
|
|
|
goto drop;
|
|
|
|
}
|
2007-09-25 23:14:46 +00:00
|
|
|
|
2018-08-03 07:58:10 +00:00
|
|
|
rcv_priv = netdev_priv(rcv);
|
2018-08-03 07:58:18 +00:00
|
|
|
rxq = skb_get_queue_mapping(skb);
|
|
|
|
if (rxq < rcv->real_num_rx_queues) {
|
|
|
|
rq = &rcv_priv->rq[rxq];
|
2021-04-09 11:04:38 +00:00
|
|
|
|
|
|
|
/* The napi pointer is available when an XDP program is
|
|
|
|
* attached or when GRO is enabled
|
2021-04-09 11:04:39 +00:00
|
|
|
* Don't bother with napi/GRO if the skb can't be aggregated
|
2021-04-09 11:04:38 +00:00
|
|
|
*/
|
2021-04-09 11:04:39 +00:00
|
|
|
use_napi = rcu_access_pointer(rq->napi) &&
|
|
|
|
veth_skb_is_eligible_for_gro(dev, rcv, skb);
|
2018-08-03 07:58:18 +00:00
|
|
|
}
|
2018-08-03 07:58:10 +00:00
|
|
|
|
2018-08-29 15:24:11 +00:00
|
|
|
skb_tx_timestamp(skb);
|
2021-04-09 11:04:38 +00:00
|
|
|
if (likely(veth_forward_skb(rcv, skb, rq, use_napi) == NET_RX_SUCCESS)) {
|
|
|
|
if (!use_napi)
|
2019-11-08 00:27:17 +00:00
|
|
|
dev_lstats_add(dev, length);
|
2012-12-29 16:02:43 +00:00
|
|
|
} else {
|
2013-01-04 15:42:40 +00:00
|
|
|
drop:
|
2012-12-29 16:02:43 +00:00
|
|
|
atomic64_inc(&priv->dropped);
|
|
|
|
}
|
2018-08-03 07:58:10 +00:00
|
|
|
|
2021-04-09 11:04:38 +00:00
|
|
|
if (use_napi)
|
2018-08-03 07:58:18 +00:00
|
|
|
__veth_xdp_flush(rq);
|
2018-08-03 07:58:10 +00:00
|
|
|
|
2013-01-04 15:42:40 +00:00
|
|
|
rcu_read_unlock();
|
2018-08-03 07:58:10 +00:00
|
|
|
|
2009-06-23 06:03:08 +00:00
|
|
|
return NETDEV_TX_OK;
|
2007-09-25 23:14:46 +00:00
|
|
|
}
|
|
|
|
|
2019-11-08 00:27:17 +00:00
|
|
|
static u64 veth_stats_tx(struct net_device *dev, u64 *packets, u64 *bytes)
|
2007-09-25 23:14:46 +00:00
|
|
|
{
|
2011-06-20 05:48:34 +00:00
|
|
|
struct veth_priv *priv = netdev_priv(dev);
|
|
|
|
|
2019-11-08 00:27:17 +00:00
|
|
|
dev_lstats_read(dev, packets, bytes);
|
2012-12-29 16:02:43 +00:00
|
|
|
return atomic64_read(&priv->dropped);
|
|
|
|
}
|
|
|
|
|
2020-03-19 16:41:25 +00:00
|
|
|
static void veth_stats_rx(struct veth_stats *result, struct net_device *dev)
|
2018-10-11 09:36:49 +00:00
|
|
|
{
|
|
|
|
struct veth_priv *priv = netdev_priv(dev);
|
|
|
|
int i;
|
|
|
|
|
2020-03-26 22:10:20 +00:00
|
|
|
result->peer_tq_xdp_xmit_err = 0;
|
2018-10-11 09:36:49 +00:00
|
|
|
result->xdp_packets = 0;
|
2020-03-19 16:41:29 +00:00
|
|
|
result->xdp_tx_err = 0;
|
2018-10-11 09:36:49 +00:00
|
|
|
result->xdp_bytes = 0;
|
2020-03-19 16:41:27 +00:00
|
|
|
result->rx_drops = 0;
|
2018-10-11 09:36:49 +00:00
|
|
|
for (i = 0; i < dev->num_rx_queues; i++) {
|
2020-03-26 22:10:20 +00:00
|
|
|
u64 packets, bytes, drops, xdp_tx_err, peer_tq_xdp_xmit_err;
|
2018-10-11 09:36:49 +00:00
|
|
|
struct veth_rq_stats *stats = &priv->rq[i].stats;
|
|
|
|
unsigned int start;
|
|
|
|
|
|
|
|
do {
|
|
|
|
start = u64_stats_fetch_begin_irq(&stats->syncp);
|
2020-03-26 22:10:20 +00:00
|
|
|
peer_tq_xdp_xmit_err = stats->vs.peer_tq_xdp_xmit_err;
|
2020-03-19 16:41:29 +00:00
|
|
|
xdp_tx_err = stats->vs.xdp_tx_err;
|
2020-03-19 16:41:25 +00:00
|
|
|
packets = stats->vs.xdp_packets;
|
|
|
|
bytes = stats->vs.xdp_bytes;
|
2020-03-19 16:41:27 +00:00
|
|
|
drops = stats->vs.rx_drops;
|
2018-10-11 09:36:49 +00:00
|
|
|
} while (u64_stats_fetch_retry_irq(&stats->syncp, start));
|
2020-03-26 22:10:20 +00:00
|
|
|
result->peer_tq_xdp_xmit_err += peer_tq_xdp_xmit_err;
|
2020-03-19 16:41:29 +00:00
|
|
|
result->xdp_tx_err += xdp_tx_err;
|
2018-10-11 09:36:49 +00:00
|
|
|
result->xdp_packets += packets;
|
|
|
|
result->xdp_bytes += bytes;
|
2020-03-19 16:41:27 +00:00
|
|
|
result->rx_drops += drops;
|
2018-10-11 09:36:49 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-01-07 03:12:52 +00:00
|
|
|
static void veth_get_stats64(struct net_device *dev,
|
|
|
|
struct rtnl_link_stats64 *tot)
|
2012-12-29 16:02:43 +00:00
|
|
|
{
|
|
|
|
struct veth_priv *priv = netdev_priv(dev);
|
2013-01-04 15:42:40 +00:00
|
|
|
struct net_device *peer;
|
2020-03-19 16:41:25 +00:00
|
|
|
struct veth_stats rx;
|
2019-11-08 00:27:17 +00:00
|
|
|
u64 packets, bytes;
|
2018-10-11 09:36:49 +00:00
|
|
|
|
2019-11-08 00:27:17 +00:00
|
|
|
tot->tx_dropped = veth_stats_tx(dev, &packets, &bytes);
|
|
|
|
tot->tx_bytes = bytes;
|
|
|
|
tot->tx_packets = packets;
|
2012-12-29 16:02:43 +00:00
|
|
|
|
2018-10-11 09:36:49 +00:00
|
|
|
veth_stats_rx(&rx, dev);
|
2020-03-26 22:10:20 +00:00
|
|
|
tot->tx_dropped += rx.xdp_tx_err;
|
|
|
|
tot->rx_dropped = rx.rx_drops + rx.peer_tq_xdp_xmit_err;
|
2018-10-11 09:36:49 +00:00
|
|
|
tot->rx_bytes = rx.xdp_bytes;
|
|
|
|
tot->rx_packets = rx.xdp_packets;
|
2012-12-29 16:02:43 +00:00
|
|
|
|
2013-01-04 15:42:40 +00:00
|
|
|
rcu_read_lock();
|
|
|
|
peer = rcu_dereference(priv->peer);
|
|
|
|
if (peer) {
|
2020-03-04 01:49:29 +00:00
|
|
|
veth_stats_tx(peer, &packets, &bytes);
|
2019-11-08 00:27:17 +00:00
|
|
|
tot->rx_bytes += bytes;
|
|
|
|
tot->rx_packets += packets;
|
2018-10-11 09:36:49 +00:00
|
|
|
|
|
|
|
veth_stats_rx(&rx, peer);
|
2020-03-26 22:10:20 +00:00
|
|
|
tot->tx_dropped += rx.peer_tq_xdp_xmit_err;
|
|
|
|
tot->rx_dropped += rx.xdp_tx_err;
|
2018-10-11 09:36:49 +00:00
|
|
|
tot->tx_bytes += rx.xdp_bytes;
|
|
|
|
tot->tx_packets += rx.xdp_packets;
|
2013-01-04 15:42:40 +00:00
|
|
|
}
|
|
|
|
rcu_read_unlock();
|
2007-09-25 23:14:46 +00:00
|
|
|
}
|
|
|
|
|
2013-10-04 08:52:24 +00:00
|
|
|
/* fake multicast ability */
|
|
|
|
static void veth_set_multicast_list(struct net_device *dev)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
2018-08-03 07:58:10 +00:00
|
|
|
static struct sk_buff *veth_build_skb(void *head, int headroom, int len,
|
|
|
|
int buflen)
|
|
|
|
{
|
|
|
|
struct sk_buff *skb;
|
|
|
|
|
|
|
|
skb = build_skb(head, buflen);
|
|
|
|
if (!skb)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
skb_reserve(skb, headroom);
|
|
|
|
skb_put(skb, len);
|
|
|
|
|
|
|
|
return skb;
|
|
|
|
}
|
|
|
|
|
2018-08-03 07:58:18 +00:00
|
|
|
static int veth_select_rxq(struct net_device *dev)
|
|
|
|
{
|
|
|
|
return smp_processor_id() % dev->real_num_rx_queues;
|
|
|
|
}
|
|
|
|
|
2020-10-10 23:40:02 +00:00
|
|
|
static struct net_device *veth_peer_dev(struct net_device *dev)
|
|
|
|
{
|
|
|
|
struct veth_priv *priv = netdev_priv(dev);
|
|
|
|
|
|
|
|
/* Callers must be under RCU read side. */
|
|
|
|
return rcu_dereference(priv->peer);
|
|
|
|
}
|
|
|
|
|
2018-08-03 07:58:14 +00:00
|
|
|
static int veth_xdp_xmit(struct net_device *dev, int n,
|
2020-03-19 16:41:28 +00:00
|
|
|
struct xdp_frame **frames,
|
|
|
|
u32 flags, bool ndo_xmit)
|
2018-08-03 07:58:14 +00:00
|
|
|
{
|
|
|
|
struct veth_priv *rcv_priv, *priv = netdev_priv(dev);
|
2021-03-08 11:06:58 +00:00
|
|
|
int i, ret = -ENXIO, nxmit = 0;
|
2018-08-03 07:58:14 +00:00
|
|
|
struct net_device *rcv;
|
2020-03-26 22:10:20 +00:00
|
|
|
unsigned int max_len;
|
2018-08-03 07:58:18 +00:00
|
|
|
struct veth_rq *rq;
|
2018-08-03 07:58:14 +00:00
|
|
|
|
2020-03-26 22:10:20 +00:00
|
|
|
if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
|
2020-03-19 16:41:29 +00:00
|
|
|
return -EINVAL;
|
2018-08-03 07:58:14 +00:00
|
|
|
|
2020-03-26 22:10:20 +00:00
|
|
|
rcu_read_lock();
|
2018-08-03 07:58:14 +00:00
|
|
|
rcv = rcu_dereference(priv->peer);
|
2020-03-26 22:10:20 +00:00
|
|
|
if (unlikely(!rcv))
|
|
|
|
goto out;
|
2018-08-03 07:58:14 +00:00
|
|
|
|
|
|
|
rcv_priv = netdev_priv(rcv);
|
2020-03-26 22:10:20 +00:00
|
|
|
rq = &rcv_priv->rq[veth_select_rxq(rcv)];
|
2021-04-16 15:47:45 +00:00
|
|
|
/* The napi pointer is set if NAPI is enabled, which ensures that
|
|
|
|
* xdp_ring is initialized on receive side and the peer device is up.
|
2018-08-03 07:58:14 +00:00
|
|
|
*/
|
2021-04-16 15:47:45 +00:00
|
|
|
if (!rcu_access_pointer(rq->napi))
|
2020-03-26 22:10:20 +00:00
|
|
|
goto out;
|
2018-08-03 07:58:14 +00:00
|
|
|
|
|
|
|
max_len = rcv->mtu + rcv->hard_header_len + VLAN_HLEN;
|
|
|
|
|
2018-08-03 07:58:18 +00:00
|
|
|
spin_lock(&rq->xdp_ring.producer_lock);
|
2018-08-03 07:58:14 +00:00
|
|
|
for (i = 0; i < n; i++) {
|
|
|
|
struct xdp_frame *frame = frames[i];
|
|
|
|
void *ptr = veth_xdp_to_ptr(frame);
|
|
|
|
|
|
|
|
if (unlikely(frame->len > max_len ||
|
2021-03-08 11:06:58 +00:00
|
|
|
__ptr_ring_produce(&rq->xdp_ring, ptr)))
|
|
|
|
break;
|
|
|
|
nxmit++;
|
2018-08-03 07:58:14 +00:00
|
|
|
}
|
2018-08-03 07:58:18 +00:00
|
|
|
spin_unlock(&rq->xdp_ring.producer_lock);
|
2018-08-03 07:58:14 +00:00
|
|
|
|
|
|
|
if (flags & XDP_XMIT_FLUSH)
|
2018-08-03 07:58:18 +00:00
|
|
|
__veth_xdp_flush(rq);
|
2018-08-03 07:58:14 +00:00
|
|
|
|
2021-03-08 11:06:58 +00:00
|
|
|
ret = nxmit;
|
2020-03-19 16:41:28 +00:00
|
|
|
if (ndo_xmit) {
|
2020-03-26 22:10:20 +00:00
|
|
|
u64_stats_update_begin(&rq->stats.syncp);
|
2021-03-08 11:06:58 +00:00
|
|
|
rq->stats.vs.peer_tq_xdp_xmit += nxmit;
|
|
|
|
rq->stats.vs.peer_tq_xdp_xmit_err += n - nxmit;
|
2020-03-26 22:10:20 +00:00
|
|
|
u64_stats_update_end(&rq->stats.syncp);
|
2020-03-19 16:41:28 +00:00
|
|
|
}
|
|
|
|
|
2020-03-26 22:10:20 +00:00
|
|
|
out:
|
2020-01-27 00:14:02 +00:00
|
|
|
rcu_read_unlock();
|
2018-10-11 09:36:48 +00:00
|
|
|
|
|
|
|
return ret;
|
2018-08-03 07:58:14 +00:00
|
|
|
}
|
|
|
|
|
2020-03-19 16:41:28 +00:00
|
|
|
static int veth_ndo_xdp_xmit(struct net_device *dev, int n,
|
|
|
|
struct xdp_frame **frames, u32 flags)
|
|
|
|
{
|
2020-03-26 22:10:20 +00:00
|
|
|
int err;
|
|
|
|
|
|
|
|
err = veth_xdp_xmit(dev, n, frames, flags, true);
|
|
|
|
if (err < 0) {
|
|
|
|
struct veth_priv *priv = netdev_priv(dev);
|
|
|
|
|
|
|
|
atomic64_add(n, &priv->dropped);
|
|
|
|
}
|
|
|
|
|
|
|
|
return err;
|
2020-03-19 16:41:28 +00:00
|
|
|
}
|
|
|
|
|
2020-03-26 22:10:19 +00:00
|
|
|
static void veth_xdp_flush_bq(struct veth_rq *rq, struct veth_xdp_tx_bq *bq)
|
2019-06-13 09:39:59 +00:00
|
|
|
{
|
2021-03-08 11:06:58 +00:00
|
|
|
int sent, i, err = 0, drops;
|
2019-06-13 09:39:59 +00:00
|
|
|
|
2020-03-26 22:10:19 +00:00
|
|
|
sent = veth_xdp_xmit(rq->dev, bq->count, bq->q, 0, false);
|
2019-06-13 09:39:59 +00:00
|
|
|
if (sent < 0) {
|
|
|
|
err = sent;
|
|
|
|
sent = 0;
|
|
|
|
}
|
2021-03-08 11:06:58 +00:00
|
|
|
|
|
|
|
for (i = sent; unlikely(i < bq->count); i++)
|
|
|
|
xdp_return_frame(bq->q[i]);
|
|
|
|
|
|
|
|
drops = bq->count - sent;
|
|
|
|
trace_xdp_bulk_tx(rq->dev, sent, drops, err);
|
2019-06-13 09:39:59 +00:00
|
|
|
|
2020-03-26 22:10:20 +00:00
|
|
|
u64_stats_update_begin(&rq->stats.syncp);
|
|
|
|
rq->stats.vs.xdp_tx += sent;
|
2021-03-08 11:06:58 +00:00
|
|
|
rq->stats.vs.xdp_tx_err += drops;
|
2020-03-26 22:10:20 +00:00
|
|
|
u64_stats_update_end(&rq->stats.syncp);
|
|
|
|
|
2019-06-13 09:39:59 +00:00
|
|
|
bq->count = 0;
|
|
|
|
}
|
|
|
|
|
2020-03-26 22:10:19 +00:00
|
|
|
static void veth_xdp_flush(struct veth_rq *rq, struct veth_xdp_tx_bq *bq)
|
2018-08-03 07:58:17 +00:00
|
|
|
{
|
2020-03-26 22:10:19 +00:00
|
|
|
struct veth_priv *rcv_priv, *priv = netdev_priv(rq->dev);
|
2018-08-03 07:58:17 +00:00
|
|
|
struct net_device *rcv;
|
2020-03-26 22:10:19 +00:00
|
|
|
struct veth_rq *rcv_rq;
|
2018-08-03 07:58:17 +00:00
|
|
|
|
|
|
|
rcu_read_lock();
|
2020-03-26 22:10:19 +00:00
|
|
|
veth_xdp_flush_bq(rq, bq);
|
2018-08-03 07:58:17 +00:00
|
|
|
rcv = rcu_dereference(priv->peer);
|
|
|
|
if (unlikely(!rcv))
|
|
|
|
goto out;
|
|
|
|
|
|
|
|
rcv_priv = netdev_priv(rcv);
|
2020-03-26 22:10:19 +00:00
|
|
|
rcv_rq = &rcv_priv->rq[veth_select_rxq(rcv)];
|
2018-08-03 07:58:17 +00:00
|
|
|
/* xdp_ring is initialized on receive side? */
|
2020-03-26 22:10:19 +00:00
|
|
|
if (unlikely(!rcu_access_pointer(rcv_rq->xdp_prog)))
|
2018-08-03 07:58:17 +00:00
|
|
|
goto out;
|
|
|
|
|
2020-03-26 22:10:19 +00:00
|
|
|
__veth_xdp_flush(rcv_rq);
|
2018-08-03 07:58:17 +00:00
|
|
|
out:
|
|
|
|
rcu_read_unlock();
|
|
|
|
}
|
|
|
|
|
2020-03-26 22:10:19 +00:00
|
|
|
static int veth_xdp_tx(struct veth_rq *rq, struct xdp_buff *xdp,
|
2019-06-13 09:39:59 +00:00
|
|
|
struct veth_xdp_tx_bq *bq)
|
2018-08-03 07:58:17 +00:00
|
|
|
{
|
2020-05-28 20:47:29 +00:00
|
|
|
struct xdp_frame *frame = xdp_convert_buff_to_frame(xdp);
|
2018-08-03 07:58:17 +00:00
|
|
|
|
|
|
|
if (unlikely(!frame))
|
|
|
|
return -EOVERFLOW;
|
|
|
|
|
2019-06-13 09:39:59 +00:00
|
|
|
if (unlikely(bq->count == VETH_XDP_TX_BULK_SIZE))
|
2020-03-26 22:10:19 +00:00
|
|
|
veth_xdp_flush_bq(rq, bq);
|
2019-06-13 09:39:59 +00:00
|
|
|
|
|
|
|
bq->q[bq->count++] = frame;
|
|
|
|
|
|
|
|
return 0;
|
2018-08-03 07:58:17 +00:00
|
|
|
}
|
|
|
|
|
2021-01-29 22:04:08 +00:00
|
|
|
static struct xdp_frame *veth_xdp_rcv_one(struct veth_rq *rq,
|
|
|
|
struct xdp_frame *frame,
|
|
|
|
struct veth_xdp_tx_bq *bq,
|
|
|
|
struct veth_stats *stats)
|
2018-08-03 07:58:13 +00:00
|
|
|
{
|
2018-08-03 07:58:17 +00:00
|
|
|
struct xdp_frame orig_frame;
|
2018-08-03 07:58:13 +00:00
|
|
|
struct bpf_prog *xdp_prog;
|
|
|
|
|
|
|
|
rcu_read_lock();
|
2018-08-03 07:58:18 +00:00
|
|
|
xdp_prog = rcu_dereference(rq->xdp_prog);
|
2018-08-03 07:58:13 +00:00
|
|
|
if (likely(xdp_prog)) {
|
|
|
|
struct xdp_buff xdp;
|
|
|
|
u32 act;
|
|
|
|
|
2020-05-28 20:47:28 +00:00
|
|
|
xdp_convert_frame_to_buff(frame, &xdp);
|
2018-08-03 07:58:18 +00:00
|
|
|
xdp.rxq = &rq->xdp_rxq;
|
2018-08-03 07:58:13 +00:00
|
|
|
|
|
|
|
act = bpf_prog_run_xdp(xdp_prog, &xdp);
|
|
|
|
|
|
|
|
switch (act) {
|
|
|
|
case XDP_PASS:
|
2021-01-12 18:26:13 +00:00
|
|
|
if (xdp_update_frame_from_buff(&xdp, frame))
|
|
|
|
goto err_xdp;
|
2018-08-03 07:58:13 +00:00
|
|
|
break;
|
2018-08-03 07:58:17 +00:00
|
|
|
case XDP_TX:
|
|
|
|
orig_frame = *frame;
|
|
|
|
xdp.rxq->mem = frame->mem;
|
2020-03-26 22:10:19 +00:00
|
|
|
if (unlikely(veth_xdp_tx(rq, &xdp, bq) < 0)) {
|
2018-08-03 07:58:18 +00:00
|
|
|
trace_xdp_exception(rq->dev, xdp_prog, act);
|
2018-08-03 07:58:17 +00:00
|
|
|
frame = &orig_frame;
|
2020-03-19 16:41:26 +00:00
|
|
|
stats->rx_drops++;
|
2018-08-03 07:58:17 +00:00
|
|
|
goto err_xdp;
|
|
|
|
}
|
2020-03-19 16:41:26 +00:00
|
|
|
stats->xdp_tx++;
|
2018-08-03 07:58:17 +00:00
|
|
|
rcu_read_unlock();
|
|
|
|
goto xdp_xmit;
|
|
|
|
case XDP_REDIRECT:
|
|
|
|
orig_frame = *frame;
|
|
|
|
xdp.rxq->mem = frame->mem;
|
2018-08-03 07:58:18 +00:00
|
|
|
if (xdp_do_redirect(rq->dev, &xdp, xdp_prog)) {
|
2018-08-03 07:58:17 +00:00
|
|
|
frame = &orig_frame;
|
2020-03-19 16:41:26 +00:00
|
|
|
stats->rx_drops++;
|
2018-08-03 07:58:17 +00:00
|
|
|
goto err_xdp;
|
|
|
|
}
|
2020-03-19 16:41:26 +00:00
|
|
|
stats->xdp_redirect++;
|
2018-08-03 07:58:17 +00:00
|
|
|
rcu_read_unlock();
|
|
|
|
goto xdp_xmit;
|
2018-08-03 07:58:13 +00:00
|
|
|
default:
|
2021-11-30 10:08:07 +00:00
|
|
|
bpf_warn_invalid_xdp_action(rq->dev, xdp_prog, act);
|
2020-08-23 22:36:59 +00:00
|
|
|
fallthrough;
|
2018-08-03 07:58:13 +00:00
|
|
|
case XDP_ABORTED:
|
2018-08-03 07:58:18 +00:00
|
|
|
trace_xdp_exception(rq->dev, xdp_prog, act);
|
2020-08-23 22:36:59 +00:00
|
|
|
fallthrough;
|
2018-08-03 07:58:13 +00:00
|
|
|
case XDP_DROP:
|
2020-03-19 16:41:26 +00:00
|
|
|
stats->xdp_drops++;
|
2018-08-03 07:58:13 +00:00
|
|
|
goto err_xdp;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
rcu_read_unlock();
|
|
|
|
|
2021-01-29 22:04:08 +00:00
|
|
|
return frame;
|
2018-08-03 07:58:13 +00:00
|
|
|
err_xdp:
|
|
|
|
rcu_read_unlock();
|
|
|
|
xdp_return_frame(frame);
|
2018-08-03 07:58:17 +00:00
|
|
|
xdp_xmit:
|
2018-08-03 07:58:13 +00:00
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2021-01-29 22:04:08 +00:00
|
|
|
/* frames array contains VETH_XDP_BATCH at most */
|
|
|
|
static void veth_xdp_rcv_bulk_skb(struct veth_rq *rq, void **frames,
|
|
|
|
int n_xdpf, struct veth_xdp_tx_bq *bq,
|
|
|
|
struct veth_stats *stats)
|
|
|
|
{
|
|
|
|
void *skbs[VETH_XDP_BATCH];
|
|
|
|
int i;
|
|
|
|
|
|
|
|
if (xdp_alloc_skb_bulk(skbs, n_xdpf,
|
|
|
|
GFP_ATOMIC | __GFP_ZERO) < 0) {
|
|
|
|
for (i = 0; i < n_xdpf; i++)
|
|
|
|
xdp_return_frame(frames[i]);
|
|
|
|
stats->rx_drops += n_xdpf;
|
|
|
|
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
for (i = 0; i < n_xdpf; i++) {
|
|
|
|
struct sk_buff *skb = skbs[i];
|
|
|
|
|
|
|
|
skb = __xdp_build_skb_from_frame(frames[i], skb,
|
|
|
|
rq->dev);
|
|
|
|
if (!skb) {
|
|
|
|
xdp_return_frame(frames[i]);
|
|
|
|
stats->rx_drops++;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
napi_gro_receive(&rq->xdp_napi, skb);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-03-19 16:41:26 +00:00
|
|
|
static struct sk_buff *veth_xdp_rcv_skb(struct veth_rq *rq,
|
|
|
|
struct sk_buff *skb,
|
|
|
|
struct veth_xdp_tx_bq *bq,
|
|
|
|
struct veth_stats *stats)
|
2018-08-03 07:58:10 +00:00
|
|
|
{
|
2020-12-22 21:09:28 +00:00
|
|
|
u32 pktlen, headroom, act, metalen, frame_sz;
|
2018-08-03 07:58:10 +00:00
|
|
|
void *orig_data, *orig_data_end;
|
|
|
|
struct bpf_prog *xdp_prog;
|
|
|
|
int mac_len, delta, off;
|
|
|
|
struct xdp_buff xdp;
|
|
|
|
|
2021-07-28 16:24:04 +00:00
|
|
|
skb_prepare_for_gro(skb);
|
2018-09-14 04:33:44 +00:00
|
|
|
|
2018-08-03 07:58:10 +00:00
|
|
|
rcu_read_lock();
|
2018-08-03 07:58:18 +00:00
|
|
|
xdp_prog = rcu_dereference(rq->xdp_prog);
|
2018-08-03 07:58:10 +00:00
|
|
|
if (unlikely(!xdp_prog)) {
|
|
|
|
rcu_read_unlock();
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
|
|
|
mac_len = skb->data - skb_mac_header(skb);
|
|
|
|
pktlen = skb->len + mac_len;
|
|
|
|
headroom = skb_headroom(skb) - mac_len;
|
|
|
|
|
|
|
|
if (skb_shared(skb) || skb_head_is_locked(skb) ||
|
|
|
|
skb_is_nonlinear(skb) || headroom < XDP_PACKET_HEADROOM) {
|
|
|
|
struct sk_buff *nskb;
|
|
|
|
int size, head_off;
|
|
|
|
void *head, *start;
|
|
|
|
struct page *page;
|
|
|
|
|
|
|
|
size = SKB_DATA_ALIGN(VETH_XDP_HEADROOM + pktlen) +
|
|
|
|
SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
|
|
|
|
if (size > PAGE_SIZE)
|
|
|
|
goto drop;
|
|
|
|
|
|
|
|
page = alloc_page(GFP_ATOMIC | __GFP_NOWARN);
|
|
|
|
if (!page)
|
|
|
|
goto drop;
|
|
|
|
|
|
|
|
head = page_address(page);
|
|
|
|
start = head + VETH_XDP_HEADROOM;
|
|
|
|
if (skb_copy_bits(skb, -mac_len, start, pktlen)) {
|
|
|
|
page_frag_free(head);
|
|
|
|
goto drop;
|
|
|
|
}
|
|
|
|
|
2020-05-14 10:49:48 +00:00
|
|
|
nskb = veth_build_skb(head, VETH_XDP_HEADROOM + mac_len,
|
|
|
|
skb->len, PAGE_SIZE);
|
2018-08-03 07:58:10 +00:00
|
|
|
if (!nskb) {
|
|
|
|
page_frag_free(head);
|
|
|
|
goto drop;
|
|
|
|
}
|
|
|
|
|
|
|
|
skb_copy_header(nskb, skb);
|
|
|
|
head_off = skb_headroom(nskb) - skb_headroom(skb);
|
|
|
|
skb_headers_offset_update(nskb, head_off);
|
|
|
|
consume_skb(skb);
|
|
|
|
skb = nskb;
|
|
|
|
}
|
|
|
|
|
2020-05-14 10:49:48 +00:00
|
|
|
/* SKB "head" area always have tailroom for skb_shared_info */
|
2020-12-22 21:09:29 +00:00
|
|
|
frame_sz = skb_end_pointer(skb) - skb->head;
|
2020-12-22 21:09:28 +00:00
|
|
|
frame_sz += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
|
|
|
|
xdp_init_buff(&xdp, frame_sz, &rq->xdp_rxq);
|
2020-12-22 21:09:29 +00:00
|
|
|
xdp_prepare_buff(&xdp, skb->head, skb->mac_header, pktlen, true);
|
2020-05-14 10:49:48 +00:00
|
|
|
|
2018-08-03 07:58:10 +00:00
|
|
|
orig_data = xdp.data;
|
|
|
|
orig_data_end = xdp.data_end;
|
|
|
|
|
|
|
|
act = bpf_prog_run_xdp(xdp_prog, &xdp);
|
|
|
|
|
|
|
|
switch (act) {
|
|
|
|
case XDP_PASS:
|
|
|
|
break;
|
2018-08-03 07:58:17 +00:00
|
|
|
case XDP_TX:
|
|
|
|
get_page(virt_to_page(xdp.data));
|
|
|
|
consume_skb(skb);
|
2018-08-03 07:58:18 +00:00
|
|
|
xdp.rxq->mem = rq->xdp_mem;
|
2020-03-26 22:10:19 +00:00
|
|
|
if (unlikely(veth_xdp_tx(rq, &xdp, bq) < 0)) {
|
2018-08-03 07:58:18 +00:00
|
|
|
trace_xdp_exception(rq->dev, xdp_prog, act);
|
2020-03-19 16:41:26 +00:00
|
|
|
stats->rx_drops++;
|
2018-08-03 07:58:17 +00:00
|
|
|
goto err_xdp;
|
|
|
|
}
|
2020-03-19 16:41:26 +00:00
|
|
|
stats->xdp_tx++;
|
2018-08-03 07:58:17 +00:00
|
|
|
rcu_read_unlock();
|
|
|
|
goto xdp_xmit;
|
|
|
|
case XDP_REDIRECT:
|
|
|
|
get_page(virt_to_page(xdp.data));
|
|
|
|
consume_skb(skb);
|
2018-08-03 07:58:18 +00:00
|
|
|
xdp.rxq->mem = rq->xdp_mem;
|
2020-03-19 16:41:26 +00:00
|
|
|
if (xdp_do_redirect(rq->dev, &xdp, xdp_prog)) {
|
|
|
|
stats->rx_drops++;
|
2018-08-03 07:58:17 +00:00
|
|
|
goto err_xdp;
|
2020-03-19 16:41:26 +00:00
|
|
|
}
|
|
|
|
stats->xdp_redirect++;
|
2018-08-03 07:58:17 +00:00
|
|
|
rcu_read_unlock();
|
|
|
|
goto xdp_xmit;
|
2018-08-03 07:58:10 +00:00
|
|
|
default:
|
2021-11-30 10:08:07 +00:00
|
|
|
bpf_warn_invalid_xdp_action(rq->dev, xdp_prog, act);
|
2020-08-23 22:36:59 +00:00
|
|
|
fallthrough;
|
2018-08-03 07:58:10 +00:00
|
|
|
case XDP_ABORTED:
|
2018-08-03 07:58:18 +00:00
|
|
|
trace_xdp_exception(rq->dev, xdp_prog, act);
|
2020-08-23 22:36:59 +00:00
|
|
|
fallthrough;
|
2018-08-03 07:58:10 +00:00
|
|
|
case XDP_DROP:
|
2020-03-19 16:41:26 +00:00
|
|
|
stats->xdp_drops++;
|
|
|
|
goto xdp_drop;
|
2018-08-03 07:58:10 +00:00
|
|
|
}
|
|
|
|
rcu_read_unlock();
|
|
|
|
|
2020-05-14 10:49:48 +00:00
|
|
|
/* check if bpf_xdp_adjust_head was used */
|
2018-08-03 07:58:10 +00:00
|
|
|
delta = orig_data - xdp.data;
|
|
|
|
off = mac_len + delta;
|
|
|
|
if (off > 0)
|
|
|
|
__skb_push(skb, off);
|
|
|
|
else if (off < 0)
|
|
|
|
__skb_pull(skb, -off);
|
|
|
|
skb->mac_header -= delta;
|
2020-05-14 10:49:48 +00:00
|
|
|
|
|
|
|
/* check if bpf_xdp_adjust_tail was used */
|
2018-08-03 07:58:10 +00:00
|
|
|
off = xdp.data_end - orig_data_end;
|
|
|
|
if (off != 0)
|
2020-05-14 10:49:48 +00:00
|
|
|
__skb_put(skb, off); /* positive on grow, negative on shrink */
|
2018-08-03 07:58:18 +00:00
|
|
|
skb->protocol = eth_type_trans(skb, rq->dev);
|
2018-08-03 07:58:10 +00:00
|
|
|
|
|
|
|
metalen = xdp.data - xdp.data_meta;
|
|
|
|
if (metalen)
|
|
|
|
skb_metadata_set(skb, metalen);
|
|
|
|
out:
|
|
|
|
return skb;
|
|
|
|
drop:
|
2020-03-19 16:41:26 +00:00
|
|
|
stats->rx_drops++;
|
|
|
|
xdp_drop:
|
2018-08-03 07:58:10 +00:00
|
|
|
rcu_read_unlock();
|
|
|
|
kfree_skb(skb);
|
|
|
|
return NULL;
|
2018-08-03 07:58:17 +00:00
|
|
|
err_xdp:
|
|
|
|
rcu_read_unlock();
|
|
|
|
page_frag_free(xdp.data);
|
|
|
|
xdp_xmit:
|
|
|
|
return NULL;
|
2018-08-03 07:58:10 +00:00
|
|
|
}
|
|
|
|
|
2020-03-19 16:41:26 +00:00
|
|
|
static int veth_xdp_rcv(struct veth_rq *rq, int budget,
|
|
|
|
struct veth_xdp_tx_bq *bq,
|
|
|
|
struct veth_stats *stats)
|
2018-08-03 07:58:10 +00:00
|
|
|
{
|
2021-01-29 22:04:08 +00:00
|
|
|
int i, done = 0, n_xdpf = 0;
|
|
|
|
void *xdpf[VETH_XDP_BATCH];
|
2018-08-03 07:58:10 +00:00
|
|
|
|
|
|
|
for (i = 0; i < budget; i++) {
|
2018-08-03 07:58:18 +00:00
|
|
|
void *ptr = __ptr_ring_consume(&rq->xdp_ring);
|
2018-08-03 07:58:10 +00:00
|
|
|
|
2018-08-03 07:58:13 +00:00
|
|
|
if (!ptr)
|
2018-08-03 07:58:10 +00:00
|
|
|
break;
|
|
|
|
|
2018-08-03 07:58:17 +00:00
|
|
|
if (veth_is_xdp_frame(ptr)) {
|
2021-01-29 22:04:08 +00:00
|
|
|
/* ndo_xdp_xmit */
|
2018-10-11 09:36:49 +00:00
|
|
|
struct xdp_frame *frame = veth_ptr_to_xdp(ptr);
|
|
|
|
|
2020-03-19 16:41:26 +00:00
|
|
|
stats->xdp_bytes += frame->len;
|
2021-01-29 22:04:08 +00:00
|
|
|
frame = veth_xdp_rcv_one(rq, frame, bq, stats);
|
|
|
|
if (frame) {
|
|
|
|
/* XDP_PASS */
|
|
|
|
xdpf[n_xdpf++] = frame;
|
|
|
|
if (n_xdpf == VETH_XDP_BATCH) {
|
|
|
|
veth_xdp_rcv_bulk_skb(rq, xdpf, n_xdpf,
|
|
|
|
bq, stats);
|
|
|
|
n_xdpf = 0;
|
|
|
|
}
|
|
|
|
}
|
2018-08-03 07:58:17 +00:00
|
|
|
} else {
|
2021-01-29 22:04:08 +00:00
|
|
|
/* ndo_start_xmit */
|
|
|
|
struct sk_buff *skb = ptr;
|
|
|
|
|
2020-03-19 16:41:26 +00:00
|
|
|
stats->xdp_bytes += skb->len;
|
|
|
|
skb = veth_xdp_rcv_skb(rq, skb, bq, stats);
|
2021-12-22 18:39:52 +00:00
|
|
|
if (skb) {
|
|
|
|
if (skb_shared(skb) || skb_unclone(skb, GFP_ATOMIC))
|
|
|
|
netif_receive_skb(skb);
|
|
|
|
else
|
|
|
|
napi_gro_receive(&rq->xdp_napi, skb);
|
|
|
|
}
|
2018-08-03 07:58:17 +00:00
|
|
|
}
|
2018-08-03 07:58:10 +00:00
|
|
|
done++;
|
|
|
|
}
|
|
|
|
|
2021-01-29 22:04:08 +00:00
|
|
|
if (n_xdpf)
|
|
|
|
veth_xdp_rcv_bulk_skb(rq, xdpf, n_xdpf, bq, stats);
|
|
|
|
|
2018-10-11 09:36:49 +00:00
|
|
|
u64_stats_update_begin(&rq->stats.syncp);
|
2020-03-19 16:41:28 +00:00
|
|
|
rq->stats.vs.xdp_redirect += stats->xdp_redirect;
|
2020-03-19 16:41:26 +00:00
|
|
|
rq->stats.vs.xdp_bytes += stats->xdp_bytes;
|
2020-03-19 16:41:27 +00:00
|
|
|
rq->stats.vs.xdp_drops += stats->xdp_drops;
|
|
|
|
rq->stats.vs.rx_drops += stats->rx_drops;
|
2020-03-19 16:41:25 +00:00
|
|
|
rq->stats.vs.xdp_packets += done;
|
2018-10-11 09:36:49 +00:00
|
|
|
u64_stats_update_end(&rq->stats.syncp);
|
|
|
|
|
2018-08-03 07:58:10 +00:00
|
|
|
return done;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int veth_poll(struct napi_struct *napi, int budget)
|
|
|
|
{
|
2018-08-03 07:58:18 +00:00
|
|
|
struct veth_rq *rq =
|
|
|
|
container_of(napi, struct veth_rq, xdp_napi);
|
2020-03-19 16:41:26 +00:00
|
|
|
struct veth_stats stats = {};
|
2019-06-13 09:39:59 +00:00
|
|
|
struct veth_xdp_tx_bq bq;
|
2018-08-03 07:58:10 +00:00
|
|
|
int done;
|
|
|
|
|
2019-06-13 09:39:59 +00:00
|
|
|
bq.count = 0;
|
|
|
|
|
2018-08-03 07:58:17 +00:00
|
|
|
xdp_set_return_frame_no_direct();
|
2020-03-19 16:41:26 +00:00
|
|
|
done = veth_xdp_rcv(rq, budget, &bq, &stats);
|
2018-08-03 07:58:10 +00:00
|
|
|
|
|
|
|
if (done < budget && napi_complete_done(napi, done)) {
|
|
|
|
/* Write rx_notify_masked before reading ptr_ring */
|
2018-08-03 07:58:18 +00:00
|
|
|
smp_store_mb(rq->rx_notify_masked, false);
|
|
|
|
if (unlikely(!__ptr_ring_empty(&rq->xdp_ring))) {
|
2022-02-08 23:28:22 +00:00
|
|
|
if (napi_schedule_prep(&rq->xdp_napi)) {
|
|
|
|
WRITE_ONCE(rq->rx_notify_masked, true);
|
|
|
|
__napi_schedule(&rq->xdp_napi);
|
|
|
|
}
|
2018-08-03 07:58:10 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-03-19 16:41:26 +00:00
|
|
|
if (stats.xdp_tx > 0)
|
2020-03-26 22:10:19 +00:00
|
|
|
veth_xdp_flush(rq, &bq);
|
2020-03-19 16:41:26 +00:00
|
|
|
if (stats.xdp_redirect > 0)
|
xdp: Use bulking for non-map XDP_REDIRECT and consolidate code paths
Since the bulk queue used by XDP_REDIRECT now lives in struct net_device,
we can re-use the bulking for the non-map version of the bpf_redirect()
helper. This is a simple matter of having xdp_do_redirect_slow() queue the
frame on the bulk queue instead of sending it out with __bpf_tx_xdp().
Unfortunately we can't make the bpf_redirect() helper return an error if
the ifindex doesn't exit (as bpf_redirect_map() does), because we don't
have a reference to the network namespace of the ingress device at the time
the helper is called. So we have to leave it as-is and keep the device
lookup in xdp_do_redirect_slow().
Since this leaves less reason to have the non-map redirect code in a
separate function, so we get rid of the xdp_do_redirect_slow() function
entirely. This does lose us the tracepoint disambiguation, but fortunately
the xdp_redirect and xdp_redirect_map tracepoints use the same tracepoint
entry structures. This means both can contain a map index, so we can just
amend the tracepoint definitions so we always emit the xdp_redirect(_err)
tracepoints, but with the map ID only populated if a map is present. This
means we retire the xdp_redirect_map(_err) tracepoints entirely, but keep
the definitions around in case someone is still listening for them.
With this change, the performance of the xdp_redirect sample program goes
from 5Mpps to 8.4Mpps (a 68% increase).
Since the flush functions are no longer map-specific, rename the flush()
functions to drop _map from their names. One of the renamed functions is
the xdp_do_flush_map() callback used in all the xdp-enabled drivers. To
keep from having to update all drivers, use a #define to keep the old name
working, and only update the virtual drivers in this patch.
Signed-off-by: Toke Høiland-Jørgensen <toke@redhat.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Acked-by: John Fastabend <john.fastabend@gmail.com>
Link: https://lore.kernel.org/bpf/157918768505.1458396.17518057312953572912.stgit@toke.dk
2020-01-16 15:14:45 +00:00
|
|
|
xdp_do_flush();
|
2018-08-03 07:58:17 +00:00
|
|
|
xdp_clear_return_frame_no_direct();
|
|
|
|
|
2018-08-03 07:58:10 +00:00
|
|
|
return done;
|
|
|
|
}
|
|
|
|
|
2021-07-20 08:41:49 +00:00
|
|
|
static int __veth_napi_enable_range(struct net_device *dev, int start, int end)
|
2018-08-03 07:58:10 +00:00
|
|
|
{
|
|
|
|
struct veth_priv *priv = netdev_priv(dev);
|
2018-08-03 07:58:18 +00:00
|
|
|
int err, i;
|
2018-08-03 07:58:10 +00:00
|
|
|
|
2021-07-20 08:41:49 +00:00
|
|
|
for (i = start; i < end; i++) {
|
2018-08-03 07:58:18 +00:00
|
|
|
struct veth_rq *rq = &priv->rq[i];
|
|
|
|
|
|
|
|
err = ptr_ring_init(&rq->xdp_ring, VETH_RING_SIZE, GFP_KERNEL);
|
|
|
|
if (err)
|
|
|
|
goto err_xdp_ring;
|
|
|
|
}
|
2018-08-03 07:58:10 +00:00
|
|
|
|
2021-07-20 08:41:49 +00:00
|
|
|
for (i = start; i < end; i++) {
|
2018-08-03 07:58:18 +00:00
|
|
|
struct veth_rq *rq = &priv->rq[i];
|
|
|
|
|
|
|
|
napi_enable(&rq->xdp_napi);
|
2021-04-09 11:04:38 +00:00
|
|
|
rcu_assign_pointer(priv->rq[i].napi, &priv->rq[i].xdp_napi);
|
2018-08-03 07:58:18 +00:00
|
|
|
}
|
2018-08-03 07:58:10 +00:00
|
|
|
|
|
|
|
return 0;
|
2021-07-20 08:41:49 +00:00
|
|
|
|
2018-08-03 07:58:18 +00:00
|
|
|
err_xdp_ring:
|
2021-07-20 08:41:49 +00:00
|
|
|
for (i--; i >= start; i--)
|
2018-08-03 07:58:18 +00:00
|
|
|
ptr_ring_cleanup(&priv->rq[i].xdp_ring, veth_ptr_free);
|
|
|
|
|
|
|
|
return err;
|
2018-08-03 07:58:10 +00:00
|
|
|
}
|
|
|
|
|
2021-07-20 08:41:49 +00:00
|
|
|
static int __veth_napi_enable(struct net_device *dev)
|
|
|
|
{
|
|
|
|
return __veth_napi_enable_range(dev, 0, dev->real_num_rx_queues);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void veth_napi_del_range(struct net_device *dev, int start, int end)
|
2018-08-03 07:58:10 +00:00
|
|
|
{
|
|
|
|
struct veth_priv *priv = netdev_priv(dev);
|
2018-08-03 07:58:18 +00:00
|
|
|
int i;
|
2018-08-03 07:58:10 +00:00
|
|
|
|
2021-07-20 08:41:49 +00:00
|
|
|
for (i = start; i < end; i++) {
|
2018-08-03 07:58:18 +00:00
|
|
|
struct veth_rq *rq = &priv->rq[i];
|
|
|
|
|
2021-04-09 11:04:38 +00:00
|
|
|
rcu_assign_pointer(priv->rq[i].napi, NULL);
|
2018-08-03 07:58:18 +00:00
|
|
|
napi_disable(&rq->xdp_napi);
|
2020-09-09 17:37:51 +00:00
|
|
|
__netif_napi_del(&rq->xdp_napi);
|
2018-08-03 07:58:18 +00:00
|
|
|
}
|
|
|
|
synchronize_net();
|
|
|
|
|
2021-07-20 08:41:49 +00:00
|
|
|
for (i = start; i < end; i++) {
|
2018-08-03 07:58:18 +00:00
|
|
|
struct veth_rq *rq = &priv->rq[i];
|
|
|
|
|
|
|
|
rq->rx_notify_masked = false;
|
|
|
|
ptr_ring_cleanup(&rq->xdp_ring, veth_ptr_free);
|
|
|
|
}
|
2018-08-03 07:58:10 +00:00
|
|
|
}
|
|
|
|
|
2021-07-20 08:41:49 +00:00
|
|
|
static void veth_napi_del(struct net_device *dev)
|
|
|
|
{
|
|
|
|
veth_napi_del_range(dev, 0, dev->real_num_rx_queues);
|
|
|
|
}
|
|
|
|
|
2021-04-09 11:04:38 +00:00
|
|
|
static bool veth_gro_requested(const struct net_device *dev)
|
|
|
|
{
|
|
|
|
return !!(dev->wanted_features & NETIF_F_GRO);
|
|
|
|
}
|
|
|
|
|
2021-07-20 08:41:49 +00:00
|
|
|
static int veth_enable_xdp_range(struct net_device *dev, int start, int end,
|
|
|
|
bool napi_already_on)
|
2018-08-03 07:58:10 +00:00
|
|
|
{
|
|
|
|
struct veth_priv *priv = netdev_priv(dev);
|
2018-08-03 07:58:18 +00:00
|
|
|
int err, i;
|
2018-08-03 07:58:10 +00:00
|
|
|
|
2021-07-20 08:41:49 +00:00
|
|
|
for (i = start; i < end; i++) {
|
|
|
|
struct veth_rq *rq = &priv->rq[i];
|
2018-08-03 07:58:10 +00:00
|
|
|
|
2021-07-20 08:41:49 +00:00
|
|
|
if (!napi_already_on)
|
|
|
|
netif_napi_add(dev, &rq->xdp_napi, veth_poll, NAPI_POLL_WEIGHT);
|
|
|
|
err = xdp_rxq_info_reg(&rq->xdp_rxq, dev, i, rq->xdp_napi.napi_id);
|
|
|
|
if (err < 0)
|
|
|
|
goto err_rxq_reg;
|
|
|
|
|
|
|
|
err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq,
|
|
|
|
MEM_TYPE_PAGE_SHARED,
|
|
|
|
NULL);
|
|
|
|
if (err < 0)
|
|
|
|
goto err_reg_mem;
|
|
|
|
|
|
|
|
/* Save original mem info as it can be overwritten */
|
|
|
|
rq->xdp_mem = rq->xdp_rxq.mem;
|
|
|
|
}
|
|
|
|
return 0;
|
2018-08-03 07:58:18 +00:00
|
|
|
|
2021-07-20 08:41:49 +00:00
|
|
|
err_reg_mem:
|
|
|
|
xdp_rxq_info_unreg(&priv->rq[i].xdp_rxq);
|
|
|
|
err_rxq_reg:
|
|
|
|
for (i--; i >= start; i--) {
|
|
|
|
struct veth_rq *rq = &priv->rq[i];
|
2018-08-03 07:58:18 +00:00
|
|
|
|
2021-07-20 08:41:49 +00:00
|
|
|
xdp_rxq_info_unreg(&rq->xdp_rxq);
|
|
|
|
if (!napi_already_on)
|
|
|
|
netif_napi_del(&rq->xdp_napi);
|
|
|
|
}
|
|
|
|
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void veth_disable_xdp_range(struct net_device *dev, int start, int end,
|
|
|
|
bool delete_napi)
|
|
|
|
{
|
|
|
|
struct veth_priv *priv = netdev_priv(dev);
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = start; i < end; i++) {
|
|
|
|
struct veth_rq *rq = &priv->rq[i];
|
|
|
|
|
|
|
|
rq->xdp_rxq.mem = rq->xdp_mem;
|
|
|
|
xdp_rxq_info_unreg(&rq->xdp_rxq);
|
|
|
|
|
|
|
|
if (delete_napi)
|
|
|
|
netif_napi_del(&rq->xdp_napi);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static int veth_enable_xdp(struct net_device *dev)
|
|
|
|
{
|
|
|
|
bool napi_already_on = veth_gro_requested(dev) && (dev->flags & IFF_UP);
|
|
|
|
struct veth_priv *priv = netdev_priv(dev);
|
|
|
|
int err, i;
|
|
|
|
|
|
|
|
if (!xdp_rxq_info_is_reg(&priv->rq[0].xdp_rxq)) {
|
|
|
|
err = veth_enable_xdp_range(dev, 0, dev->real_num_rx_queues, napi_already_on);
|
|
|
|
if (err)
|
|
|
|
return err;
|
2018-08-03 07:58:10 +00:00
|
|
|
|
2021-04-09 11:04:38 +00:00
|
|
|
if (!napi_already_on) {
|
|
|
|
err = __veth_napi_enable(dev);
|
2021-07-20 08:41:49 +00:00
|
|
|
if (err) {
|
|
|
|
veth_disable_xdp_range(dev, 0, dev->real_num_rx_queues, true);
|
|
|
|
return err;
|
|
|
|
}
|
2021-04-09 11:04:38 +00:00
|
|
|
|
|
|
|
if (!veth_gro_requested(dev)) {
|
|
|
|
/* user-space did not require GRO, but adding XDP
|
|
|
|
* is supposed to get GRO working
|
|
|
|
*/
|
|
|
|
dev->features |= NETIF_F_GRO;
|
|
|
|
netdev_features_change(dev);
|
|
|
|
}
|
|
|
|
}
|
2018-08-03 07:58:10 +00:00
|
|
|
}
|
|
|
|
|
2021-04-09 11:04:38 +00:00
|
|
|
for (i = 0; i < dev->real_num_rx_queues; i++) {
|
2018-08-03 07:58:18 +00:00
|
|
|
rcu_assign_pointer(priv->rq[i].xdp_prog, priv->_xdp_prog);
|
2021-04-09 11:04:38 +00:00
|
|
|
rcu_assign_pointer(priv->rq[i].napi, &priv->rq[i].xdp_napi);
|
|
|
|
}
|
2018-08-03 07:58:10 +00:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void veth_disable_xdp(struct net_device *dev)
|
|
|
|
{
|
|
|
|
struct veth_priv *priv = netdev_priv(dev);
|
2018-08-03 07:58:18 +00:00
|
|
|
int i;
|
2018-08-03 07:58:10 +00:00
|
|
|
|
2018-08-03 07:58:18 +00:00
|
|
|
for (i = 0; i < dev->real_num_rx_queues; i++)
|
|
|
|
rcu_assign_pointer(priv->rq[i].xdp_prog, NULL);
|
2021-04-09 11:04:38 +00:00
|
|
|
|
|
|
|
if (!netif_running(dev) || !veth_gro_requested(dev)) {
|
|
|
|
veth_napi_del(dev);
|
|
|
|
|
|
|
|
/* if user-space did not require GRO, since adding XDP
|
|
|
|
* enabled it, clear it now
|
|
|
|
*/
|
|
|
|
if (!veth_gro_requested(dev) && netif_running(dev)) {
|
|
|
|
dev->features &= ~NETIF_F_GRO;
|
|
|
|
netdev_features_change(dev);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-07-20 08:41:49 +00:00
|
|
|
veth_disable_xdp_range(dev, 0, dev->real_num_rx_queues, false);
|
2018-08-03 07:58:10 +00:00
|
|
|
}
|
|
|
|
|
2021-07-20 08:41:49 +00:00
|
|
|
static int veth_napi_enable_range(struct net_device *dev, int start, int end)
|
2021-04-09 11:04:38 +00:00
|
|
|
{
|
|
|
|
struct veth_priv *priv = netdev_priv(dev);
|
|
|
|
int err, i;
|
|
|
|
|
2021-07-20 08:41:49 +00:00
|
|
|
for (i = start; i < end; i++) {
|
2021-04-09 11:04:38 +00:00
|
|
|
struct veth_rq *rq = &priv->rq[i];
|
|
|
|
|
|
|
|
netif_napi_add(dev, &rq->xdp_napi, veth_poll, NAPI_POLL_WEIGHT);
|
|
|
|
}
|
|
|
|
|
2021-07-20 08:41:49 +00:00
|
|
|
err = __veth_napi_enable_range(dev, start, end);
|
2021-04-09 11:04:38 +00:00
|
|
|
if (err) {
|
2021-07-20 08:41:49 +00:00
|
|
|
for (i = start; i < end; i++) {
|
2021-04-09 11:04:38 +00:00
|
|
|
struct veth_rq *rq = &priv->rq[i];
|
|
|
|
|
|
|
|
netif_napi_del(&rq->xdp_napi);
|
|
|
|
}
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
2021-07-20 08:41:49 +00:00
|
|
|
static int veth_napi_enable(struct net_device *dev)
|
|
|
|
{
|
|
|
|
return veth_napi_enable_range(dev, 0, dev->real_num_rx_queues);
|
|
|
|
}
|
|
|
|
|
2021-07-20 08:41:50 +00:00
|
|
|
static void veth_disable_range_safe(struct net_device *dev, int start, int end)
|
|
|
|
{
|
|
|
|
struct veth_priv *priv = netdev_priv(dev);
|
|
|
|
|
|
|
|
if (start >= end)
|
|
|
|
return;
|
|
|
|
|
|
|
|
if (priv->_xdp_prog) {
|
|
|
|
veth_napi_del_range(dev, start, end);
|
|
|
|
veth_disable_xdp_range(dev, start, end, false);
|
|
|
|
} else if (veth_gro_requested(dev)) {
|
|
|
|
veth_napi_del_range(dev, start, end);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static int veth_enable_range_safe(struct net_device *dev, int start, int end)
|
|
|
|
{
|
|
|
|
struct veth_priv *priv = netdev_priv(dev);
|
|
|
|
int err;
|
|
|
|
|
|
|
|
if (start >= end)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
if (priv->_xdp_prog) {
|
|
|
|
/* these channels are freshly initialized, napi is not on there even
|
|
|
|
* when GRO is requeste
|
|
|
|
*/
|
|
|
|
err = veth_enable_xdp_range(dev, start, end, false);
|
|
|
|
if (err)
|
|
|
|
return err;
|
|
|
|
|
|
|
|
err = __veth_napi_enable_range(dev, start, end);
|
|
|
|
if (err) {
|
|
|
|
/* on error always delete the newly added napis */
|
|
|
|
veth_disable_xdp_range(dev, start, end, true);
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
} else if (veth_gro_requested(dev)) {
|
|
|
|
return veth_napi_enable_range(dev, start, end);
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int veth_set_channels(struct net_device *dev,
|
|
|
|
struct ethtool_channels *ch)
|
|
|
|
{
|
|
|
|
struct veth_priv *priv = netdev_priv(dev);
|
|
|
|
unsigned int old_rx_count, new_rx_count;
|
|
|
|
struct veth_priv *peer_priv;
|
|
|
|
struct net_device *peer;
|
|
|
|
int err;
|
|
|
|
|
|
|
|
/* sanity check. Upper bounds are already enforced by the caller */
|
|
|
|
if (!ch->rx_count || !ch->tx_count)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
/* avoid braking XDP, if that is enabled */
|
|
|
|
peer = rtnl_dereference(priv->peer);
|
|
|
|
peer_priv = peer ? netdev_priv(peer) : NULL;
|
|
|
|
if (priv->_xdp_prog && peer && ch->rx_count < peer->real_num_tx_queues)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
if (peer && peer_priv && peer_priv->_xdp_prog && ch->tx_count > peer->real_num_rx_queues)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
old_rx_count = dev->real_num_rx_queues;
|
|
|
|
new_rx_count = ch->rx_count;
|
|
|
|
if (netif_running(dev)) {
|
|
|
|
/* turn device off */
|
|
|
|
netif_carrier_off(dev);
|
|
|
|
if (peer)
|
|
|
|
netif_carrier_off(peer);
|
|
|
|
|
|
|
|
/* try to allocate new resurces, as needed*/
|
|
|
|
err = veth_enable_range_safe(dev, old_rx_count, new_rx_count);
|
|
|
|
if (err)
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
|
|
|
err = netif_set_real_num_rx_queues(dev, ch->rx_count);
|
|
|
|
if (err)
|
|
|
|
goto revert;
|
|
|
|
|
|
|
|
err = netif_set_real_num_tx_queues(dev, ch->tx_count);
|
|
|
|
if (err) {
|
|
|
|
int err2 = netif_set_real_num_rx_queues(dev, old_rx_count);
|
|
|
|
|
|
|
|
/* this error condition could happen only if rx and tx change
|
|
|
|
* in opposite directions (e.g. tx nr raises, rx nr decreases)
|
|
|
|
* and we can't do anything to fully restore the original
|
|
|
|
* status
|
|
|
|
*/
|
|
|
|
if (err2)
|
|
|
|
pr_warn("Can't restore rx queues config %d -> %d %d",
|
|
|
|
new_rx_count, old_rx_count, err2);
|
|
|
|
else
|
|
|
|
goto revert;
|
|
|
|
}
|
|
|
|
|
|
|
|
out:
|
|
|
|
if (netif_running(dev)) {
|
|
|
|
/* note that we need to swap the arguments WRT the enable part
|
|
|
|
* to identify the range we have to disable
|
|
|
|
*/
|
|
|
|
veth_disable_range_safe(dev, new_rx_count, old_rx_count);
|
|
|
|
netif_carrier_on(dev);
|
|
|
|
if (peer)
|
|
|
|
netif_carrier_on(peer);
|
|
|
|
}
|
|
|
|
return err;
|
|
|
|
|
|
|
|
revert:
|
|
|
|
new_rx_count = old_rx_count;
|
|
|
|
old_rx_count = ch->rx_count;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
2007-09-25 23:14:46 +00:00
|
|
|
static int veth_open(struct net_device *dev)
|
|
|
|
{
|
2013-01-04 15:42:40 +00:00
|
|
|
struct veth_priv *priv = netdev_priv(dev);
|
|
|
|
struct net_device *peer = rtnl_dereference(priv->peer);
|
2018-08-03 07:58:10 +00:00
|
|
|
int err;
|
2007-09-25 23:14:46 +00:00
|
|
|
|
2013-01-04 15:42:40 +00:00
|
|
|
if (!peer)
|
2007-09-25 23:14:46 +00:00
|
|
|
return -ENOTCONN;
|
|
|
|
|
2018-08-03 07:58:10 +00:00
|
|
|
if (priv->_xdp_prog) {
|
|
|
|
err = veth_enable_xdp(dev);
|
|
|
|
if (err)
|
|
|
|
return err;
|
2021-04-09 11:04:38 +00:00
|
|
|
} else if (veth_gro_requested(dev)) {
|
|
|
|
err = veth_napi_enable(dev);
|
|
|
|
if (err)
|
|
|
|
return err;
|
2018-08-03 07:58:10 +00:00
|
|
|
}
|
|
|
|
|
2013-01-04 15:42:40 +00:00
|
|
|
if (peer->flags & IFF_UP) {
|
2007-09-25 23:14:46 +00:00
|
|
|
netif_carrier_on(dev);
|
2013-01-04 15:42:40 +00:00
|
|
|
netif_carrier_on(peer);
|
2007-09-25 23:14:46 +00:00
|
|
|
}
|
2018-08-03 07:58:10 +00:00
|
|
|
|
2007-09-25 23:14:46 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2009-02-25 19:47:29 +00:00
|
|
|
static int veth_close(struct net_device *dev)
|
|
|
|
{
|
|
|
|
struct veth_priv *priv = netdev_priv(dev);
|
2013-01-10 08:32:45 +00:00
|
|
|
struct net_device *peer = rtnl_dereference(priv->peer);
|
2009-02-25 19:47:29 +00:00
|
|
|
|
|
|
|
netif_carrier_off(dev);
|
2013-01-10 08:32:45 +00:00
|
|
|
if (peer)
|
|
|
|
netif_carrier_off(peer);
|
2009-02-25 19:47:29 +00:00
|
|
|
|
2018-08-03 07:58:10 +00:00
|
|
|
if (priv->_xdp_prog)
|
|
|
|
veth_disable_xdp(dev);
|
2021-04-09 11:04:38 +00:00
|
|
|
else if (veth_gro_requested(dev))
|
|
|
|
veth_napi_del(dev);
|
2018-08-03 07:58:10 +00:00
|
|
|
|
2009-02-25 19:47:29 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
net: use core MTU range checking in core net infra
geneve:
- Merge __geneve_change_mtu back into geneve_change_mtu, set max_mtu
- This one isn't quite as straight-forward as others, could use some
closer inspection and testing
macvlan:
- set min/max_mtu
tun:
- set min/max_mtu, remove tun_net_change_mtu
vxlan:
- Merge __vxlan_change_mtu back into vxlan_change_mtu
- Set max_mtu to IP_MAX_MTU and retain dynamic MTU range checks in
change_mtu function
- This one is also not as straight-forward and could use closer inspection
and testing from vxlan folks
bridge:
- set max_mtu of IP_MAX_MTU and retain dynamic MTU range checks in
change_mtu function
openvswitch:
- set min/max_mtu, remove internal_dev_change_mtu
- note: max_mtu wasn't checked previously, it's been set to 65535, which
is the largest possible size supported
sch_teql:
- set min/max_mtu (note: max_mtu previously unchecked, used max of 65535)
macsec:
- min_mtu = 0, max_mtu = 65535
macvlan:
- min_mtu = 0, max_mtu = 65535
ntb_netdev:
- min_mtu = 0, max_mtu = 65535
veth:
- min_mtu = 68, max_mtu = 65535
8021q:
- min_mtu = 0, max_mtu = 65535
CC: netdev@vger.kernel.org
CC: Nicolas Dichtel <nicolas.dichtel@6wind.com>
CC: Hannes Frederic Sowa <hannes@stressinduktion.org>
CC: Tom Herbert <tom@herbertland.com>
CC: Daniel Borkmann <daniel@iogearbox.net>
CC: Alexander Duyck <alexander.h.duyck@intel.com>
CC: Paolo Abeni <pabeni@redhat.com>
CC: Jiri Benc <jbenc@redhat.com>
CC: WANG Cong <xiyou.wangcong@gmail.com>
CC: Roopa Prabhu <roopa@cumulusnetworks.com>
CC: Pravin B Shelar <pshelar@ovn.org>
CC: Sabrina Dubroca <sd@queasysnail.net>
CC: Patrick McHardy <kaber@trash.net>
CC: Stephen Hemminger <stephen@networkplumber.org>
CC: Pravin Shelar <pshelar@nicira.com>
CC: Maxim Krasnyansky <maxk@qti.qualcomm.com>
Signed-off-by: Jarod Wilson <jarod@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-10-20 17:55:20 +00:00
|
|
|
static int is_valid_veth_mtu(int mtu)
|
2009-03-04 07:36:04 +00:00
|
|
|
{
|
net: use core MTU range checking in core net infra
geneve:
- Merge __geneve_change_mtu back into geneve_change_mtu, set max_mtu
- This one isn't quite as straight-forward as others, could use some
closer inspection and testing
macvlan:
- set min/max_mtu
tun:
- set min/max_mtu, remove tun_net_change_mtu
vxlan:
- Merge __vxlan_change_mtu back into vxlan_change_mtu
- Set max_mtu to IP_MAX_MTU and retain dynamic MTU range checks in
change_mtu function
- This one is also not as straight-forward and could use closer inspection
and testing from vxlan folks
bridge:
- set max_mtu of IP_MAX_MTU and retain dynamic MTU range checks in
change_mtu function
openvswitch:
- set min/max_mtu, remove internal_dev_change_mtu
- note: max_mtu wasn't checked previously, it's been set to 65535, which
is the largest possible size supported
sch_teql:
- set min/max_mtu (note: max_mtu previously unchecked, used max of 65535)
macsec:
- min_mtu = 0, max_mtu = 65535
macvlan:
- min_mtu = 0, max_mtu = 65535
ntb_netdev:
- min_mtu = 0, max_mtu = 65535
veth:
- min_mtu = 68, max_mtu = 65535
8021q:
- min_mtu = 0, max_mtu = 65535
CC: netdev@vger.kernel.org
CC: Nicolas Dichtel <nicolas.dichtel@6wind.com>
CC: Hannes Frederic Sowa <hannes@stressinduktion.org>
CC: Tom Herbert <tom@herbertland.com>
CC: Daniel Borkmann <daniel@iogearbox.net>
CC: Alexander Duyck <alexander.h.duyck@intel.com>
CC: Paolo Abeni <pabeni@redhat.com>
CC: Jiri Benc <jbenc@redhat.com>
CC: WANG Cong <xiyou.wangcong@gmail.com>
CC: Roopa Prabhu <roopa@cumulusnetworks.com>
CC: Pravin B Shelar <pshelar@ovn.org>
CC: Sabrina Dubroca <sd@queasysnail.net>
CC: Patrick McHardy <kaber@trash.net>
CC: Stephen Hemminger <stephen@networkplumber.org>
CC: Pravin Shelar <pshelar@nicira.com>
CC: Maxim Krasnyansky <maxk@qti.qualcomm.com>
Signed-off-by: Jarod Wilson <jarod@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-10-20 17:55:20 +00:00
|
|
|
return mtu >= ETH_MIN_MTU && mtu <= ETH_MAX_MTU;
|
2009-03-04 07:36:04 +00:00
|
|
|
}
|
|
|
|
|
2018-08-15 08:07:29 +00:00
|
|
|
static int veth_alloc_queues(struct net_device *dev)
|
|
|
|
{
|
|
|
|
struct veth_priv *priv = netdev_priv(dev);
|
|
|
|
int i;
|
|
|
|
|
|
|
|
priv->rq = kcalloc(dev->num_rx_queues, sizeof(*priv->rq), GFP_KERNEL);
|
|
|
|
if (!priv->rq)
|
|
|
|
return -ENOMEM;
|
|
|
|
|
2018-10-11 09:36:49 +00:00
|
|
|
for (i = 0; i < dev->num_rx_queues; i++) {
|
2018-08-15 08:07:29 +00:00
|
|
|
priv->rq[i].dev = dev;
|
2018-10-11 09:36:49 +00:00
|
|
|
u64_stats_init(&priv->rq[i].stats.syncp);
|
|
|
|
}
|
2018-08-15 08:07:29 +00:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void veth_free_queues(struct net_device *dev)
|
|
|
|
{
|
|
|
|
struct veth_priv *priv = netdev_priv(dev);
|
|
|
|
|
|
|
|
kfree(priv->rq);
|
|
|
|
}
|
|
|
|
|
2007-09-25 23:14:46 +00:00
|
|
|
static int veth_dev_init(struct net_device *dev)
|
|
|
|
{
|
2018-08-15 08:07:29 +00:00
|
|
|
int err;
|
|
|
|
|
2018-09-17 10:46:55 +00:00
|
|
|
dev->lstats = netdev_alloc_pcpu_stats(struct pcpu_lstats);
|
|
|
|
if (!dev->lstats)
|
2007-09-25 23:14:46 +00:00
|
|
|
return -ENOMEM;
|
2018-08-15 08:07:29 +00:00
|
|
|
|
|
|
|
err = veth_alloc_queues(dev);
|
|
|
|
if (err) {
|
2018-09-17 10:46:55 +00:00
|
|
|
free_percpu(dev->lstats);
|
2018-08-15 08:07:29 +00:00
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
2007-09-25 23:14:46 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2009-06-25 09:45:42 +00:00
|
|
|
static void veth_dev_free(struct net_device *dev)
|
|
|
|
{
|
2018-08-15 08:07:29 +00:00
|
|
|
veth_free_queues(dev);
|
2018-09-17 10:46:55 +00:00
|
|
|
free_percpu(dev->lstats);
|
2009-06-25 09:45:42 +00:00
|
|
|
}
|
|
|
|
|
2014-06-23 22:36:02 +00:00
|
|
|
#ifdef CONFIG_NET_POLL_CONTROLLER
|
|
|
|
static void veth_poll_controller(struct net_device *dev)
|
|
|
|
{
|
|
|
|
/* veth only receives frames when its peer sends one
|
2018-08-03 07:58:10 +00:00
|
|
|
* Since it has nothing to do with disabling irqs, we are guaranteed
|
2014-06-23 22:36:02 +00:00
|
|
|
* never to have pending data when we poll for it so
|
|
|
|
* there is nothing to do here.
|
|
|
|
*
|
|
|
|
* We need this though so netpoll recognizes us as an interface that
|
|
|
|
* supports polling, which enables bridge devices in virt setups to
|
|
|
|
* still use netconsole
|
|
|
|
*/
|
|
|
|
}
|
|
|
|
#endif /* CONFIG_NET_POLL_CONTROLLER */
|
|
|
|
|
2015-04-02 15:07:11 +00:00
|
|
|
static int veth_get_iflink(const struct net_device *dev)
|
|
|
|
{
|
|
|
|
struct veth_priv *priv = netdev_priv(dev);
|
|
|
|
struct net_device *peer;
|
|
|
|
int iflink;
|
|
|
|
|
|
|
|
rcu_read_lock();
|
|
|
|
peer = rcu_dereference(priv->peer);
|
|
|
|
iflink = peer ? peer->ifindex : 0;
|
|
|
|
rcu_read_unlock();
|
|
|
|
|
|
|
|
return iflink;
|
|
|
|
}
|
|
|
|
|
2018-08-03 07:58:11 +00:00
|
|
|
static netdev_features_t veth_fix_features(struct net_device *dev,
|
|
|
|
netdev_features_t features)
|
|
|
|
{
|
|
|
|
struct veth_priv *priv = netdev_priv(dev);
|
|
|
|
struct net_device *peer;
|
|
|
|
|
|
|
|
peer = rtnl_dereference(priv->peer);
|
|
|
|
if (peer) {
|
|
|
|
struct veth_priv *peer_priv = netdev_priv(peer);
|
|
|
|
|
|
|
|
if (peer_priv->_xdp_prog)
|
|
|
|
features &= ~NETIF_F_GSO_SOFTWARE;
|
|
|
|
}
|
2021-04-09 11:04:38 +00:00
|
|
|
if (priv->_xdp_prog)
|
|
|
|
features |= NETIF_F_GRO;
|
2018-08-03 07:58:11 +00:00
|
|
|
|
|
|
|
return features;
|
|
|
|
}
|
|
|
|
|
2021-04-09 11:04:38 +00:00
|
|
|
static int veth_set_features(struct net_device *dev,
|
|
|
|
netdev_features_t features)
|
|
|
|
{
|
|
|
|
netdev_features_t changed = features ^ dev->features;
|
|
|
|
struct veth_priv *priv = netdev_priv(dev);
|
|
|
|
int err;
|
|
|
|
|
|
|
|
if (!(changed & NETIF_F_GRO) || !(dev->flags & IFF_UP) || priv->_xdp_prog)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
if (features & NETIF_F_GRO) {
|
|
|
|
err = veth_napi_enable(dev);
|
|
|
|
if (err)
|
|
|
|
return err;
|
|
|
|
} else {
|
|
|
|
veth_napi_del(dev);
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2016-02-26 09:45:41 +00:00
|
|
|
static void veth_set_rx_headroom(struct net_device *dev, int new_hr)
|
|
|
|
{
|
|
|
|
struct veth_priv *peer_priv, *priv = netdev_priv(dev);
|
|
|
|
struct net_device *peer;
|
|
|
|
|
|
|
|
if (new_hr < 0)
|
|
|
|
new_hr = 0;
|
|
|
|
|
|
|
|
rcu_read_lock();
|
|
|
|
peer = rcu_dereference(priv->peer);
|
|
|
|
if (unlikely(!peer))
|
|
|
|
goto out;
|
|
|
|
|
|
|
|
peer_priv = netdev_priv(peer);
|
|
|
|
priv->requested_headroom = new_hr;
|
|
|
|
new_hr = max(priv->requested_headroom, peer_priv->requested_headroom);
|
|
|
|
dev->needed_headroom = new_hr;
|
|
|
|
peer->needed_headroom = new_hr;
|
|
|
|
|
|
|
|
out:
|
|
|
|
rcu_read_unlock();
|
|
|
|
}
|
|
|
|
|
2018-08-03 07:58:10 +00:00
|
|
|
static int veth_xdp_set(struct net_device *dev, struct bpf_prog *prog,
|
|
|
|
struct netlink_ext_ack *extack)
|
|
|
|
{
|
|
|
|
struct veth_priv *priv = netdev_priv(dev);
|
|
|
|
struct bpf_prog *old_prog;
|
|
|
|
struct net_device *peer;
|
2018-08-03 07:58:11 +00:00
|
|
|
unsigned int max_mtu;
|
2018-08-03 07:58:10 +00:00
|
|
|
int err;
|
|
|
|
|
|
|
|
old_prog = priv->_xdp_prog;
|
|
|
|
priv->_xdp_prog = prog;
|
|
|
|
peer = rtnl_dereference(priv->peer);
|
|
|
|
|
|
|
|
if (prog) {
|
|
|
|
if (!peer) {
|
|
|
|
NL_SET_ERR_MSG_MOD(extack, "Cannot set XDP when peer is detached");
|
|
|
|
err = -ENOTCONN;
|
|
|
|
goto err;
|
|
|
|
}
|
|
|
|
|
2018-08-03 07:58:11 +00:00
|
|
|
max_mtu = PAGE_SIZE - VETH_XDP_HEADROOM -
|
|
|
|
peer->hard_header_len -
|
|
|
|
SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
|
|
|
|
if (peer->mtu > max_mtu) {
|
|
|
|
NL_SET_ERR_MSG_MOD(extack, "Peer MTU is too large to set XDP");
|
|
|
|
err = -ERANGE;
|
|
|
|
goto err;
|
|
|
|
}
|
|
|
|
|
2018-08-03 07:58:18 +00:00
|
|
|
if (dev->real_num_rx_queues < peer->real_num_tx_queues) {
|
|
|
|
NL_SET_ERR_MSG_MOD(extack, "XDP expects number of rx queues not less than peer tx queues");
|
|
|
|
err = -ENOSPC;
|
|
|
|
goto err;
|
|
|
|
}
|
|
|
|
|
2018-08-03 07:58:10 +00:00
|
|
|
if (dev->flags & IFF_UP) {
|
|
|
|
err = veth_enable_xdp(dev);
|
|
|
|
if (err) {
|
|
|
|
NL_SET_ERR_MSG_MOD(extack, "Setup for XDP failed");
|
|
|
|
goto err;
|
|
|
|
}
|
|
|
|
}
|
2018-08-03 07:58:11 +00:00
|
|
|
|
|
|
|
if (!old_prog) {
|
|
|
|
peer->hw_features &= ~NETIF_F_GSO_SOFTWARE;
|
|
|
|
peer->max_mtu = max_mtu;
|
|
|
|
}
|
2018-08-03 07:58:10 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if (old_prog) {
|
2018-08-03 07:58:11 +00:00
|
|
|
if (!prog) {
|
|
|
|
if (dev->flags & IFF_UP)
|
|
|
|
veth_disable_xdp(dev);
|
|
|
|
|
|
|
|
if (peer) {
|
|
|
|
peer->hw_features |= NETIF_F_GSO_SOFTWARE;
|
|
|
|
peer->max_mtu = ETH_MAX_MTU;
|
|
|
|
}
|
|
|
|
}
|
2018-08-03 07:58:10 +00:00
|
|
|
bpf_prog_put(old_prog);
|
|
|
|
}
|
|
|
|
|
2018-08-03 07:58:11 +00:00
|
|
|
if ((!!old_prog ^ !!prog) && peer)
|
|
|
|
netdev_update_features(peer);
|
|
|
|
|
2018-08-03 07:58:10 +00:00
|
|
|
return 0;
|
|
|
|
err:
|
|
|
|
priv->_xdp_prog = old_prog;
|
|
|
|
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int veth_xdp(struct net_device *dev, struct netdev_bpf *xdp)
|
|
|
|
{
|
|
|
|
switch (xdp->command) {
|
|
|
|
case XDP_SETUP_PROG:
|
|
|
|
return veth_xdp_set(dev, xdp->prog, xdp->extack);
|
|
|
|
default:
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2008-11-20 05:50:10 +00:00
|
|
|
static const struct net_device_ops veth_netdev_ops = {
|
2009-02-22 08:04:45 +00:00
|
|
|
.ndo_init = veth_dev_init,
|
|
|
|
.ndo_open = veth_open,
|
2009-02-25 19:47:29 +00:00
|
|
|
.ndo_stop = veth_close,
|
2009-02-22 08:04:45 +00:00
|
|
|
.ndo_start_xmit = veth_xmit,
|
2011-06-08 14:53:59 +00:00
|
|
|
.ndo_get_stats64 = veth_get_stats64,
|
2013-10-04 08:52:24 +00:00
|
|
|
.ndo_set_rx_mode = veth_set_multicast_list,
|
2009-02-22 08:04:45 +00:00
|
|
|
.ndo_set_mac_address = eth_mac_addr,
|
2014-06-23 22:36:02 +00:00
|
|
|
#ifdef CONFIG_NET_POLL_CONTROLLER
|
|
|
|
.ndo_poll_controller = veth_poll_controller,
|
|
|
|
#endif
|
2015-04-02 15:07:11 +00:00
|
|
|
.ndo_get_iflink = veth_get_iflink,
|
2018-08-03 07:58:11 +00:00
|
|
|
.ndo_fix_features = veth_fix_features,
|
2021-04-09 11:04:38 +00:00
|
|
|
.ndo_set_features = veth_set_features,
|
2015-07-31 06:03:25 +00:00
|
|
|
.ndo_features_check = passthru_features_check,
|
2016-02-26 09:45:41 +00:00
|
|
|
.ndo_set_rx_headroom = veth_set_rx_headroom,
|
2018-08-03 07:58:10 +00:00
|
|
|
.ndo_bpf = veth_xdp,
|
2020-03-19 16:41:28 +00:00
|
|
|
.ndo_xdp_xmit = veth_ndo_xdp_xmit,
|
2020-10-10 23:40:02 +00:00
|
|
|
.ndo_get_peer_dev = veth_peer_dev,
|
2008-11-20 05:50:10 +00:00
|
|
|
};
|
|
|
|
|
2016-04-19 18:02:26 +00:00
|
|
|
#define VETH_FEATURES (NETIF_F_SG | NETIF_F_FRAGLIST | NETIF_F_HW_CSUM | \
|
2016-08-25 05:21:49 +00:00
|
|
|
NETIF_F_RXCSUM | NETIF_F_SCTP_CRC | NETIF_F_HIGHDMA | \
|
2016-04-19 18:02:26 +00:00
|
|
|
NETIF_F_GSO_SOFTWARE | NETIF_F_GSO_ENCAP_ALL | \
|
2013-04-19 02:04:32 +00:00
|
|
|
NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX | \
|
|
|
|
NETIF_F_HW_VLAN_STAG_TX | NETIF_F_HW_VLAN_STAG_RX )
|
2012-12-29 16:26:10 +00:00
|
|
|
|
2007-09-25 23:14:46 +00:00
|
|
|
static void veth_setup(struct net_device *dev)
|
|
|
|
{
|
|
|
|
ether_setup(dev);
|
|
|
|
|
2011-07-26 06:05:38 +00:00
|
|
|
dev->priv_flags &= ~IFF_TX_SKB_SHARING;
|
2012-10-30 16:22:01 +00:00
|
|
|
dev->priv_flags |= IFF_LIVE_ADDR_CHANGE;
|
2015-08-18 08:30:29 +00:00
|
|
|
dev->priv_flags |= IFF_NO_QUEUE;
|
2016-02-26 09:45:41 +00:00
|
|
|
dev->priv_flags |= IFF_PHONY_HEADROOM;
|
2011-07-26 06:05:38 +00:00
|
|
|
|
2008-11-20 05:50:10 +00:00
|
|
|
dev->netdev_ops = &veth_netdev_ops;
|
2007-09-25 23:14:46 +00:00
|
|
|
dev->ethtool_ops = &veth_ethtool_ops;
|
|
|
|
dev->features |= NETIF_F_LLTX;
|
2012-12-29 16:26:10 +00:00
|
|
|
dev->features |= VETH_FEATURES;
|
2014-02-18 12:20:08 +00:00
|
|
|
dev->vlan_features = dev->features &
|
2014-03-28 02:14:48 +00:00
|
|
|
~(NETIF_F_HW_VLAN_CTAG_TX |
|
|
|
|
NETIF_F_HW_VLAN_STAG_TX |
|
|
|
|
NETIF_F_HW_VLAN_CTAG_RX |
|
|
|
|
NETIF_F_HW_VLAN_STAG_RX);
|
net: Fix inconsistent teardown and release of private netdev state.
Network devices can allocate reasources and private memory using
netdev_ops->ndo_init(). However, the release of these resources
can occur in one of two different places.
Either netdev_ops->ndo_uninit() or netdev->destructor().
The decision of which operation frees the resources depends upon
whether it is necessary for all netdev refs to be released before it
is safe to perform the freeing.
netdev_ops->ndo_uninit() presumably can occur right after the
NETDEV_UNREGISTER notifier completes and the unicast and multicast
address lists are flushed.
netdev->destructor(), on the other hand, does not run until the
netdev references all go away.
Further complicating the situation is that netdev->destructor()
almost universally does also a free_netdev().
This creates a problem for the logic in register_netdevice().
Because all callers of register_netdevice() manage the freeing
of the netdev, and invoke free_netdev(dev) if register_netdevice()
fails.
If netdev_ops->ndo_init() succeeds, but something else fails inside
of register_netdevice(), it does call ndo_ops->ndo_uninit(). But
it is not able to invoke netdev->destructor().
This is because netdev->destructor() will do a free_netdev() and
then the caller of register_netdevice() will do the same.
However, this means that the resources that would normally be released
by netdev->destructor() will not be.
Over the years drivers have added local hacks to deal with this, by
invoking their destructor parts by hand when register_netdevice()
fails.
Many drivers do not try to deal with this, and instead we have leaks.
Let's close this hole by formalizing the distinction between what
private things need to be freed up by netdev->destructor() and whether
the driver needs unregister_netdevice() to perform the free_netdev().
netdev->priv_destructor() performs all actions to free up the private
resources that used to be freed by netdev->destructor(), except for
free_netdev().
netdev->needs_free_netdev is a boolean that indicates whether
free_netdev() should be done at the end of unregister_netdevice().
Now, register_netdevice() can sanely release all resources after
ndo_ops->ndo_init() succeeds, by invoking both ndo_ops->ndo_uninit()
and netdev->priv_destructor().
And at the end of unregister_netdevice(), we invoke
netdev->priv_destructor() and optionally call free_netdev().
Signed-off-by: David S. Miller <davem@davemloft.net>
2017-05-08 16:52:56 +00:00
|
|
|
dev->needs_free_netdev = true;
|
|
|
|
dev->priv_destructor = veth_dev_free;
|
net: use core MTU range checking in core net infra
geneve:
- Merge __geneve_change_mtu back into geneve_change_mtu, set max_mtu
- This one isn't quite as straight-forward as others, could use some
closer inspection and testing
macvlan:
- set min/max_mtu
tun:
- set min/max_mtu, remove tun_net_change_mtu
vxlan:
- Merge __vxlan_change_mtu back into vxlan_change_mtu
- Set max_mtu to IP_MAX_MTU and retain dynamic MTU range checks in
change_mtu function
- This one is also not as straight-forward and could use closer inspection
and testing from vxlan folks
bridge:
- set max_mtu of IP_MAX_MTU and retain dynamic MTU range checks in
change_mtu function
openvswitch:
- set min/max_mtu, remove internal_dev_change_mtu
- note: max_mtu wasn't checked previously, it's been set to 65535, which
is the largest possible size supported
sch_teql:
- set min/max_mtu (note: max_mtu previously unchecked, used max of 65535)
macsec:
- min_mtu = 0, max_mtu = 65535
macvlan:
- min_mtu = 0, max_mtu = 65535
ntb_netdev:
- min_mtu = 0, max_mtu = 65535
veth:
- min_mtu = 68, max_mtu = 65535
8021q:
- min_mtu = 0, max_mtu = 65535
CC: netdev@vger.kernel.org
CC: Nicolas Dichtel <nicolas.dichtel@6wind.com>
CC: Hannes Frederic Sowa <hannes@stressinduktion.org>
CC: Tom Herbert <tom@herbertland.com>
CC: Daniel Borkmann <daniel@iogearbox.net>
CC: Alexander Duyck <alexander.h.duyck@intel.com>
CC: Paolo Abeni <pabeni@redhat.com>
CC: Jiri Benc <jbenc@redhat.com>
CC: WANG Cong <xiyou.wangcong@gmail.com>
CC: Roopa Prabhu <roopa@cumulusnetworks.com>
CC: Pravin B Shelar <pshelar@ovn.org>
CC: Sabrina Dubroca <sd@queasysnail.net>
CC: Patrick McHardy <kaber@trash.net>
CC: Stephen Hemminger <stephen@networkplumber.org>
CC: Pravin Shelar <pshelar@nicira.com>
CC: Maxim Krasnyansky <maxk@qti.qualcomm.com>
Signed-off-by: Jarod Wilson <jarod@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-10-20 17:55:20 +00:00
|
|
|
dev->max_mtu = ETH_MAX_MTU;
|
2011-03-31 01:01:35 +00:00
|
|
|
|
2012-12-29 16:26:10 +00:00
|
|
|
dev->hw_features = VETH_FEATURES;
|
2013-10-26 01:25:03 +00:00
|
|
|
dev->hw_enc_features = VETH_FEATURES;
|
2016-08-25 03:10:45 +00:00
|
|
|
dev->mpls_features = NETIF_F_HW_CSUM | NETIF_F_GSO_SOFTWARE;
|
2007-09-25 23:14:46 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* netlink interface
|
|
|
|
*/
|
|
|
|
|
2017-06-25 21:56:01 +00:00
|
|
|
static int veth_validate(struct nlattr *tb[], struct nlattr *data[],
|
|
|
|
struct netlink_ext_ack *extack)
|
2007-09-25 23:14:46 +00:00
|
|
|
{
|
|
|
|
if (tb[IFLA_ADDRESS]) {
|
|
|
|
if (nla_len(tb[IFLA_ADDRESS]) != ETH_ALEN)
|
|
|
|
return -EINVAL;
|
|
|
|
if (!is_valid_ether_addr(nla_data(tb[IFLA_ADDRESS])))
|
|
|
|
return -EADDRNOTAVAIL;
|
|
|
|
}
|
2009-03-04 07:36:04 +00:00
|
|
|
if (tb[IFLA_MTU]) {
|
|
|
|
if (!is_valid_veth_mtu(nla_get_u32(tb[IFLA_MTU])))
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
2007-09-25 23:14:46 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct rtnl_link_ops veth_link_ops;
|
|
|
|
|
2021-04-09 11:04:38 +00:00
|
|
|
static void veth_disable_gro(struct net_device *dev)
|
|
|
|
{
|
|
|
|
dev->features &= ~NETIF_F_GRO;
|
|
|
|
dev->wanted_features &= ~NETIF_F_GRO;
|
|
|
|
netdev_update_features(dev);
|
|
|
|
}
|
|
|
|
|
2021-07-20 08:41:51 +00:00
|
|
|
static int veth_init_queues(struct net_device *dev, struct nlattr *tb[])
|
|
|
|
{
|
|
|
|
int err;
|
|
|
|
|
|
|
|
if (!tb[IFLA_NUM_TX_QUEUES] && dev->num_tx_queues > 1) {
|
|
|
|
err = netif_set_real_num_tx_queues(dev, 1);
|
|
|
|
if (err)
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
if (!tb[IFLA_NUM_RX_QUEUES] && dev->num_rx_queues > 1) {
|
|
|
|
err = netif_set_real_num_rx_queues(dev, 1);
|
|
|
|
if (err)
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2009-11-08 08:53:51 +00:00
|
|
|
static int veth_newlink(struct net *src_net, struct net_device *dev,
|
2017-06-25 21:55:59 +00:00
|
|
|
struct nlattr *tb[], struct nlattr *data[],
|
|
|
|
struct netlink_ext_ack *extack)
|
2007-09-25 23:14:46 +00:00
|
|
|
{
|
2018-08-15 08:07:29 +00:00
|
|
|
int err;
|
2007-09-25 23:14:46 +00:00
|
|
|
struct net_device *peer;
|
|
|
|
struct veth_priv *priv;
|
|
|
|
char ifname[IFNAMSIZ];
|
|
|
|
struct nlattr *peer_tb[IFLA_MAX + 1], **tbp;
|
2014-07-14 14:37:25 +00:00
|
|
|
unsigned char name_assign_type;
|
rtnetlink: support specifying device flags on device creation
commit e8469ed959c373c2ff9e6f488aa5a14971aebe1f
Author: Patrick McHardy <kaber@trash.net>
Date: Tue Feb 23 20:41:30 2010 +0100
Support specifying the initial device flags when creating a device though
rtnl_link. Devices allocated by rtnl_create_link() are marked as INITIALIZING
in order to surpress netlink registration notifications. To complete setup,
rtnl_configure_link() must be called, which performs the device flag changes
and invokes the deferred notifiers if everything went well.
Two examples:
# add macvlan to eth0
#
$ ip link add link eth0 up allmulticast on type macvlan
[LINK]11: macvlan0@eth0: <BROADCAST,MULTICAST,ALLMULTI,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN
link/ether 26:f8:84:02:f9:2a brd ff:ff:ff:ff:ff:ff
[ROUTE]ff00::/8 dev macvlan0 table local metric 256 mtu 1500 advmss 1440 hoplimit 0
[ROUTE]fe80::/64 dev macvlan0 proto kernel metric 256 mtu 1500 advmss 1440 hoplimit 0
[LINK]11: macvlan0@eth0: <BROADCAST,MULTICAST,ALLMULTI,UP,LOWER_UP> mtu 1500
link/ether 26:f8:84:02:f9:2a
[ADDR]11: macvlan0 inet6 fe80::24f8:84ff:fe02:f92a/64 scope link
valid_lft forever preferred_lft forever
[ROUTE]local fe80::24f8:84ff:fe02:f92a via :: dev lo table local proto none metric 0 mtu 16436 advmss 16376 hoplimit 0
[ROUTE]default via fe80::215:e9ff:fef0:10f8 dev macvlan0 proto kernel metric 1024 mtu 1500 advmss 1440 hoplimit 0
[NEIGH]fe80::215:e9ff:fef0:10f8 dev macvlan0 lladdr 00:15:e9:f0:10:f8 router STALE
[ROUTE]2001:6f8:974::/64 dev macvlan0 proto kernel metric 256 expires 0sec mtu 1500 advmss 1440 hoplimit 0
[PREFIX]prefix 2001:6f8:974::/64 dev macvlan0 onlink autoconf valid 14400 preferred 131084
[ADDR]11: macvlan0 inet6 2001:6f8:974:0:24f8:84ff:fe02:f92a/64 scope global dynamic
valid_lft 86399sec preferred_lft 14399sec
# add VLAN to eth1, eth1 is down
#
$ ip link add link eth1 up type vlan id 1000
RTNETLINK answers: Network is down
<no events>
Signed-off-by: Patrick McHardy <kaber@trash.net>
Signed-off-by: David S. Miller <davem@davemloft.net>
2010-02-26 06:34:54 +00:00
|
|
|
struct ifinfomsg *ifmp;
|
2009-11-08 08:53:51 +00:00
|
|
|
struct net *net;
|
2007-09-25 23:14:46 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* create and register peer first
|
|
|
|
*/
|
|
|
|
if (data != NULL && data[VETH_INFO_PEER] != NULL) {
|
|
|
|
struct nlattr *nla_peer;
|
|
|
|
|
|
|
|
nla_peer = data[VETH_INFO_PEER];
|
rtnetlink: support specifying device flags on device creation
commit e8469ed959c373c2ff9e6f488aa5a14971aebe1f
Author: Patrick McHardy <kaber@trash.net>
Date: Tue Feb 23 20:41:30 2010 +0100
Support specifying the initial device flags when creating a device though
rtnl_link. Devices allocated by rtnl_create_link() are marked as INITIALIZING
in order to surpress netlink registration notifications. To complete setup,
rtnl_configure_link() must be called, which performs the device flag changes
and invokes the deferred notifiers if everything went well.
Two examples:
# add macvlan to eth0
#
$ ip link add link eth0 up allmulticast on type macvlan
[LINK]11: macvlan0@eth0: <BROADCAST,MULTICAST,ALLMULTI,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN
link/ether 26:f8:84:02:f9:2a brd ff:ff:ff:ff:ff:ff
[ROUTE]ff00::/8 dev macvlan0 table local metric 256 mtu 1500 advmss 1440 hoplimit 0
[ROUTE]fe80::/64 dev macvlan0 proto kernel metric 256 mtu 1500 advmss 1440 hoplimit 0
[LINK]11: macvlan0@eth0: <BROADCAST,MULTICAST,ALLMULTI,UP,LOWER_UP> mtu 1500
link/ether 26:f8:84:02:f9:2a
[ADDR]11: macvlan0 inet6 fe80::24f8:84ff:fe02:f92a/64 scope link
valid_lft forever preferred_lft forever
[ROUTE]local fe80::24f8:84ff:fe02:f92a via :: dev lo table local proto none metric 0 mtu 16436 advmss 16376 hoplimit 0
[ROUTE]default via fe80::215:e9ff:fef0:10f8 dev macvlan0 proto kernel metric 1024 mtu 1500 advmss 1440 hoplimit 0
[NEIGH]fe80::215:e9ff:fef0:10f8 dev macvlan0 lladdr 00:15:e9:f0:10:f8 router STALE
[ROUTE]2001:6f8:974::/64 dev macvlan0 proto kernel metric 256 expires 0sec mtu 1500 advmss 1440 hoplimit 0
[PREFIX]prefix 2001:6f8:974::/64 dev macvlan0 onlink autoconf valid 14400 preferred 131084
[ADDR]11: macvlan0 inet6 2001:6f8:974:0:24f8:84ff:fe02:f92a/64 scope global dynamic
valid_lft 86399sec preferred_lft 14399sec
# add VLAN to eth1, eth1 is down
#
$ ip link add link eth1 up type vlan id 1000
RTNETLINK answers: Network is down
<no events>
Signed-off-by: Patrick McHardy <kaber@trash.net>
Signed-off-by: David S. Miller <davem@davemloft.net>
2010-02-26 06:34:54 +00:00
|
|
|
ifmp = nla_data(nla_peer);
|
2014-02-18 19:53:18 +00:00
|
|
|
err = rtnl_nla_parse_ifla(peer_tb,
|
|
|
|
nla_data(nla_peer) + sizeof(struct ifinfomsg),
|
2017-04-12 12:34:07 +00:00
|
|
|
nla_len(nla_peer) - sizeof(struct ifinfomsg),
|
|
|
|
NULL);
|
2007-09-25 23:14:46 +00:00
|
|
|
if (err < 0)
|
|
|
|
return err;
|
|
|
|
|
2017-06-25 21:56:01 +00:00
|
|
|
err = veth_validate(peer_tb, NULL, extack);
|
2007-09-25 23:14:46 +00:00
|
|
|
if (err < 0)
|
|
|
|
return err;
|
|
|
|
|
|
|
|
tbp = peer_tb;
|
rtnetlink: support specifying device flags on device creation
commit e8469ed959c373c2ff9e6f488aa5a14971aebe1f
Author: Patrick McHardy <kaber@trash.net>
Date: Tue Feb 23 20:41:30 2010 +0100
Support specifying the initial device flags when creating a device though
rtnl_link. Devices allocated by rtnl_create_link() are marked as INITIALIZING
in order to surpress netlink registration notifications. To complete setup,
rtnl_configure_link() must be called, which performs the device flag changes
and invokes the deferred notifiers if everything went well.
Two examples:
# add macvlan to eth0
#
$ ip link add link eth0 up allmulticast on type macvlan
[LINK]11: macvlan0@eth0: <BROADCAST,MULTICAST,ALLMULTI,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN
link/ether 26:f8:84:02:f9:2a brd ff:ff:ff:ff:ff:ff
[ROUTE]ff00::/8 dev macvlan0 table local metric 256 mtu 1500 advmss 1440 hoplimit 0
[ROUTE]fe80::/64 dev macvlan0 proto kernel metric 256 mtu 1500 advmss 1440 hoplimit 0
[LINK]11: macvlan0@eth0: <BROADCAST,MULTICAST,ALLMULTI,UP,LOWER_UP> mtu 1500
link/ether 26:f8:84:02:f9:2a
[ADDR]11: macvlan0 inet6 fe80::24f8:84ff:fe02:f92a/64 scope link
valid_lft forever preferred_lft forever
[ROUTE]local fe80::24f8:84ff:fe02:f92a via :: dev lo table local proto none metric 0 mtu 16436 advmss 16376 hoplimit 0
[ROUTE]default via fe80::215:e9ff:fef0:10f8 dev macvlan0 proto kernel metric 1024 mtu 1500 advmss 1440 hoplimit 0
[NEIGH]fe80::215:e9ff:fef0:10f8 dev macvlan0 lladdr 00:15:e9:f0:10:f8 router STALE
[ROUTE]2001:6f8:974::/64 dev macvlan0 proto kernel metric 256 expires 0sec mtu 1500 advmss 1440 hoplimit 0
[PREFIX]prefix 2001:6f8:974::/64 dev macvlan0 onlink autoconf valid 14400 preferred 131084
[ADDR]11: macvlan0 inet6 2001:6f8:974:0:24f8:84ff:fe02:f92a/64 scope global dynamic
valid_lft 86399sec preferred_lft 14399sec
# add VLAN to eth1, eth1 is down
#
$ ip link add link eth1 up type vlan id 1000
RTNETLINK answers: Network is down
<no events>
Signed-off-by: Patrick McHardy <kaber@trash.net>
Signed-off-by: David S. Miller <davem@davemloft.net>
2010-02-26 06:34:54 +00:00
|
|
|
} else {
|
|
|
|
ifmp = NULL;
|
2007-09-25 23:14:46 +00:00
|
|
|
tbp = tb;
|
rtnetlink: support specifying device flags on device creation
commit e8469ed959c373c2ff9e6f488aa5a14971aebe1f
Author: Patrick McHardy <kaber@trash.net>
Date: Tue Feb 23 20:41:30 2010 +0100
Support specifying the initial device flags when creating a device though
rtnl_link. Devices allocated by rtnl_create_link() are marked as INITIALIZING
in order to surpress netlink registration notifications. To complete setup,
rtnl_configure_link() must be called, which performs the device flag changes
and invokes the deferred notifiers if everything went well.
Two examples:
# add macvlan to eth0
#
$ ip link add link eth0 up allmulticast on type macvlan
[LINK]11: macvlan0@eth0: <BROADCAST,MULTICAST,ALLMULTI,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN
link/ether 26:f8:84:02:f9:2a brd ff:ff:ff:ff:ff:ff
[ROUTE]ff00::/8 dev macvlan0 table local metric 256 mtu 1500 advmss 1440 hoplimit 0
[ROUTE]fe80::/64 dev macvlan0 proto kernel metric 256 mtu 1500 advmss 1440 hoplimit 0
[LINK]11: macvlan0@eth0: <BROADCAST,MULTICAST,ALLMULTI,UP,LOWER_UP> mtu 1500
link/ether 26:f8:84:02:f9:2a
[ADDR]11: macvlan0 inet6 fe80::24f8:84ff:fe02:f92a/64 scope link
valid_lft forever preferred_lft forever
[ROUTE]local fe80::24f8:84ff:fe02:f92a via :: dev lo table local proto none metric 0 mtu 16436 advmss 16376 hoplimit 0
[ROUTE]default via fe80::215:e9ff:fef0:10f8 dev macvlan0 proto kernel metric 1024 mtu 1500 advmss 1440 hoplimit 0
[NEIGH]fe80::215:e9ff:fef0:10f8 dev macvlan0 lladdr 00:15:e9:f0:10:f8 router STALE
[ROUTE]2001:6f8:974::/64 dev macvlan0 proto kernel metric 256 expires 0sec mtu 1500 advmss 1440 hoplimit 0
[PREFIX]prefix 2001:6f8:974::/64 dev macvlan0 onlink autoconf valid 14400 preferred 131084
[ADDR]11: macvlan0 inet6 2001:6f8:974:0:24f8:84ff:fe02:f92a/64 scope global dynamic
valid_lft 86399sec preferred_lft 14399sec
# add VLAN to eth1, eth1 is down
#
$ ip link add link eth1 up type vlan id 1000
RTNETLINK answers: Network is down
<no events>
Signed-off-by: Patrick McHardy <kaber@trash.net>
Signed-off-by: David S. Miller <davem@davemloft.net>
2010-02-26 06:34:54 +00:00
|
|
|
}
|
2007-09-25 23:14:46 +00:00
|
|
|
|
2017-06-21 09:12:24 +00:00
|
|
|
if (ifmp && tbp[IFLA_IFNAME]) {
|
2020-11-15 17:08:06 +00:00
|
|
|
nla_strscpy(ifname, tbp[IFLA_IFNAME], IFNAMSIZ);
|
2014-07-14 14:37:25 +00:00
|
|
|
name_assign_type = NET_NAME_USER;
|
|
|
|
} else {
|
2007-09-25 23:14:46 +00:00
|
|
|
snprintf(ifname, IFNAMSIZ, DRV_NAME "%%d");
|
2014-07-14 14:37:25 +00:00
|
|
|
name_assign_type = NET_NAME_ENUM;
|
|
|
|
}
|
2007-09-25 23:14:46 +00:00
|
|
|
|
2009-11-08 08:53:51 +00:00
|
|
|
net = rtnl_link_get_net(src_net, tbp);
|
|
|
|
if (IS_ERR(net))
|
|
|
|
return PTR_ERR(net);
|
|
|
|
|
2014-07-14 14:37:25 +00:00
|
|
|
peer = rtnl_create_link(net, ifname, name_assign_type,
|
2018-11-06 20:51:14 +00:00
|
|
|
&veth_link_ops, tbp, extack);
|
2009-11-08 08:53:51 +00:00
|
|
|
if (IS_ERR(peer)) {
|
|
|
|
put_net(net);
|
2007-09-25 23:14:46 +00:00
|
|
|
return PTR_ERR(peer);
|
2009-11-08 08:53:51 +00:00
|
|
|
}
|
2007-09-25 23:14:46 +00:00
|
|
|
|
2017-06-21 09:12:24 +00:00
|
|
|
if (!ifmp || !tbp[IFLA_ADDRESS])
|
2012-02-15 06:45:39 +00:00
|
|
|
eth_hw_addr_random(peer);
|
2012-08-08 21:53:03 +00:00
|
|
|
|
|
|
|
if (ifmp && (dev->ifindex != 0))
|
|
|
|
peer->ifindex = ifmp->ifi_index;
|
2007-09-25 23:14:46 +00:00
|
|
|
|
2021-11-19 15:43:31 +00:00
|
|
|
netif_set_gso_max_size(peer, dev->gso_max_size);
|
2021-11-19 15:43:32 +00:00
|
|
|
netif_set_gso_max_segs(peer, dev->gso_max_segs);
|
2017-12-07 23:40:20 +00:00
|
|
|
|
2007-09-25 23:14:46 +00:00
|
|
|
err = register_netdevice(peer);
|
2009-11-08 08:53:51 +00:00
|
|
|
put_net(net);
|
|
|
|
net = NULL;
|
2007-09-25 23:14:46 +00:00
|
|
|
if (err < 0)
|
|
|
|
goto err_register_peer;
|
|
|
|
|
2021-04-09 11:04:38 +00:00
|
|
|
/* keep GRO disabled by default to be consistent with the established
|
|
|
|
* veth behavior
|
|
|
|
*/
|
|
|
|
veth_disable_gro(peer);
|
2007-09-25 23:14:46 +00:00
|
|
|
netif_carrier_off(peer);
|
|
|
|
|
rtnetlink: support specifying device flags on device creation
commit e8469ed959c373c2ff9e6f488aa5a14971aebe1f
Author: Patrick McHardy <kaber@trash.net>
Date: Tue Feb 23 20:41:30 2010 +0100
Support specifying the initial device flags when creating a device though
rtnl_link. Devices allocated by rtnl_create_link() are marked as INITIALIZING
in order to surpress netlink registration notifications. To complete setup,
rtnl_configure_link() must be called, which performs the device flag changes
and invokes the deferred notifiers if everything went well.
Two examples:
# add macvlan to eth0
#
$ ip link add link eth0 up allmulticast on type macvlan
[LINK]11: macvlan0@eth0: <BROADCAST,MULTICAST,ALLMULTI,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN
link/ether 26:f8:84:02:f9:2a brd ff:ff:ff:ff:ff:ff
[ROUTE]ff00::/8 dev macvlan0 table local metric 256 mtu 1500 advmss 1440 hoplimit 0
[ROUTE]fe80::/64 dev macvlan0 proto kernel metric 256 mtu 1500 advmss 1440 hoplimit 0
[LINK]11: macvlan0@eth0: <BROADCAST,MULTICAST,ALLMULTI,UP,LOWER_UP> mtu 1500
link/ether 26:f8:84:02:f9:2a
[ADDR]11: macvlan0 inet6 fe80::24f8:84ff:fe02:f92a/64 scope link
valid_lft forever preferred_lft forever
[ROUTE]local fe80::24f8:84ff:fe02:f92a via :: dev lo table local proto none metric 0 mtu 16436 advmss 16376 hoplimit 0
[ROUTE]default via fe80::215:e9ff:fef0:10f8 dev macvlan0 proto kernel metric 1024 mtu 1500 advmss 1440 hoplimit 0
[NEIGH]fe80::215:e9ff:fef0:10f8 dev macvlan0 lladdr 00:15:e9:f0:10:f8 router STALE
[ROUTE]2001:6f8:974::/64 dev macvlan0 proto kernel metric 256 expires 0sec mtu 1500 advmss 1440 hoplimit 0
[PREFIX]prefix 2001:6f8:974::/64 dev macvlan0 onlink autoconf valid 14400 preferred 131084
[ADDR]11: macvlan0 inet6 2001:6f8:974:0:24f8:84ff:fe02:f92a/64 scope global dynamic
valid_lft 86399sec preferred_lft 14399sec
# add VLAN to eth1, eth1 is down
#
$ ip link add link eth1 up type vlan id 1000
RTNETLINK answers: Network is down
<no events>
Signed-off-by: Patrick McHardy <kaber@trash.net>
Signed-off-by: David S. Miller <davem@davemloft.net>
2010-02-26 06:34:54 +00:00
|
|
|
err = rtnl_configure_link(peer, ifmp);
|
|
|
|
if (err < 0)
|
|
|
|
goto err_configure_peer;
|
|
|
|
|
2007-09-25 23:14:46 +00:00
|
|
|
/*
|
|
|
|
* register dev last
|
|
|
|
*
|
|
|
|
* note, that since we've registered new device the dev's name
|
|
|
|
* should be re-allocated
|
|
|
|
*/
|
|
|
|
|
|
|
|
if (tb[IFLA_ADDRESS] == NULL)
|
2012-02-15 06:45:39 +00:00
|
|
|
eth_hw_addr_random(dev);
|
2007-09-25 23:14:46 +00:00
|
|
|
|
2011-04-30 01:28:17 +00:00
|
|
|
if (tb[IFLA_IFNAME])
|
2020-11-15 17:08:06 +00:00
|
|
|
nla_strscpy(dev->name, tb[IFLA_IFNAME], IFNAMSIZ);
|
2011-04-30 01:28:17 +00:00
|
|
|
else
|
|
|
|
snprintf(dev->name, IFNAMSIZ, DRV_NAME "%%d");
|
|
|
|
|
2007-09-25 23:14:46 +00:00
|
|
|
err = register_netdevice(dev);
|
|
|
|
if (err < 0)
|
|
|
|
goto err_register_dev;
|
|
|
|
|
|
|
|
netif_carrier_off(dev);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* tie the deviced together
|
|
|
|
*/
|
|
|
|
|
|
|
|
priv = netdev_priv(dev);
|
2013-01-04 15:42:40 +00:00
|
|
|
rcu_assign_pointer(priv->peer, peer);
|
2021-07-20 08:41:51 +00:00
|
|
|
err = veth_init_queues(dev, tb);
|
|
|
|
if (err)
|
|
|
|
goto err_queues;
|
2007-09-25 23:14:46 +00:00
|
|
|
|
|
|
|
priv = netdev_priv(peer);
|
2013-01-04 15:42:40 +00:00
|
|
|
rcu_assign_pointer(priv->peer, dev);
|
2021-07-20 08:41:51 +00:00
|
|
|
err = veth_init_queues(peer, tb);
|
|
|
|
if (err)
|
|
|
|
goto err_queues;
|
2018-08-03 07:58:10 +00:00
|
|
|
|
2021-04-09 11:04:38 +00:00
|
|
|
veth_disable_gro(dev);
|
2007-09-25 23:14:46 +00:00
|
|
|
return 0;
|
|
|
|
|
2021-07-20 08:41:51 +00:00
|
|
|
err_queues:
|
|
|
|
unregister_netdevice(dev);
|
2007-09-25 23:14:46 +00:00
|
|
|
err_register_dev:
|
|
|
|
/* nothing to do */
|
rtnetlink: support specifying device flags on device creation
commit e8469ed959c373c2ff9e6f488aa5a14971aebe1f
Author: Patrick McHardy <kaber@trash.net>
Date: Tue Feb 23 20:41:30 2010 +0100
Support specifying the initial device flags when creating a device though
rtnl_link. Devices allocated by rtnl_create_link() are marked as INITIALIZING
in order to surpress netlink registration notifications. To complete setup,
rtnl_configure_link() must be called, which performs the device flag changes
and invokes the deferred notifiers if everything went well.
Two examples:
# add macvlan to eth0
#
$ ip link add link eth0 up allmulticast on type macvlan
[LINK]11: macvlan0@eth0: <BROADCAST,MULTICAST,ALLMULTI,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN
link/ether 26:f8:84:02:f9:2a brd ff:ff:ff:ff:ff:ff
[ROUTE]ff00::/8 dev macvlan0 table local metric 256 mtu 1500 advmss 1440 hoplimit 0
[ROUTE]fe80::/64 dev macvlan0 proto kernel metric 256 mtu 1500 advmss 1440 hoplimit 0
[LINK]11: macvlan0@eth0: <BROADCAST,MULTICAST,ALLMULTI,UP,LOWER_UP> mtu 1500
link/ether 26:f8:84:02:f9:2a
[ADDR]11: macvlan0 inet6 fe80::24f8:84ff:fe02:f92a/64 scope link
valid_lft forever preferred_lft forever
[ROUTE]local fe80::24f8:84ff:fe02:f92a via :: dev lo table local proto none metric 0 mtu 16436 advmss 16376 hoplimit 0
[ROUTE]default via fe80::215:e9ff:fef0:10f8 dev macvlan0 proto kernel metric 1024 mtu 1500 advmss 1440 hoplimit 0
[NEIGH]fe80::215:e9ff:fef0:10f8 dev macvlan0 lladdr 00:15:e9:f0:10:f8 router STALE
[ROUTE]2001:6f8:974::/64 dev macvlan0 proto kernel metric 256 expires 0sec mtu 1500 advmss 1440 hoplimit 0
[PREFIX]prefix 2001:6f8:974::/64 dev macvlan0 onlink autoconf valid 14400 preferred 131084
[ADDR]11: macvlan0 inet6 2001:6f8:974:0:24f8:84ff:fe02:f92a/64 scope global dynamic
valid_lft 86399sec preferred_lft 14399sec
# add VLAN to eth1, eth1 is down
#
$ ip link add link eth1 up type vlan id 1000
RTNETLINK answers: Network is down
<no events>
Signed-off-by: Patrick McHardy <kaber@trash.net>
Signed-off-by: David S. Miller <davem@davemloft.net>
2010-02-26 06:34:54 +00:00
|
|
|
err_configure_peer:
|
2007-09-25 23:14:46 +00:00
|
|
|
unregister_netdevice(peer);
|
|
|
|
return err;
|
|
|
|
|
|
|
|
err_register_peer:
|
|
|
|
free_netdev(peer);
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
2009-10-27 07:06:36 +00:00
|
|
|
static void veth_dellink(struct net_device *dev, struct list_head *head)
|
2007-09-25 23:14:46 +00:00
|
|
|
{
|
|
|
|
struct veth_priv *priv;
|
|
|
|
struct net_device *peer;
|
|
|
|
|
|
|
|
priv = netdev_priv(dev);
|
2013-01-04 15:42:40 +00:00
|
|
|
peer = rtnl_dereference(priv->peer);
|
|
|
|
|
|
|
|
/* Note : dellink() is called from default_device_exit_batch(),
|
|
|
|
* before a rcu_synchronize() point. The devices are guaranteed
|
|
|
|
* not being freed before one RCU grace period.
|
|
|
|
*/
|
|
|
|
RCU_INIT_POINTER(priv->peer, NULL);
|
2009-10-30 08:00:27 +00:00
|
|
|
unregister_netdevice_queue(dev, head);
|
2013-02-08 20:10:49 +00:00
|
|
|
|
|
|
|
if (peer) {
|
|
|
|
priv = netdev_priv(peer);
|
|
|
|
RCU_INIT_POINTER(priv->peer, NULL);
|
|
|
|
unregister_netdevice_queue(peer, head);
|
|
|
|
}
|
2007-09-25 23:14:46 +00:00
|
|
|
}
|
|
|
|
|
2012-02-15 04:09:46 +00:00
|
|
|
static const struct nla_policy veth_policy[VETH_INFO_MAX + 1] = {
|
|
|
|
[VETH_INFO_PEER] = { .len = sizeof(struct ifinfomsg) },
|
|
|
|
};
|
2007-09-25 23:14:46 +00:00
|
|
|
|
2015-01-20 14:15:46 +00:00
|
|
|
static struct net *veth_get_link_net(const struct net_device *dev)
|
|
|
|
{
|
|
|
|
struct veth_priv *priv = netdev_priv(dev);
|
|
|
|
struct net_device *peer = rtnl_dereference(priv->peer);
|
|
|
|
|
|
|
|
return peer ? dev_net(peer) : dev_net(dev);
|
|
|
|
}
|
|
|
|
|
2021-07-20 08:41:51 +00:00
|
|
|
static unsigned int veth_get_num_queues(void)
|
|
|
|
{
|
|
|
|
/* enforce the same queue limit as rtnl_create_link */
|
|
|
|
int queues = num_possible_cpus();
|
|
|
|
|
|
|
|
if (queues > 4096)
|
|
|
|
queues = 4096;
|
|
|
|
return queues;
|
|
|
|
}
|
|
|
|
|
2007-09-25 23:14:46 +00:00
|
|
|
static struct rtnl_link_ops veth_link_ops = {
|
|
|
|
.kind = DRV_NAME,
|
|
|
|
.priv_size = sizeof(struct veth_priv),
|
|
|
|
.setup = veth_setup,
|
|
|
|
.validate = veth_validate,
|
|
|
|
.newlink = veth_newlink,
|
|
|
|
.dellink = veth_dellink,
|
|
|
|
.policy = veth_policy,
|
|
|
|
.maxtype = VETH_INFO_MAX,
|
2015-01-20 14:15:46 +00:00
|
|
|
.get_link_net = veth_get_link_net,
|
2021-07-20 08:41:51 +00:00
|
|
|
.get_num_tx_queues = veth_get_num_queues,
|
|
|
|
.get_num_rx_queues = veth_get_num_queues,
|
2007-09-25 23:14:46 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
/*
|
|
|
|
* init/fini
|
|
|
|
*/
|
|
|
|
|
|
|
|
static __init int veth_init(void)
|
|
|
|
{
|
|
|
|
return rtnl_link_register(&veth_link_ops);
|
|
|
|
}
|
|
|
|
|
|
|
|
static __exit void veth_exit(void)
|
|
|
|
{
|
2008-01-21 01:25:14 +00:00
|
|
|
rtnl_link_unregister(&veth_link_ops);
|
2007-09-25 23:14:46 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
module_init(veth_init);
|
|
|
|
module_exit(veth_exit);
|
|
|
|
|
|
|
|
MODULE_DESCRIPTION("Virtual Ethernet Tunnel");
|
|
|
|
MODULE_LICENSE("GPL v2");
|
|
|
|
MODULE_ALIAS_RTNL_LINK(DRV_NAME);
|